Skip to content

Commit 1504eaa

Browse files
author
Chris Elder
committed
FAB-2010 CouchDB Query Data Wrapper fails
Motivation for this change: The data query wrapper used to "wrap" CouchDB queries for the ledger will fail for some instances of numbers and for complex queries using cascading operators. - Simplify query_wrapper.go to simplify and remove duplicate code - Add unit tests (query_wrapper_test.go) for test the query wrapper - Extend query tests in test_common.go for additional query tests Change-Id: I9d075871a29173a1220b8af721a714ab83e22906 Signed-off-by: Chris Elder <[email protected]>
1 parent 5090331 commit 1504eaa

File tree

4 files changed

+473
-96
lines changed

4 files changed

+473
-96
lines changed

core/ledger/kvledger/txmgmt/statedb/commontests/test_common.go

+68-7
Original file line numberDiff line numberDiff line change
@@ -183,11 +183,27 @@ func TestQuery(t *testing.T, dbProvider statedb.VersionedDBProvider) {
183183
db.Open()
184184
defer db.Close()
185185
batch := statedb.NewUpdateBatch()
186-
jsonValue1 := "{\"asset_name\": \"marble1\",\"color\": \"blue\",\"size\": 35,\"owner\": \"tom\"}"
186+
jsonValue1 := "{\"asset_name\": \"marble1\",\"color\": \"blue\",\"size\": 1,\"owner\": \"tom\"}"
187187
batch.Put("ns1", "key1", []byte(jsonValue1), version.NewHeight(1, 1))
188-
jsonValue2 := "{\"asset_name\": \"marble1\",\"color\": \"blue\",\"size\": 35,\"owner\": \"jerry\"}"
188+
jsonValue2 := "{\"asset_name\": \"marble2\",\"color\": \"blue\",\"size\": 2,\"owner\": \"jerry\"}"
189189
batch.Put("ns1", "key2", []byte(jsonValue2), version.NewHeight(1, 2))
190-
savePoint := version.NewHeight(2, 5)
190+
jsonValue3 := "{\"asset_name\": \"marble3\",\"color\": \"blue\",\"size\": 3,\"owner\": \"fred\"}"
191+
batch.Put("ns1", "key3", []byte(jsonValue3), version.NewHeight(1, 3))
192+
jsonValue4 := "{\"asset_name\": \"marble4\",\"color\": \"blue\",\"size\": 4,\"owner\": \"martha\"}"
193+
batch.Put("ns1", "key4", []byte(jsonValue4), version.NewHeight(1, 4))
194+
jsonValue5 := "{\"asset_name\": \"marble5\",\"color\": \"blue\",\"size\": 5,\"owner\": \"fred\"}"
195+
batch.Put("ns1", "key5", []byte(jsonValue5), version.NewHeight(1, 5))
196+
jsonValue6 := "{\"asset_name\": \"marble6\",\"color\": \"blue\",\"size\": 6,\"owner\": \"elaine\"}"
197+
batch.Put("ns1", "key6", []byte(jsonValue6), version.NewHeight(1, 6))
198+
jsonValue7 := "{\"asset_name\": \"marble7\",\"color\": \"blue\",\"size\": 7,\"owner\": \"fred\"}"
199+
batch.Put("ns1", "key7", []byte(jsonValue7), version.NewHeight(1, 7))
200+
jsonValue8 := "{\"asset_name\": \"marble8\",\"color\": \"blue\",\"size\": 8,\"owner\": \"elaine\"}"
201+
batch.Put("ns1", "key8", []byte(jsonValue8), version.NewHeight(1, 8))
202+
jsonValue9 := "{\"asset_name\": \"marble9\",\"color\": \"green\",\"size\": 9,\"owner\": \"fred\"}"
203+
batch.Put("ns1", "key9", []byte(jsonValue9), version.NewHeight(1, 9))
204+
jsonValue10 := "{\"asset_name\": \"marble10\",\"color\": \"green\",\"size\": 10,\"owner\": \"mary\"}"
205+
batch.Put("ns1", "key10", []byte(jsonValue10), version.NewHeight(1, 12))
206+
savePoint := version.NewHeight(2, 12)
191207
db.ApplyUpdates(batch, savePoint)
192208

193209
// query for owner=jerry
@@ -200,8 +216,8 @@ func TestQuery(t *testing.T, dbProvider statedb.VersionedDBProvider) {
200216
testutil.AssertNotNil(t, queryResult1)
201217
versionedQueryRecord := queryResult1.(*statedb.VersionedQueryRecord)
202218
stringRecord := string(versionedQueryRecord.Record)
203-
bFoundJerry := strings.Contains(stringRecord, "jerry")
204-
testutil.AssertEquals(t, bFoundJerry, true)
219+
bFoundRecord := strings.Contains(stringRecord, "jerry")
220+
testutil.AssertEquals(t, bFoundRecord, true)
205221

206222
// verify no more results
207223
queryResult2, err := itr.Next()
@@ -231,12 +247,57 @@ func TestQuery(t *testing.T, dbProvider statedb.VersionedDBProvider) {
231247
testutil.AssertNotNil(t, queryResult1)
232248
versionedQueryRecord = queryResult1.(*statedb.VersionedQueryRecord)
233249
stringRecord = string(versionedQueryRecord.Record)
234-
bFoundJerry = strings.Contains(stringRecord, "jerry")
235-
testutil.AssertEquals(t, bFoundJerry, true)
250+
bFoundRecord = strings.Contains(stringRecord, "jerry")
251+
testutil.AssertEquals(t, bFoundRecord, true)
236252

237253
// verify no more results
238254
queryResult2, err = itr.Next()
239255
testutil.AssertNoError(t, err, "")
240256
testutil.AssertNil(t, queryResult2)
241257

258+
// query with complex selector
259+
itr, err = db.ExecuteQuery("{\"selector\":{\"$and\":[{\"size\":{\"$gt\": 5}},{\"size\":{\"$lt\":8}},{\"$not\":{\"size\":6}}]}}")
260+
testutil.AssertNoError(t, err, "")
261+
262+
// verify one fred result
263+
queryResult1, err = itr.Next()
264+
testutil.AssertNoError(t, err, "")
265+
testutil.AssertNotNil(t, queryResult1)
266+
versionedQueryRecord = queryResult1.(*statedb.VersionedQueryRecord)
267+
stringRecord = string(versionedQueryRecord.Record)
268+
bFoundRecord = strings.Contains(stringRecord, "fred")
269+
testutil.AssertEquals(t, bFoundRecord, true)
270+
271+
// verify no more results
272+
queryResult2, err = itr.Next()
273+
testutil.AssertNoError(t, err, "")
274+
testutil.AssertNil(t, queryResult2)
275+
276+
// query with embedded implicit "AND" and explicit "OR"
277+
itr, err = db.ExecuteQuery("{\"selector\":{\"color\":\"green\",\"$or\":[{\"owner\":\"fred\"},{\"owner\":\"mary\"}]}}")
278+
testutil.AssertNoError(t, err, "")
279+
280+
// verify one green result
281+
queryResult1, err = itr.Next()
282+
testutil.AssertNoError(t, err, "")
283+
testutil.AssertNotNil(t, queryResult1)
284+
versionedQueryRecord = queryResult1.(*statedb.VersionedQueryRecord)
285+
stringRecord = string(versionedQueryRecord.Record)
286+
bFoundRecord = strings.Contains(stringRecord, "green")
287+
testutil.AssertEquals(t, bFoundRecord, true)
288+
289+
// verify another green result
290+
queryResult2, err = itr.Next()
291+
testutil.AssertNoError(t, err, "")
292+
testutil.AssertNotNil(t, queryResult2)
293+
versionedQueryRecord = queryResult2.(*statedb.VersionedQueryRecord)
294+
stringRecord = string(versionedQueryRecord.Record)
295+
bFoundRecord = strings.Contains(stringRecord, "green")
296+
testutil.AssertEquals(t, bFoundRecord, true)
297+
298+
// verify no more results
299+
queryResult3, err = itr.Next()
300+
testutil.AssertNoError(t, err, "")
301+
testutil.AssertNil(t, queryResult3)
302+
242303
}

core/ledger/kvledger/txmgmt/statedb/statecouchdb/query_wrapper.go

+112-84
Original file line numberDiff line numberDiff line change
@@ -3,137 +3,165 @@ package statecouchdb
33
import (
44
"encoding/json"
55
"fmt"
6-
"strings"
6+
"reflect"
77
)
88

9-
//ApplyQueryWrapper parses the query string passed to CouchDB
10-
//the wrapper prepends the wrapper "data." to all fields specified in the query
11-
//All fields in the selector must have "data." prepended to the field names
12-
//Fields listed in fields key will have "data." prepended
13-
//Fields in the sort key will have "data." prepended
14-
func ApplyQueryWrapper(queryString string) []byte {
9+
var dataWrapper = "data"
10+
var jsonQueryFields = "fields"
1511

16-
//create a generic map for the query json
17-
jsonQuery := make(map[string]interface{})
12+
var validOperators = []string{"$and", "$or", "$not", "$nor", "$all", "$elemMatch",
13+
"$lt", "$lte", "$eq", "$ne", "$gte", "$gt", "$exits", "$type", "$in", "$nin",
14+
"$size", "$mod", "$regex"}
1815

19-
//unmarshal the selected json into the generic map
20-
json.Unmarshal([]byte(queryString), &jsonQuery)
16+
/*
17+
ApplyQueryWrapper parses the query string passed to CouchDB
18+
the wrapper prepends the wrapper "data." to all fields specified in the query
19+
All fields in the selector must have "data." prepended to the field names
20+
Fields listed in fields key will have "data." prepended
21+
Fields in the sort key will have "data." prepended
2122
22-
//iterate through the JSON query
23-
for jsonKey, jsonValue := range jsonQuery {
23+
Example:
2424
25-
//create a case for the data types found in the JSON query
26-
switch jsonQueryPart := jsonValue.(type) {
25+
Source Query:
26+
{"selector":{"owner": {"$eq": "tom"}},
27+
"fields": ["owner", "asset_name", "color", "size"],
28+
"sort": ["size", "color"], "limit": 10, "skip": 0}
2729
28-
//if the type is an array, then this is either the "fields" or "sort" part of the query
29-
case []interface{}:
30+
Result Wrapped Query:
31+
{"selector":{"data.owner":{"$eq":"tom"}},
32+
"fields": ["data.owner","data.asset_name","data.color","data.size","_id","version"],
33+
"sort":["data.size","data.color"],"limit":10,"skip":0}
3034
31-
//check to see if this is a "fields" or "sort" array
32-
//if jsonKey == jsonQueryFields || jsonKey == jsonQuerySort {
33-
if jsonKey == jsonQueryFields {
3435
35-
//iterate through the names and add the data wrapper for each field
36-
for itemKey, fieldName := range jsonQueryPart {
36+
*/
37+
func ApplyQueryWrapper(queryString string) string {
3738

38-
//add "data" wrapper to each field definition
39-
jsonQueryPart[itemKey] = fmt.Sprintf("%v.%v", dataWrapper, fieldName)
40-
}
39+
//create a generic map for the query json
40+
jsonQueryMap := make(map[string]interface{})
4141

42-
//Add the "_id" and "version" fields, these are needed by default
43-
if jsonKey == jsonQueryFields {
42+
//unmarshal the selected json into the generic map
43+
json.Unmarshal([]byte(queryString), &jsonQueryMap)
4444

45-
jsonQueryPart = append(jsonQueryPart, "_id")
46-
jsonQueryPart = append(jsonQueryPart, "version")
45+
//traverse through the json query and wrap any field names
46+
processAndWrapQuery(jsonQueryMap)
4747

48-
//Overwrite the query fields if the "_id" field has been added
49-
jsonQuery[jsonQueryFields] = jsonQueryPart
50-
}
48+
//process the query and add the version and fields if fields are specified
49+
for jsonKey, jsonValue := range jsonQueryMap {
50+
51+
//Add the "_id" and "version" fields, these are needed by default
52+
if jsonKey == jsonQueryFields {
5153

54+
//check to see if this is an interface map
55+
if reflect.TypeOf(jsonValue).String() == "[]interface {}" {
56+
57+
//Add the "_id" and "version" fields, these are needed by default
58+
//Overwrite the query fields if the "_id" field has been added
59+
jsonQueryMap[jsonQueryFields] = append(jsonValue.([]interface{}), "_id", "version")
5260
}
5361

54-
if jsonKey == jsonQuerySort {
62+
}
63+
}
64+
65+
//Marshal the updated json query
66+
editedQuery, _ := json.Marshal(jsonQueryMap)
67+
68+
logger.Debugf("Rewritten query with data wrapper: %s", editedQuery)
69+
70+
return string(editedQuery)
71+
72+
}
5573

56-
//iterate through the names and add the data wrapper for each field
57-
for sortItemKey, sortField := range jsonQueryPart {
74+
func processAndWrapQuery(jsonQueryMap map[string]interface{}) {
5875

59-
//create a case for the data types found in the JSON query
60-
switch sortFieldType := sortField.(type) {
76+
//iterate through the JSON query
77+
for _, jsonValue := range jsonQueryMap {
6178

62-
//if the type is string, then this is a simple array of field names.
63-
//Add the datawrapper to the field name
64-
case string:
79+
//create a case for the data types found in the JSON query
80+
switch jsonValueType := jsonValue.(type) {
6581

66-
//simple case, update the existing array item with the updated name
67-
jsonQueryPart[sortItemKey] = fmt.Sprintf("%v.%v", dataWrapper, sortField)
82+
case string:
83+
//intercept the string case and prevent the []interface{} case from
84+
//incorrectly processing the string
6885

69-
case interface{}:
86+
case float64:
87+
//intercept the float64 case and prevent the []interface{} case from
88+
//incorrectly processing the float64
7089

71-
//this case is a little more complicated. Here we need to
72-
//iterate over the mapped field names since this is an array of objects
73-
//example: {"fieldname":"desc"}
74-
for key, itemValue := range sortField.(map[string]interface{}) {
75-
//delete the mapping for the field definition, since we have to change the
76-
//value of the key
77-
delete(sortField.(map[string]interface{}), key)
90+
//if the type is an array, then iterate through the items
91+
case []interface{}:
7892

79-
//add the key back into the map with the field name wrapped with then "data" wrapper
80-
sortField.(map[string]interface{})[fmt.Sprintf("%v.%v", dataWrapper, key)] = itemValue
81-
}
93+
//iterate the the items in the array
94+
for itemKey, itemValue := range jsonValueType {
8295

83-
default:
96+
switch itemValue.(type) {
8497

85-
logger.Debugf("The type %v was not recognized as a valid sort field type.", sortFieldType)
98+
case string:
8699

87-
}
100+
//This is a simple string, so wrap the field and replace in the array
101+
jsonValueType[itemKey] = fmt.Sprintf("%v.%v", dataWrapper, itemValue)
88102

89-
}
90-
}
103+
case []interface{}:
91104

92-
case interface{}:
105+
//This is a array, so traverse to the next level
106+
processAndWrapQuery(itemValue.(map[string]interface{}))
93107

94-
//if this is the "selector", the field names need to be mapped with the
95-
//data wrapper
96-
if jsonKey == jsonQuerySelector {
108+
case interface{}:
97109

98-
processSelector(jsonQueryPart.(map[string]interface{}))
110+
//process this part as a map
111+
processInterfaceMap(itemValue.(map[string]interface{}))
99112

113+
}
100114
}
101115

102-
default:
116+
case interface{}:
103117

104-
logger.Debugf("The value %v was not recognized as a valid selector field.", jsonKey)
118+
//process this part as a map
119+
processInterfaceMap(jsonValue.(map[string]interface{}))
105120

106121
}
107122
}
123+
}
108124

109-
//Marshal the updated json query
110-
editedQuery, _ := json.Marshal(jsonQuery)
125+
//processInterfaceMap processes an interface map and wraps field names or traverses the next level of the json query
126+
func processInterfaceMap(jsonFragment map[string]interface{}) {
111127

112-
logger.Debugf("Rewritten query with data wrapper: %s", editedQuery)
128+
//iterate the the item in the map
129+
for keyVal, itemVal := range jsonFragment {
113130

114-
return editedQuery
131+
//check to see if the key is an operator
132+
if arrayContains(validOperators, keyVal) {
115133

116-
}
134+
//if this is an operator, traverse the next level of the json query
135+
processAndWrapQuery(jsonFragment)
117136

118-
//processSelector is a recursion function for traversing the selector part of the query
119-
func processSelector(selectorFragment map[string]interface{}) {
137+
} else {
120138

121-
//iterate through the top level definitions
122-
for itemKey, itemValue := range selectorFragment {
139+
//if this is not an operator, this is a field name and needs to be wrapped
140+
wrapFieldName(jsonFragment, keyVal, itemVal)
141+
142+
}
143+
}
144+
}
123145

124-
//check to see if the itemKey starts with a $. If so, this indicates an operator
125-
if strings.HasPrefix(fmt.Sprintf("%s", itemKey), "$") {
146+
//wrapFieldName "wraps" the field name with the data wrapper, and replaces the key in the json fragment
147+
func wrapFieldName(jsonFragment map[string]interface{}, key string, value interface{}) {
126148

127-
processSelector(itemValue.(map[string]interface{}))
149+
//delete the mapping for the field definition, since we have to change the
150+
//value of the key
151+
delete(jsonFragment, key)
128152

129-
} else {
153+
//add the key back into the map with the field name wrapped with then "data" wrapper
154+
jsonFragment[fmt.Sprintf("%v.%v", dataWrapper, key)] = value
130155

131-
//delete the mapping for the field definition, since we have to change the
132-
//value of the key
133-
delete(selectorFragment, itemKey)
134-
//add the key back into the map with the field name wrapped with then "data" wrapper
135-
selectorFragment[fmt.Sprintf("%v.%v", dataWrapper, itemKey)] = itemValue
156+
}
136157

137-
}
158+
//arrayContains is a function to detect if a soure array of strings contains the selected string
159+
//for this application, it is used to determine if a string is a valid CouchDB operator
160+
func arrayContains(sourceArray []string, selectItem string) bool {
161+
set := make(map[string]struct{}, len(sourceArray))
162+
for _, s := range sourceArray {
163+
set[s] = struct{}{}
138164
}
165+
_, ok := set[selectItem]
166+
return ok
139167
}

0 commit comments

Comments
 (0)