Skip to content

Commit 7984725

Browse files
author
Chris Elder
committed
FAB-3047 Change attachment arrays to use pointers
This is change 3 of 4 for FAB-2725 CouchDB optimizations Motivation for this change: Interactions with CouchDB are currently done individually. Need to switch to using bulk operations to get optimal performance from CouchDB. Need to performance test and stress test. - Change CouchDoc to use pointers for attachments Change-Id: I8dcbb29432073514c20033e4cdc1204ecbabc0ef Signed-off-by: Chris Elder <[email protected]>
1 parent a932b54 commit 7984725

File tree

3 files changed

+61
-56
lines changed

3 files changed

+61
-56
lines changed

‎core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func (vdb *VersionedDB) GetState(namespace string, key string) (*statedb.Version
132132
return &statedb.VersionedValue{Value: returnValue, Version: &returnVersion}, nil
133133
}
134134

135-
func removeDataWrapper(wrappedValue []byte, attachments []couchdb.Attachment) ([]byte, version.Height) {
135+
func removeDataWrapper(wrappedValue []byte, attachments []*couchdb.Attachment) ([]byte, version.Height) {
136136

137137
//initialize the return value
138138
returnValue := []byte{} // TODO: empty byte or nil
@@ -264,12 +264,14 @@ func (vdb *VersionedDB) ApplyUpdates(batch *statedb.UpdateBatch, height *version
264264
// Handle it as json
265265
couchDoc.JSONValue = addVersionAndChainCodeID(vv.Value, ns, vv.Version)
266266
} else { // if the data is not JSON, save as binary attachment in Couch
267-
//Create an attachment structure and load the bytes
267+
268268
attachment := &couchdb.Attachment{}
269269
attachment.AttachmentBytes = vv.Value
270270
attachment.ContentType = "application/octet-stream"
271271
attachment.Name = binaryWrapper
272-
couchDoc.Attachments = append(couchDoc.Attachments, *attachment)
272+
attachments := append([]*couchdb.Attachment{}, attachment)
273+
274+
couchDoc.Attachments = attachments
273275
couchDoc.JSONValue = addVersionAndChainCodeID(nil, ns, vv.Version)
274276
}
275277

‎core/ledger/util/couchdb/couchdb.go

+8-5
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ type DocID struct {
122122
type QueryResult struct {
123123
ID string
124124
Value []byte
125-
Attachments []Attachment
125+
Attachments []*Attachment
126126
}
127127

128128
//CouchConnectionDef contains parameters
@@ -175,7 +175,7 @@ type FileDetails struct {
175175
//CouchDoc defines the structure for a JSON document value
176176
type CouchDoc struct {
177177
JSONValue []byte
178-
Attachments []Attachment
178+
Attachments []*Attachment
179179
}
180180

181181
//BatchRetrieveDocMedatadataResponse is used for processing REST batch responses from CouchDB
@@ -597,6 +597,7 @@ func getRevisionHeader(resp *http.Response) (string, error) {
597597
//ReadDoc method provides function to retrieve a document from the database by id
598598
func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
599599
var couchDoc CouchDoc
600+
attachments := []*Attachment{}
600601

601602
logger.Debugf("Entering ReadDoc() id=[%s]", id)
602603
if !utf8.ValidString(id) {
@@ -668,7 +669,7 @@ func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
668669
default:
669670

670671
//Create an attachment structure and load it
671-
attachment := Attachment{}
672+
attachment := &Attachment{}
672673
attachment.ContentType = p.Header.Get("Content-Type")
673674
contentDispositionParts := strings.Split(p.Header.Get("Content-Disposition"), ";")
674675
if strings.TrimSpace(contentDispositionParts[0]) == "attachment" {
@@ -689,7 +690,7 @@ func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
689690
logger.Debugf("Retrieved attachment data")
690691
attachment.AttachmentBytes = respBody
691692
attachment.Name = p.FileName()
692-
couchDoc.Attachments = append(couchDoc.Attachments, attachment)
693+
attachments = append(attachments, attachment)
693694

694695
default:
695696

@@ -701,13 +702,15 @@ func (dbclient *CouchDatabase) ReadDoc(id string) (*CouchDoc, string, error) {
701702
logger.Debugf("Retrieved attachment data")
702703
attachment.AttachmentBytes = partdata
703704
attachment.Name = p.FileName()
704-
couchDoc.Attachments = append(couchDoc.Attachments, attachment)
705+
attachments = append(attachments, attachment)
705706

706707
} // end content-encoding switch
707708
} // end if attachment
708709
} // end content-type switch
709710
} // for all multiparts
710711

712+
couchDoc.Attachments = attachments
713+
711714
return &couchDoc, revision, nil
712715
}
713716

‎core/ledger/util/couchdb/couchdb_test.go

+48-48
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ func TestDBBadConnectionDef(t *testing.T) {
9191

9292
func TestDBCreateSaveWithoutRevision(t *testing.T) {
9393

94-
if ledgerconfig.IsCouchDBEnabled() == true {
94+
if ledgerconfig.IsCouchDBEnabled() {
9595

9696
database := "testdbcreatesavewithoutrevision"
9797
err := cleanup(database)
@@ -117,7 +117,7 @@ func TestDBCreateSaveWithoutRevision(t *testing.T) {
117117

118118
func TestDBCreateEnsureFullCommit(t *testing.T) {
119119

120-
if ledgerconfig.IsCouchDBEnabled() == true {
120+
if ledgerconfig.IsCouchDBEnabled() {
121121

122122
database := "testdbensurefullcommit"
123123
err := cleanup(database)
@@ -148,7 +148,7 @@ func TestDBCreateEnsureFullCommit(t *testing.T) {
148148

149149
func TestDBBadDatabaseName(t *testing.T) {
150150

151-
if ledgerconfig.IsCouchDBEnabled() == true {
151+
if ledgerconfig.IsCouchDBEnabled() {
152152

153153
//create a new instance and database object using a valid database name mixed case
154154
couchInstance, err := CreateCouchInstance(connectURL, username, password)
@@ -187,7 +187,7 @@ func TestDBBadConnection(t *testing.T) {
187187
// TODO Re-enable once configurable retry logic is introduced
188188
t.Skip()
189189

190-
if ledgerconfig.IsCouchDBEnabled() == true {
190+
if ledgerconfig.IsCouchDBEnabled() {
191191

192192
//create a new instance and database object
193193
_, err := CreateCouchInstance(badConnectURL, username, password)
@@ -197,7 +197,7 @@ func TestDBBadConnection(t *testing.T) {
197197

198198
func TestDBCreateDatabaseAndPersist(t *testing.T) {
199199

200-
if ledgerconfig.IsCouchDBEnabled() == true {
200+
if ledgerconfig.IsCouchDBEnabled() {
201201

202202
database := "testdbcreatedatabaseandpersist"
203203
err := cleanup(database)
@@ -286,7 +286,7 @@ func TestDBCreateDatabaseAndPersist(t *testing.T) {
286286

287287
func TestDBBadJSON(t *testing.T) {
288288

289-
if ledgerconfig.IsCouchDBEnabled() == true {
289+
if ledgerconfig.IsCouchDBEnabled() {
290290

291291
database := "testdbbadjson"
292292
err := cleanup(database)
@@ -322,7 +322,7 @@ func TestDBBadJSON(t *testing.T) {
322322
}
323323

324324
func TestPrefixScan(t *testing.T) {
325-
if !ledgerconfig.IsCouchDBEnabled() == true {
325+
if !ledgerconfig.IsCouchDBEnabled() {
326326
return
327327
}
328328
database := "testprefixscan"
@@ -384,7 +384,7 @@ func TestPrefixScan(t *testing.T) {
384384

385385
func TestDBSaveAttachment(t *testing.T) {
386386

387-
if ledgerconfig.IsCouchDBEnabled() == true {
387+
if ledgerconfig.IsCouchDBEnabled() {
388388

389389
database := "testdbsaveattachment"
390390
err := cleanup(database)
@@ -395,12 +395,12 @@ func TestDBSaveAttachment(t *testing.T) {
395395

396396
byteText := []byte(`This is a test document. This is only a test`)
397397

398-
attachment := Attachment{}
398+
attachment := &Attachment{}
399399
attachment.AttachmentBytes = byteText
400400
attachment.ContentType = "text/plain"
401401
attachment.Name = "valueBytes"
402402

403-
attachments := []Attachment{}
403+
attachments := []*Attachment{}
404404
attachments = append(attachments, attachment)
405405

406406
//create a new instance and database object
@@ -428,7 +428,7 @@ func TestDBSaveAttachment(t *testing.T) {
428428

429429
func TestDBDeleteDocument(t *testing.T) {
430430

431-
if ledgerconfig.IsCouchDBEnabled() == true {
431+
if ledgerconfig.IsCouchDBEnabled() {
432432

433433
database := "testdbdeletedocument"
434434
err := cleanup(database)
@@ -466,7 +466,7 @@ func TestDBDeleteDocument(t *testing.T) {
466466

467467
func TestDBDeleteNonExistingDocument(t *testing.T) {
468468

469-
if ledgerconfig.IsCouchDBEnabled() == true {
469+
if ledgerconfig.IsCouchDBEnabled() {
470470

471471
database := "testdbdeletenonexistingdocument"
472472
err := cleanup(database)
@@ -508,7 +508,7 @@ func TestCouchDBVersion(t *testing.T) {
508508

509509
func TestRichQuery(t *testing.T) {
510510

511-
if ledgerconfig.IsCouchDBEnabled() == true {
511+
if ledgerconfig.IsCouchDBEnabled() {
512512

513513
byteJSON01 := []byte(`{"asset_name":"marble01","color":"blue","size":1,"owner":"jerry"}`)
514514
byteJSON02 := []byte(`{"asset_name":"marble02","color":"red","size":2,"owner":"tom"}`)
@@ -527,85 +527,85 @@ func TestRichQuery(t *testing.T) {
527527
attachment1.AttachmentBytes = []byte(`marble01 - test attachment`)
528528
attachment1.ContentType = "application/octet-stream"
529529
attachment1.Name = "data"
530-
attachments1 := []Attachment{}
531-
attachments1 = append(attachments1, *attachment1)
530+
attachments1 := []*Attachment{}
531+
attachments1 = append(attachments1, attachment1)
532532

533533
attachment2 := &Attachment{}
534534
attachment2.AttachmentBytes = []byte(`marble02 - test attachment`)
535535
attachment2.ContentType = "application/octet-stream"
536536
attachment2.Name = "data"
537-
attachments2 := []Attachment{}
538-
attachments2 = append(attachments2, *attachment2)
537+
attachments2 := []*Attachment{}
538+
attachments2 = append(attachments2, attachment2)
539539

540540
attachment3 := &Attachment{}
541541
attachment3.AttachmentBytes = []byte(`marble03 - test attachment`)
542542
attachment3.ContentType = "application/octet-stream"
543543
attachment3.Name = "data"
544-
attachments3 := []Attachment{}
545-
attachments3 = append(attachments3, *attachment3)
544+
attachments3 := []*Attachment{}
545+
attachments3 = append(attachments3, attachment3)
546546

547547
attachment4 := &Attachment{}
548548
attachment4.AttachmentBytes = []byte(`marble04 - test attachment`)
549549
attachment4.ContentType = "application/octet-stream"
550550
attachment4.Name = "data"
551-
attachments4 := []Attachment{}
552-
attachments4 = append(attachments4, *attachment4)
551+
attachments4 := []*Attachment{}
552+
attachments4 = append(attachments4, attachment4)
553553

554554
attachment5 := &Attachment{}
555555
attachment5.AttachmentBytes = []byte(`marble05 - test attachment`)
556556
attachment5.ContentType = "application/octet-stream"
557557
attachment5.Name = "data"
558-
attachments5 := []Attachment{}
559-
attachments5 = append(attachments5, *attachment5)
558+
attachments5 := []*Attachment{}
559+
attachments5 = append(attachments5, attachment5)
560560

561561
attachment6 := &Attachment{}
562562
attachment6.AttachmentBytes = []byte(`marble06 - test attachment`)
563563
attachment6.ContentType = "application/octet-stream"
564564
attachment6.Name = "data"
565-
attachments6 := []Attachment{}
566-
attachments6 = append(attachments6, *attachment6)
565+
attachments6 := []*Attachment{}
566+
attachments6 = append(attachments6, attachment6)
567567

568568
attachment7 := &Attachment{}
569569
attachment7.AttachmentBytes = []byte(`marble07 - test attachment`)
570570
attachment7.ContentType = "application/octet-stream"
571571
attachment7.Name = "data"
572-
attachments7 := []Attachment{}
573-
attachments7 = append(attachments7, *attachment7)
572+
attachments7 := []*Attachment{}
573+
attachments7 = append(attachments7, attachment7)
574574

575575
attachment8 := &Attachment{}
576576
attachment8.AttachmentBytes = []byte(`marble08 - test attachment`)
577577
attachment8.ContentType = "application/octet-stream"
578578
attachment7.Name = "data"
579-
attachments8 := []Attachment{}
580-
attachments8 = append(attachments8, *attachment8)
579+
attachments8 := []*Attachment{}
580+
attachments8 = append(attachments8, attachment8)
581581

582582
attachment9 := &Attachment{}
583583
attachment9.AttachmentBytes = []byte(`marble09 - test attachment`)
584584
attachment9.ContentType = "application/octet-stream"
585585
attachment9.Name = "data"
586-
attachments9 := []Attachment{}
587-
attachments9 = append(attachments9, *attachment9)
586+
attachments9 := []*Attachment{}
587+
attachments9 = append(attachments9, attachment9)
588588

589589
attachment10 := &Attachment{}
590590
attachment10.AttachmentBytes = []byte(`marble10 - test attachment`)
591591
attachment10.ContentType = "application/octet-stream"
592592
attachment10.Name = "data"
593-
attachments10 := []Attachment{}
594-
attachments10 = append(attachments10, *attachment10)
593+
attachments10 := []*Attachment{}
594+
attachments10 = append(attachments10, attachment10)
595595

596596
attachment11 := &Attachment{}
597597
attachment11.AttachmentBytes = []byte(`marble11 - test attachment`)
598598
attachment11.ContentType = "application/octet-stream"
599599
attachment11.Name = "data"
600-
attachments11 := []Attachment{}
601-
attachments11 = append(attachments11, *attachment11)
600+
attachments11 := []*Attachment{}
601+
attachments11 = append(attachments11, attachment11)
602602

603603
attachment12 := &Attachment{}
604604
attachment12.AttachmentBytes = []byte(`marble12 - test attachment`)
605605
attachment12.ContentType = "application/octet-stream"
606606
attachment12.Name = "data"
607-
attachments12 := []Attachment{}
608-
attachments12 = append(attachments12, *attachment12)
607+
attachments12 := []*Attachment{}
608+
attachments12 = append(attachments12, attachment12)
609609

610610
database := "testrichquery"
611611
err := cleanup(database)
@@ -779,7 +779,7 @@ func TestRichQuery(t *testing.T) {
779779

780780
func TestBatchBatchOperations(t *testing.T) {
781781

782-
if ledgerconfig.IsCouchDBEnabled() == true {
782+
if ledgerconfig.IsCouchDBEnabled() {
783783

784784
byteJSON01 := []byte(`{"_id":"marble01","asset_name":"marble01","color":"blue","size":"1","owner":"jerry"}`)
785785
byteJSON02 := []byte(`{"_id":"marble02","asset_name":"marble02","color":"red","size":"2","owner":"tom"}`)
@@ -791,36 +791,36 @@ func TestBatchBatchOperations(t *testing.T) {
791791
attachment1.AttachmentBytes = []byte(`marble01 - test attachment`)
792792
attachment1.ContentType = "application/octet-stream"
793793
attachment1.Name = "data"
794-
attachments1 := []Attachment{}
795-
attachments1 = append(attachments1, *attachment1)
794+
attachments1 := []*Attachment{}
795+
attachments1 = append(attachments1, attachment1)
796796

797797
attachment2 := &Attachment{}
798798
attachment2.AttachmentBytes = []byte(`marble02 - test attachment`)
799799
attachment2.ContentType = "application/octet-stream"
800800
attachment2.Name = "data"
801-
attachments2 := []Attachment{}
802-
attachments2 = append(attachments2, *attachment2)
801+
attachments2 := []*Attachment{}
802+
attachments2 = append(attachments2, attachment2)
803803

804804
attachment3 := &Attachment{}
805805
attachment3.AttachmentBytes = []byte(`marble03 - test attachment`)
806806
attachment3.ContentType = "application/octet-stream"
807807
attachment3.Name = "data"
808-
attachments3 := []Attachment{}
809-
attachments3 = append(attachments3, *attachment3)
808+
attachments3 := []*Attachment{}
809+
attachments3 = append(attachments3, attachment3)
810810

811811
attachment4 := &Attachment{}
812812
attachment4.AttachmentBytes = []byte(`marble04 - test attachment`)
813813
attachment4.ContentType = "application/octet-stream"
814814
attachment4.Name = "data"
815-
attachments4 := []Attachment{}
816-
attachments4 = append(attachments4, *attachment4)
815+
attachments4 := []*Attachment{}
816+
attachments4 = append(attachments4, attachment4)
817817

818818
attachment5 := &Attachment{}
819819
attachment5.AttachmentBytes = []byte(`marble05 - test attachment`)
820820
attachment5.ContentType = "application/octet-stream"
821821
attachment5.Name = "data"
822-
attachments5 := []Attachment{}
823-
attachments5 = append(attachments5, *attachment5)
822+
attachments5 := []*Attachment{}
823+
attachments5 = append(attachments5, attachment5)
824824

825825
database := "testbatch"
826826
err := cleanup(database)

0 commit comments

Comments
 (0)