Skip to content

Commit 130ad7c

Browse files
manish-sethimanish-sethi
committed
Block stream across files
This commit allows a block stream consumer read across multiple files. Change-Id: I1bd053e8d022bf1daf01f61f9e3e1ae1629e334b Signed-off-by: manish <[email protected]>
1 parent 9c2ecfc commit 130ad7c

File tree

4 files changed

+299
-60
lines changed

4 files changed

+299
-60
lines changed

‎core/ledgernext/blkstorage/fsblkstorage/block_stream.go

+158
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package fsblkstorage
18+
19+
import (
20+
"bufio"
21+
"errors"
22+
"fmt"
23+
"io"
24+
"os"
25+
26+
"github.com/golang/protobuf/proto"
27+
)
28+
29+
// ErrUnexpectedEndOfBlockfile error used to indicate an unexpected end of a file segment
30+
// this can happen mainly if a crash occurs during appening a block and partial block contents
31+
// get written towards the end of the file
32+
var ErrUnexpectedEndOfBlockfile = errors.New("unexpected end of blockfile")
33+
34+
// blockfileStream reads blocks sequentially from a single file.
35+
// It starts from the given offset and can traverse till the end of the file
36+
type blockfileStream struct {
37+
file *os.File
38+
reader *bufio.Reader
39+
currentOffset int64
40+
}
41+
42+
// blockStream reads blocks sequentially from multiple files.
43+
// it starts from a given file offset and continues with the next
44+
// file segment until the end of the last segment (`endFileNum`)
45+
type blockStream struct {
46+
rootDir string
47+
currentFileNum int
48+
endFileNum int
49+
currentFileStream *blockfileStream
50+
}
51+
52+
///////////////////////////////////
53+
// blockfileStream functions
54+
////////////////////////////////////
55+
func newBlockfileStream(filePath string, startOffset int64) (*blockfileStream, error) {
56+
logger.Debugf("newBlockfileStream(): filePath=[%s], startOffset=[%d]", filePath, startOffset)
57+
var file *os.File
58+
var err error
59+
if file, err = os.OpenFile(filePath, os.O_RDONLY, 0600); err != nil {
60+
return nil, err
61+
}
62+
var newPosition int64
63+
if newPosition, err = file.Seek(startOffset, 0); err != nil {
64+
// file.Seek does not raise an error - simply seeks to the new position
65+
return nil, err
66+
}
67+
if newPosition != startOffset {
68+
panic(fmt.Sprintf("Could not seek file [%s] to given startOffset [%d]. New position = [%d]",
69+
filePath, startOffset, newPosition))
70+
}
71+
s := &blockfileStream{file, bufio.NewReader(file), startOffset}
72+
return s, nil
73+
}
74+
75+
func (s *blockfileStream) nextBlockBytes() ([]byte, error) {
76+
var lenBytes []byte
77+
var err error
78+
if lenBytes, err = s.reader.Peek(8); err != nil {
79+
// reader.Peek raises io.EOF error if enough bytes not available
80+
if err == io.EOF {
81+
if len(lenBytes) > 0 {
82+
return nil, ErrUnexpectedEndOfBlockfile
83+
}
84+
return nil, nil
85+
}
86+
return nil, err
87+
}
88+
len, n := proto.DecodeVarint(lenBytes)
89+
if n == 0 {
90+
panic(fmt.Errorf("Error in decoding varint bytes"))
91+
}
92+
if _, err = s.reader.Discard(n); err != nil {
93+
return nil, err
94+
}
95+
blockBytes := make([]byte, len)
96+
if _, err = io.ReadAtLeast(s.reader, blockBytes, int(len)); err != nil {
97+
// io.ReadAtLeast raises io.ErrUnexpectedEOF error if it is able to
98+
// read a fewer (non-zero) bytes and io.EOF is encountered
99+
if err == io.ErrUnexpectedEOF {
100+
return nil, ErrUnexpectedEndOfBlockfile
101+
}
102+
return nil, err
103+
}
104+
s.currentOffset += int64(n) + int64(len)
105+
return blockBytes, nil
106+
}
107+
108+
func (s *blockfileStream) close() error {
109+
return s.file.Close()
110+
}
111+
112+
///////////////////////////////////
113+
// blockStream functions
114+
////////////////////////////////////
115+
func newBlockStream(rootDir string, startFileNum int, startOffset int64, endFileNum int) (*blockStream, error) {
116+
startFile := deriveBlockfilePath(rootDir, startFileNum)
117+
startFileStream, err := newBlockfileStream(startFile, startOffset)
118+
if err != nil {
119+
return nil, err
120+
}
121+
return &blockStream{rootDir, startFileNum, endFileNum, startFileStream}, nil
122+
}
123+
124+
func (s *blockStream) moveToNextBlockfileStream() error {
125+
var err error
126+
if err = s.currentFileStream.close(); err != nil {
127+
return err
128+
}
129+
s.currentFileNum++
130+
nextFile := deriveBlockfilePath(s.rootDir, s.currentFileNum)
131+
if s.currentFileStream, err = newBlockfileStream(nextFile, 0); err != nil {
132+
return err
133+
}
134+
return nil
135+
}
136+
137+
func (s *blockStream) nextBlockBytes() ([]byte, error) {
138+
var blockBytes []byte
139+
var err error
140+
if blockBytes, err = s.currentFileStream.nextBlockBytes(); err != nil {
141+
logger.Debugf("current file [%d]", s.currentFileNum)
142+
logger.Debugf("blockbytes [%d]. Err:%s", len(blockBytes), err)
143+
return nil, err
144+
}
145+
logger.Debugf("blockbytes [%d] read from file [%d]", len(blockBytes), s.currentFileNum)
146+
if blockBytes == nil && s.currentFileNum < s.endFileNum {
147+
logger.Debugf("current file [%d] exhausted. Moving to next file", s.currentFileNum)
148+
if err = s.moveToNextBlockfileStream(); err != nil {
149+
return nil, err
150+
}
151+
return s.nextBlockBytes()
152+
}
153+
return blockBytes, nil
154+
}
155+
156+
func (s *blockStream) close() error {
157+
return s.currentFileStream.close()
158+
}

‎core/ledgernext/blkstorage/fsblkstorage/block_stream_test.go

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package fsblkstorage
18+
19+
import (
20+
"testing"
21+
22+
"github.com/golang/protobuf/proto"
23+
"github.com/hyperledger/fabric/core/ledgernext/testutil"
24+
)
25+
26+
func TestBlockfileStream(t *testing.T) {
27+
testBlockfileStream(t, 0)
28+
testBlockfileStream(t, 1)
29+
testBlockfileStream(t, 10)
30+
}
31+
32+
func testBlockfileStream(t *testing.T, numBlocks int) {
33+
env := newTestEnv(t)
34+
defer env.Cleanup()
35+
w := newTestBlockfileWrapper(t, env)
36+
blockfileMgr := w.blockfileMgr
37+
38+
blocks := testutil.ConstructTestBlocks(t, numBlocks)
39+
w.addBlocks(blocks)
40+
w.close()
41+
42+
s, err := newBlockfileStream(deriveBlockfilePath(blockfileMgr.rootDir, 0), 0)
43+
defer s.close()
44+
testutil.AssertNoError(t, err, "Error in constructing blockfile stream")
45+
46+
blockCount := 0
47+
for {
48+
blockBytes, err := s.nextBlockBytes()
49+
testutil.AssertNoError(t, err, "Error in getting next block")
50+
if blockBytes == nil {
51+
break
52+
}
53+
blockCount++
54+
}
55+
// After the stream has been exhausted, both blockBytes and err should be nil
56+
blockBytes, err := s.nextBlockBytes()
57+
testutil.AssertNil(t, blockBytes)
58+
testutil.AssertNoError(t, err, "Error in getting next block after exhausting the file")
59+
testutil.AssertEquals(t, blockCount, numBlocks)
60+
}
61+
62+
func TestBlockFileStreamUnexpectedEOF(t *testing.T) {
63+
partialBlockBytes := []byte{}
64+
dummyBlockBytes := testutil.ConstructRandomBytes(t, 100)
65+
lenBytes := proto.EncodeVarint(uint64(len(dummyBlockBytes)))
66+
partialBlockBytes = append(partialBlockBytes, lenBytes...)
67+
partialBlockBytes = append(partialBlockBytes, dummyBlockBytes...)
68+
testBlockFileStreamUnexpectedEOF(t, 10, partialBlockBytes[:1])
69+
testBlockFileStreamUnexpectedEOF(t, 10, partialBlockBytes[:2])
70+
testBlockFileStreamUnexpectedEOF(t, 10, partialBlockBytes[:5])
71+
testBlockFileStreamUnexpectedEOF(t, 10, partialBlockBytes[:20])
72+
}
73+
74+
func testBlockFileStreamUnexpectedEOF(t *testing.T, numBlocks int, partialBlockBytes []byte) {
75+
env := newTestEnv(t)
76+
defer env.Cleanup()
77+
w := newTestBlockfileWrapper(t, env)
78+
blockfileMgr := w.blockfileMgr
79+
blocks := testutil.ConstructTestBlocks(t, numBlocks)
80+
w.addBlocks(blocks)
81+
blockfileMgr.currentFileWriter.append(partialBlockBytes, true)
82+
w.close()
83+
s, err := newBlockfileStream(deriveBlockfilePath(blockfileMgr.rootDir, 0), 0)
84+
defer s.close()
85+
testutil.AssertNoError(t, err, "Error in constructing blockfile stream")
86+
87+
for i := 0; i < numBlocks; i++ {
88+
blockBytes, err := s.nextBlockBytes()
89+
testutil.AssertNotNil(t, blockBytes)
90+
testutil.AssertNoError(t, err, "Error in getting next block")
91+
}
92+
blockBytes, err := s.nextBlockBytes()
93+
testutil.AssertNil(t, blockBytes)
94+
testutil.AssertSame(t, err, ErrUnexpectedEndOfBlockfile)
95+
}
96+
97+
func TestBlockStream(t *testing.T) {
98+
testBlockStream(t, 1)
99+
testBlockStream(t, 2)
100+
testBlockStream(t, 10)
101+
}
102+
103+
func testBlockStream(t *testing.T, numFiles int) {
104+
env := newTestEnv(t)
105+
defer env.Cleanup()
106+
w := newTestBlockfileWrapper(t, env)
107+
defer w.close()
108+
blockfileMgr := w.blockfileMgr
109+
110+
numBlocksInEachFile := 10
111+
for i := 0; i < numFiles; i++ {
112+
blocks := testutil.ConstructTestBlocks(t, numBlocksInEachFile)
113+
w.addBlocks(blocks)
114+
blockfileMgr.moveToNextFile()
115+
}
116+
s, err := newBlockStream(blockfileMgr.rootDir, 0, 0, numFiles-1)
117+
defer s.close()
118+
testutil.AssertNoError(t, err, "Error in constructing new block stream")
119+
blockCount := 0
120+
for {
121+
blockBytes, err := s.nextBlockBytes()
122+
testutil.AssertNoError(t, err, "Error in getting next block")
123+
if blockBytes == nil {
124+
break
125+
}
126+
blockCount++
127+
}
128+
// After the stream has been exhausted, both blockBytes and err should be nil
129+
blockBytes, err := s.nextBlockBytes()
130+
testutil.AssertNil(t, blockBytes)
131+
testutil.AssertNoError(t, err, "Error in getting next block after exhausting the file")
132+
testutil.AssertEquals(t, blockCount, numFiles*numBlocksInEachFile)
133+
}

‎core/ledgernext/blkstorage/fsblkstorage/blockfile_mgr.go

+8-9
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,9 @@ func (mgr *blockfileMgr) retrieveBlocks(startNum uint64, endNum uint64) (*Blocks
269269
if lp, err = mgr.index.getBlockLocByBlockNum(startNum); err != nil {
270270
return nil, err
271271
}
272-
filePath := deriveBlockfilePath(mgr.rootDir, lp.fileSuffixNum)
273-
274272
var stream *blockStream
275-
if stream, err = newBlockStream(filePath, int64(lp.offset)); err != nil {
273+
if stream, err = newBlockStream(mgr.rootDir, lp.fileSuffixNum,
274+
int64(lp.offset), mgr.cpInfo.latestFileChunkSuffixNum); err != nil {
276275
return nil, err
277276
}
278277
return newBlockItr(stream, int(endNum-startNum)+1), nil
@@ -322,7 +321,7 @@ func (mgr *blockfileMgr) fetchTransaction(lp *fileLocPointer) (*protos.Transacti
322321

323322
func (mgr *blockfileMgr) fetchBlockBytes(lp *fileLocPointer) ([]byte, error) {
324323
filePath := deriveBlockfilePath(mgr.rootDir, lp.fileSuffixNum)
325-
stream, err := newBlockStream(filePath, int64(lp.offset))
324+
stream, err := newBlockfileStream(filePath, int64(lp.offset))
326325
if err != nil {
327326
return nil, err
328327
}
@@ -382,23 +381,23 @@ func (mgr *blockfileMgr) saveCurrentInfo(i *checkpointInfo, flush bool) error {
382381
func scanForLastCompleteBlock(filePath string, startingOffset int64) (int64, int, error) {
383382
logger.Debugf("scanForLastCompleteBlock(): filePath=[%s], startingOffset=[%d]", filePath, startingOffset)
384383
numBlocks := 0
385-
blockStream, err := newBlockStream(filePath, startingOffset)
384+
blockStream, err := newBlockfileStream(filePath, startingOffset)
386385
if err != nil {
387386
return 0, 0, err
388387
}
389388
defer blockStream.close()
390389
for {
391390
blockBytes, err := blockStream.nextBlockBytes()
392-
if blockBytes == nil || err != nil {
391+
if blockBytes == nil || err == ErrUnexpectedEndOfBlockfile {
393392
logger.Debugf(`scanForLastCompleteBlock(): error=[%s].
394-
This may happen if a crash has happened during block appending.
393+
The error may happen if a crash has happened during block appending.
395394
Returning current offset as a last complete block's end offset`, err)
396395
break
397396
}
398397
numBlocks++
399398
}
400-
logger.Debugf("scanForLastCompleteBlock(): last complete block ends at offset=[%d]", blockStream.currentFileOffset)
401-
return blockStream.currentFileOffset, numBlocks, err
399+
logger.Debugf("scanForLastCompleteBlock(): last complete block ends at offset=[%d]", blockStream.currentOffset)
400+
return blockStream.currentOffset, numBlocks, err
402401
}
403402

404403
// checkpointInfo

0 commit comments

Comments
 (0)