Skip to content

Commit 5ab0de3

Browse files
JeromeMartinezJeromeMartinez
committed
[WIP] Add an alternative method for reading files
Using mmap only seems to be very slow with some NAS
1 parent fce3a36 commit 5ab0de3

File tree

8 files changed

+206
-13
lines changed

8 files changed

+206
-13
lines changed

‎Source/CLI/Global.cpp

+41
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,33 @@ int global::SetHash(bool Value)
242242
return 0;
243243
}
244244

245+
//---------------------------------------------------------------------------
246+
int global::SetIo(const char* Value)
247+
{
248+
if (strcmp(Value, "mmap") == 0)
249+
{
250+
OpenStyle = filemap::style::mmap;
251+
return 0;
252+
}
253+
if (strcmp(Value, "fstream") == 0)
254+
{
255+
OpenStyle = filemap::style::fstream;
256+
return 0;
257+
}
258+
if (strcmp(Value, "fopen") == 0)
259+
{
260+
OpenStyle = filemap::style::fopen;
261+
return 0;
262+
}
263+
if (strcmp(Value, "open") == 0)
264+
{
265+
OpenStyle = filemap::style::open;
266+
return 0;
267+
}
268+
cerr << "Error: unknown io value '" << Value << "'." << endl;
269+
return 1;
270+
}
271+
245272
//---------------------------------------------------------------------------
246273
int global::SetAll(bool Value)
247274
{
@@ -432,6 +459,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
432459
IgnoreLicenseKey = !License.IsSupported_License();
433460
SubLicenseId = 0;
434461
SubLicenseDur = 1;
462+
OpenStyle = {}; // (filemap::style)-1;
435463
ShowLicenseKey = false;
436464
StoreLicenseKey = false;
437465
DisplayCommand = false;
@@ -748,6 +776,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
748776
if (auto Value = SetAcceptFiles())
749777
return Value;
750778
}
779+
else if (strcmp(argv[i], "--io") == 0)
780+
{
781+
if (i + 1 == argc)
782+
return Error_Missing(argv[i]);
783+
int Value = SetIo(argv[++i]);
784+
if (Value)
785+
return Value;
786+
}
751787
else if (!strcmp(argv[i], "-framerate"))
752788
{
753789
if (OptionsForOtherFiles)
@@ -831,6 +867,11 @@ int global::ManageCommandLine(const char* argv[], int argc)
831867
}
832868
if (License.ShowLicense(ShowLicenseKey, SubLicenseId, SubLicenseDur))
833869
return 1;
870+
if (OpenStyle == (filemap::style)-1)
871+
{
872+
cerr << "\nThis is a test version, please use another version if you don't know which option to test\n" << endl;
873+
return 1;
874+
}
834875
if (Inputs.empty() && (ShowLicenseKey || SubLicenseId))
835876
return 0;
836877

‎Source/CLI/Global.h

+2
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class global
3838
string LicenseKey;
3939
uint64_t SubLicenseId;
4040
uint64_t SubLicenseDur;
41+
filemap::style OpenStyle;
4142
bool IgnoreLicenseKey;
4243
bool ShowLicenseKey;
4344
bool StoreLicenseKey;
@@ -100,6 +101,7 @@ class global
100101
int SetFrameMd5An(bool Value);
101102
int SetFrameMd5FileName(const char* FileName);
102103
int SetHash(bool Value);
104+
int SetIo(const char* Value);
103105
int SetAll(bool Value);
104106

105107
private:

‎Source/CLI/Main.cpp

+6-3
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
481481
}
482482

483483
//---------------------------------------------------------------------------
484-
int ParseFile_Compressed(parse_info& ParseInfo)
484+
int ParseFile_Compressed(parse_info& ParseInfo, const string* FileName)
485485
{
486486
// Init
487487
string OutputDirectoryName;
@@ -522,6 +522,9 @@ int ParseFile_Compressed(parse_info& ParseInfo)
522522
matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
523523
M->Quiet = Global.Quiet;
524524
M->NoOutputCheck = NoOutputCheck;
525+
if (RAWcooked.OutputFileName.empty())
526+
RAWcooked.OutputFileName = *FileName;
527+
M->OpenStyle = Global.OpenStyle;
525528
if (ParseInfo.ParseFile_Input(*M))
526529
{
527530
ReturnValue = 1;
@@ -591,7 +594,7 @@ int ParseFile(size_t Files_Pos)
591594
return 1;
592595

593596
// Compressed content
594-
if (int Value = ParseFile_Compressed(ParseInfo))
597+
if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
595598
return Value;
596599
if (ParseInfo.IsDetected)
597600
return 0;
@@ -772,7 +775,7 @@ int main(int argc, const char* argv[])
772775
// Parse (check mode)
773776
Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
774777
Global.Actions.set(Action_Decode, false); // Override config
775-
Value = ParseFile_Compressed(ParseInfo);
778+
Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name);
776779
if (!Value && !ParseInfo.IsDetected)
777780
{
778781
cout << '\n' << "Error: " << Global.OutputFileName << endl;

‎Source/Lib/Compressed/Matroska/Matroska.cpp

+21-4
Original file line numberDiff line numberDiff line change
@@ -359,13 +359,16 @@ void matroska::ParseBuffer()
359359
// Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
360360
if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value
361361
{
362-
FileMap->Remap();
362+
FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
363363
Buffer = *FileMap;
364+
if (OpenStyle == filemap::style::mmap)
365+
{
364366
if (ReversibilityData)
365367
ReversibilityData->SetBaseData(Buffer.Data());
366368
for (const auto& TrackInfo_Current : TrackInfo)
367369
if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
368370
TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
371+
}
369372
Buffer_Offset_LowerLimit = Buffer_Offset;
370373
}
371374

@@ -376,13 +379,16 @@ void matroska::ParseBuffer()
376379
Buffer_Offset = Cluster_Offset;
377380
Cluster_Level = (size_t)-1;
378381

379-
FileMap->Remap();
382+
FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
380383
Buffer = *FileMap;
384+
if (OpenStyle == filemap::style::mmap)
385+
{
381386
if (ReversibilityData)
382387
ReversibilityData->SetBaseData(Buffer.Data());
383388
for (const auto& TrackInfo_Current : TrackInfo)
384389
if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
385390
TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
391+
}
386392
Buffer_Offset_LowerLimit = Buffer_Offset;
387393
}
388394
}
@@ -799,6 +805,11 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_LibraryV
799805
//---------------------------------------------------------------------------
800806
void matroska::Segment_Cluster()
801807
{
808+
IsList = true;
809+
810+
if (FileMap2)
811+
return;
812+
802813
if (RAWcooked_LibraryName.empty())
803814
{
804815
memcpy(Cluster_Levels, Levels, sizeof(Levels));
@@ -808,8 +819,6 @@ void matroska::Segment_Cluster()
808819
return;
809820
}
810821

811-
IsList = true;
812-
813822
// Check if Hashes check is useful
814823
if (Hashes_FromRAWcooked)
815824
{
@@ -850,6 +859,14 @@ void matroska::Segment_Cluster()
850859
Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
851860
if (ReversibilityData && !FrameWriter_Template->Compound)
852861
InitOutput_Find();
862+
863+
FileMap2 = FileMap;
864+
if (OpenStyle != filemap::style::mmap)
865+
{
866+
FileMap = new filemap;
867+
FileMap->Open_ReadMode(*this->FileName, OpenStyle, 0, 256 * 1024 * 1024);
868+
Buffer = *FileMap;
869+
}
853870
}
854871

855872
//---------------------------------------------------------------------------

‎Source/Lib/Utils/FileIO/FileIO.cpp

+121-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "Lib/Utils/FileIO/FileIO.h"
1212
#include <iostream>
1313
#include <sstream>
14+
#include <fstream>
1415
#if defined(_WIN32) || defined(_WINDOWS)
1516
#include "windows.h"
1617
#include <io.h> // File existence
@@ -29,10 +30,82 @@
2930
//---------------------------------------------------------------------------
3031

3132
//---------------------------------------------------------------------------
32-
int filemap::Open_ReadMode(const char* FileName)
33+
struct private_buffered
34+
{
35+
void* F;
36+
size_t Data_Shift = 0;
37+
size_t MaxSize = 0;
38+
};
39+
40+
//---------------------------------------------------------------------------
41+
int filemap::Open_ReadMode(const char* FileName, style NewStyle, size_t Begin, size_t End)
3342
{
3443
Close();
3544

45+
if (NewStyle != style::mmap)
46+
{
47+
Style = NewStyle;
48+
private_buffered* P = new private_buffered;
49+
P->MaxSize = End - Begin;
50+
size_t FileSize;
51+
52+
switch (Style)
53+
{
54+
default: // case style::fstream:
55+
{
56+
auto F = new ifstream(FileName, ios::binary);
57+
F->seekg(0, F->end);
58+
FileSize = F->tellg();
59+
F->seekg(Begin, F->beg);
60+
P->F = F;
61+
break;
62+
}
63+
case style::fopen:
64+
{
65+
auto F = fopen(FileName, "rb");
66+
fseek(F, 0, SEEK_END);
67+
FileSize = ftell(F);
68+
fseek(F, (long)Begin, SEEK_SET);
69+
P->F = F;
70+
break;
71+
}
72+
case style::open:
73+
{
74+
#if defined(_WIN32) || defined(_WINDOWS)
75+
DWORD FileSizeHigh;
76+
auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
77+
auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh);
78+
if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value)
79+
&& (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode
80+
{
81+
FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow;
82+
}
83+
else
84+
return 1;
85+
if (Begin)
86+
{
87+
LARGE_INTEGER GoTo;
88+
GoTo.QuadPart = Begin;
89+
if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0))
90+
return 1;
91+
P->Data_Shift = Begin;
92+
}
93+
P->F = NewFile;
94+
#else //defined(_WIN32) || defined(_WINDOWS)
95+
return 1;
96+
#endif //defined(_WIN32) || defined(_WINDOWS)
97+
break;
98+
}
99+
}
100+
101+
auto Buffer = new uint8_t[P->MaxSize];
102+
P->Data_Shift -= P->MaxSize;
103+
AssignBase(Buffer - P->Data_Shift, FileSize);
104+
Private = (void*)P;
105+
106+
return Remap(Begin, End);
107+
}
108+
36109
size_t NewSize;
37110
#if defined(_WIN32) || defined(_WINDOWS)
38111
auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
@@ -99,12 +172,58 @@ inline int munmap_const(const void* addr, size_t length)
99172
#pragma GCC diagnostic pop
100173
#endif
101174
#endif
102-
int filemap::Remap()
175+
int filemap::Remap(size_t Begin, size_t End)
103176
{
104177
// Special case for 0-byte files
105178
if (Empty())
106179
return 0;
107180

181+
if (Style != style::mmap)
182+
{
183+
auto P = (private_buffered*)Private;
184+
auto Buffer = Data() + P->Data_Shift;
185+
auto Buffer_MaxSize = P->MaxSize;
186+
Begin -= P->Data_Shift;
187+
if (!End)
188+
End = Size();
189+
End -= P->Data_Shift;
190+
auto Buffer_Middle = Buffer + Begin;
191+
auto Buffer_Middle_Size = Buffer_MaxSize - Begin;
192+
memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size);
193+
P->Data_Shift += Begin;
194+
AssignKeepSizeBase(Buffer - P->Data_Shift);
195+
Buffer += Buffer_Middle_Size;
196+
Buffer_MaxSize -= Buffer_Middle_Size;
197+
198+
switch (Style)
199+
{
200+
default: // case style::fstream:
201+
{
202+
auto F = (ifstream*)P->F;
203+
F->read((char*)Buffer, Buffer_MaxSize);
204+
break;
205+
}
206+
case style::fopen:
207+
{
208+
auto F = (FILE*)P->F;
209+
if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1)
210+
return 1;
211+
break;
212+
}
213+
case style::open:
214+
{
215+
#if defined(_WIN32) || defined(_WINDOWS)
216+
ReadFile(P->F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0);
217+
#else //defined(_WIN32) || defined(_WINDOWS)
218+
return 1;
219+
#endif //defined(_WIN32) || defined(_WINDOWS)
220+
break;
221+
}
222+
}
223+
224+
return 0;
225+
}
226+
108227
// Close previous map
109228
if (Data())
110229
{

‎Source/Lib/Utils/FileIO/FileIO.h

+11-3
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,17 @@ class filemap : public buffer_view
2323
~filemap() { Close(); }
2424

2525
// Actions
26-
int Open_ReadMode(const char* FileName);
27-
int Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); }
26+
enum class style
27+
{
28+
mmap,
29+
fstream,
30+
fopen,
31+
open,
32+
};
33+
int Open_ReadMode(const char* FileName, style NewStyle = {}, size_t Begin = {}, size_t End = {});
34+
int Open_ReadMode(const string& FileName, style NewStyle = {}, size_t Begin = {}, size_t End = {}) { return Open_ReadMode(FileName.c_str(), NewStyle, Begin, End); }
2835
bool IsOpen() { return Private == (decltype(Private))-1 ? false : true; }
29-
int Remap();
36+
int Remap(size_t Begin = 0, size_t End = 0);
3037
int Close();
3138

3239
private:
@@ -36,6 +43,7 @@ class filemap : public buffer_view
3643
#else //defined(_WIN32) || defined(_WINDOWS)
3744
int Private = (int)-1;
3845
#endif //defined(_WIN32) || defined(_WINDOWS)
46+
style Style = {};
3947
};
4048

4149
class file

‎Source/Lib/Utils/FileIO/Input_Base.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ input_base::~input_base()
3838
bool input_base::Parse(filemap* FileMap_Source, const buffer_view& Buffer_Source, size_t FileSize_Source)
3939
{
4040
FileMap = FileMap_Source;
41+
FileMap2 = nullptr;
4142
FileSize = FileSize_Source == (size_t)-1 ? Buffer_Source.Size() : FileSize_Source;
4243
Buffer = Buffer_Source;
4344
HashComputed = false;

0 commit comments

Comments
 (0)