Skip to content

Commit 03758be

Browse files
JeromeMartinezJeromeMartinez
committed
[WIP] Add an alternative method for reading files
Using mmap only seems to be very slow with some NAS
1 parent fce3a36 commit 03758be

File tree

8 files changed

+261
-14
lines changed

8 files changed

+261
-14
lines changed

‎Source/CLI/Global.cpp

+48
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,40 @@ int global::SetHash(bool Value)
242242
return 0;
243243
}
244244

245+
//---------------------------------------------------------------------------
246+
int global::SetFileOpenMethod(const char* Value)
247+
{
248+
if (strcmp(Value, "mmap") == 0)
249+
{
250+
FileOpenMethod = filemap::method::mmap;
251+
return 0;
252+
}
253+
if (strcmp(Value, "fstream") == 0)
254+
{
255+
FileOpenMethod = filemap::method::fstream;
256+
return 0;
257+
}
258+
if (strcmp(Value, "fopen") == 0)
259+
{
260+
FileOpenMethod = filemap::method::fopen;
261+
return 0;
262+
}
263+
if (strcmp(Value, "open") == 0)
264+
{
265+
FileOpenMethod = filemap::method::open;
266+
return 0;
267+
}
268+
#if defined(_WIN32) || defined(_WINDOWS)
269+
if (strcmp(Value, "createfile") == 0)
270+
{
271+
FileOpenMethod = filemap::method::createfile;
272+
return 0;
273+
}
274+
#endif //defined(_WIN32) || defined(_WINDOWS)
275+
cerr << "Error: unknown io value '" << Value << "'." << endl;
276+
return 1;
277+
}
278+
245279
//---------------------------------------------------------------------------
246280
int global::SetAll(bool Value)
247281
{
@@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
432466
IgnoreLicenseKey = !License.IsSupported_License();
433467
SubLicenseId = 0;
434468
SubLicenseDur = 1;
469+
FileOpenMethod = (filemap::method)-1;
435470
ShowLicenseKey = false;
436471
StoreLicenseKey = false;
437472
DisplayCommand = false;
@@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
748783
if (auto Value = SetAcceptFiles())
749784
return Value;
750785
}
786+
else if (strcmp(argv[i], "--io") == 0)
787+
{
788+
if (i + 1 == argc)
789+
return Error_Missing(argv[i]);
790+
int Value = SetFileOpenMethod(argv[++i]);
791+
if (Value)
792+
return Value;
793+
}
751794
else if (!strcmp(argv[i], "-framerate"))
752795
{
753796
if (OptionsForOtherFiles)
@@ -831,6 +874,11 @@ int global::ManageCommandLine(const char* argv[], int argc)
831874
}
832875
if (License.ShowLicense(ShowLicenseKey, SubLicenseId, SubLicenseDur))
833876
return 1;
877+
if (FileOpenMethod == (filemap::method)-1)
878+
{
879+
//cerr << "\nThis is a test version, please use another version if you don't know which option to test\n" << endl;
880+
//return 1;
881+
}
834882
if (Inputs.empty() && (ShowLicenseKey || SubLicenseId))
835883
return 0;
836884

‎Source/CLI/Global.h

+2
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class global
3838
string LicenseKey;
3939
uint64_t SubLicenseId;
4040
uint64_t SubLicenseDur;
41+
filemap::method FileOpenMethod;
4142
bool IgnoreLicenseKey;
4243
bool ShowLicenseKey;
4344
bool StoreLicenseKey;
@@ -100,6 +101,7 @@ class global
100101
int SetFrameMd5An(bool Value);
101102
int SetFrameMd5FileName(const char* FileName);
102103
int SetHash(bool Value);
104+
int SetFileOpenMethod(const char* Value);
103105
int SetAll(bool Value);
104106

105107
private:

‎Source/CLI/Main.cpp

+6-3
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
481481
}
482482

483483
//---------------------------------------------------------------------------
484-
int ParseFile_Compressed(parse_info& ParseInfo)
484+
int ParseFile_Compressed(parse_info& ParseInfo, const string* FileOpenName)
485485
{
486486
// Init
487487
string OutputDirectoryName;
@@ -522,6 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo)
522522
matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
523523
M->Quiet = Global.Quiet;
524524
M->NoOutputCheck = NoOutputCheck;
525+
M->OpenName = FileOpenName;
526+
M->OpenStyle = Global.FileOpenMethod;
525527
if (ParseInfo.ParseFile_Input(*M))
526528
{
527529
ReturnValue = 1;
@@ -591,7 +593,7 @@ int ParseFile(size_t Files_Pos)
591593
return 1;
592594

593595
// Compressed content
594-
if (int Value = ParseFile_Compressed(ParseInfo))
596+
if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
595597
return Value;
596598
if (ParseInfo.IsDetected)
597599
return 0;
@@ -755,6 +757,7 @@ int main(int argc, const char* argv[])
755757
if (!Value)
756758
{
757759
// Configure for a 2nd pass
760+
auto FileOpenName = Global.OutputFileName;
758761
ParseInfo.Name = NULL;
759762
Global.OutputFileName = Global.Inputs[0];
760763
if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes
@@ -772,7 +775,7 @@ int main(int argc, const char* argv[])
772775
// Parse (check mode)
773776
Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
774777
Global.Actions.set(Action_Decode, false); // Override config
775-
Value = ParseFile_Compressed(ParseInfo);
778+
Value = ParseFile_Compressed(ParseInfo, &FileOpenName);
776779
if (!Value && !ParseInfo.IsDetected)
777780
{
778781
cout << '\n' << "Error: " << Global.OutputFileName << endl;

‎Source/Lib/Compressed/Matroska/Matroska.cpp

+33-4
Original file line numberDiff line numberDiff line change
@@ -359,13 +359,22 @@ void matroska::ParseBuffer()
359359
// Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
360360
if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value
361361
{
362-
FileMap->Remap();
362+
if (FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024))
363+
{
364+
//delete FileMap;
365+
FileMap = FileMap2;
366+
OpenStyle = filemap::method::mmap;
367+
FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
368+
}
363369
Buffer = *FileMap;
370+
if (OpenStyle == filemap::method::mmap)
371+
{
364372
if (ReversibilityData)
365373
ReversibilityData->SetBaseData(Buffer.Data());
366374
for (const auto& TrackInfo_Current : TrackInfo)
367375
if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
368376
TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
377+
}
369378
Buffer_Offset_LowerLimit = Buffer_Offset;
370379
}
371380

@@ -376,13 +385,16 @@ void matroska::ParseBuffer()
376385
Buffer_Offset = Cluster_Offset;
377386
Cluster_Level = (size_t)-1;
378387

379-
FileMap->Remap();
388+
FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
380389
Buffer = *FileMap;
390+
if (OpenStyle == filemap::method::mmap)
391+
{
381392
if (ReversibilityData)
382393
ReversibilityData->SetBaseData(Buffer.Data());
383394
for (const auto& TrackInfo_Current : TrackInfo)
384395
if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
385396
TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
397+
}
386398
Buffer_Offset_LowerLimit = Buffer_Offset;
387399
}
388400
}
@@ -428,6 +440,7 @@ void matroska::Segment()
428440
{
429441
SetDetected();
430442
IsList = true;
443+
Actions[Action_Hash] = false;
431444

432445
// In case of partial check
433446
if (!Actions[Action_Decode] && !Actions[Action_Info] && Actions[Action_QuickCheckAfterEncode]) // Quick check after encoding
@@ -799,6 +812,11 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_LibraryV
799812
//---------------------------------------------------------------------------
800813
void matroska::Segment_Cluster()
801814
{
815+
IsList = true;
816+
817+
if (FileMap2)
818+
return;
819+
802820
if (RAWcooked_LibraryName.empty())
803821
{
804822
memcpy(Cluster_Levels, Levels, sizeof(Levels));
@@ -808,8 +826,6 @@ void matroska::Segment_Cluster()
808826
return;
809827
}
810828

811-
IsList = true;
812-
813829
// Check if Hashes check is useful
814830
if (Hashes_FromRAWcooked)
815831
{
@@ -850,6 +866,19 @@ void matroska::Segment_Cluster()
850866
Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
851867
if (ReversibilityData && !FrameWriter_Template->Compound)
852868
InitOutput_Find();
869+
870+
FileMap2 = FileMap;
871+
if (OpenStyle != filemap::method::mmap && OpenName)
872+
{
873+
FileMap = new filemap;
874+
if (FileMap->Open_ReadMode(*OpenName, OpenStyle, 0, 256 * 1024 * 1024))
875+
{
876+
//delete FileMap;
877+
FileMap = FileMap2;
878+
OpenStyle = filemap::method::mmap;
879+
}
880+
Buffer = *FileMap;
881+
}
853882
}
854883

855884
//---------------------------------------------------------------------------

0 commit comments

Comments
 (0)