Add alternative methods for reading files

Using mmap only seems to be very slow with some NAS

Author: JeromeMartinez

Source/CLI/Global.cpp

@@ -242,6 +242,40 @@ int global::SetHash(bool Value)
     return 0;
 }
 
+//---------------------------------------------------------------------------
+int global::SetFileOpenMethod(const char* Value)
+{
+    if (strcmp(Value, "mmap") == 0)
+    {
+        FileOpenMethod = filemap::method::mmap;
+        return 0;
+    }
+    if (strcmp(Value, "fstream") == 0)
+    {
+        FileOpenMethod = filemap::method::fstream;
+        return 0;
+    }
+    if (strcmp(Value, "fopen") == 0)
+    {
+        FileOpenMethod = filemap::method::fopen;
+        return 0;
+    }
+    if (strcmp(Value, "open") == 0)
+    {
+        FileOpenMethod = filemap::method::open;
+        return 0;
+    }
+    #if defined(_WIN32) || defined(_WINDOWS)
+    if (strcmp(Value, "createfile") == 0)
+    {
+        FileOpenMethod = filemap::method::createfile;
+        return 0;
+    }
+    #endif //defined(_WIN32) || defined(_WINDOWS)
+    cerr << "Error: unknown io value '" << Value << "'." << endl;
+    return 1;
+}
+
 //---------------------------------------------------------------------------
 int global::SetAll(bool Value)
 {
@@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
     IgnoreLicenseKey = !License.IsSupported_License();
     SubLicenseId = 0;
     SubLicenseDur = 1;
+    FileOpenMethod = filemap::method::mmap;
     ShowLicenseKey = false;
     StoreLicenseKey = false;
     DisplayCommand = false;
@@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
             if (auto Value = SetAcceptFiles())
                 return Value;
         }
+        else if (strcmp(argv[i], "--io") == 0)
+        {
+            if (i + 1 == argc)
+                return Error_Missing(argv[i]);
+            int Value = SetFileOpenMethod(argv[++i]);
+            if (Value)
+                return Value;
+        }
         else if (!strcmp(argv[i], "-framerate"))
         {
             if (OptionsForOtherFiles)

Source/CLI/Global.h

@@ -38,6 +38,7 @@ class global
     string                      LicenseKey;
     uint64_t                    SubLicenseId;
     uint64_t                    SubLicenseDur;
+    filemap::method             FileOpenMethod;
     bool                        IgnoreLicenseKey;
     bool                        ShowLicenseKey;
     bool                        StoreLicenseKey;
@@ -100,6 +101,7 @@ class global
     int SetFrameMd5An(bool Value);
     int SetFrameMd5FileName(const char* FileName);
     int SetHash(bool Value);
+    int SetFileOpenMethod(const char* Value);
     int SetAll(bool Value);
 
 private:

Source/CLI/Main.cpp

@@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
 }
 
 //---------------------------------------------------------------------------
-int ParseFile_Compressed(parse_info& ParseInfo)
+int ParseFile_Compressed(parse_info& ParseInfo, const string* FileOpenName)
 {
     // Init
     string OutputDirectoryName;
@@ -522,6 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo)
         matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
         M->Quiet = Global.Quiet;
         M->NoOutputCheck = NoOutputCheck;
+        M->OpenName = FileOpenName;
+        M->OpenStyle = Global.FileOpenMethod;
         if (ParseInfo.ParseFile_Input(*M))
         {
             ReturnValue = 1;
@@ -591,7 +593,7 @@ int ParseFile(size_t Files_Pos)
         return 1;
 
     // Compressed content
-    if (int Value = ParseFile_Compressed(ParseInfo))
+    if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
         return Value;
     if (ParseInfo.IsDetected)
         return 0;
@@ -755,6 +757,7 @@ int main(int argc, const char* argv[])
         if (!Value)
         {
             // Configure for a 2nd pass
+            auto FileOpenName = Global.OutputFileName;
             ParseInfo.Name = NULL;
             Global.OutputFileName = Global.Inputs[0];
             if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes
@@ -772,7 +775,7 @@ int main(int argc, const char* argv[])
             // Parse (check mode)
             Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
             Global.Actions.set(Action_Decode, false); // Override config
-            Value = ParseFile_Compressed(ParseInfo);
+            Value = ParseFile_Compressed(ParseInfo, &FileOpenName);
             if (!Value && !ParseInfo.IsDetected)
             {
                 cout << '\n' << "Error: " << Global.OutputFileName << endl;

Source/Lib/Compressed/Matroska/Matroska.cpp

@@ -359,13 +359,16 @@ void matroska::ParseBuffer()
         // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
         if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value 
         {
-            FileMap->Remap();
+            FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
             Buffer = *FileMap;
+            if (OpenStyle == filemap::method::mmap)
+            {
             if (ReversibilityData)
                 ReversibilityData->SetBaseData(Buffer.Data());
             for (const auto& TrackInfo_Current : TrackInfo)
                 if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
                     TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
+            }
             Buffer_Offset_LowerLimit = Buffer_Offset;
         }
 
@@ -376,7 +379,7 @@ void matroska::ParseBuffer()
             Buffer_Offset = Cluster_Offset;
             Cluster_Level = (size_t)-1;
 
-            FileMap->Remap();
+            FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
             Buffer = *FileMap;
             if (ReversibilityData)
                 ReversibilityData->SetBaseData(Buffer.Data());
@@ -850,6 +853,19 @@ void matroska::Segment_Cluster()
             Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
     if (ReversibilityData && !FrameWriter_Template->Compound)
         InitOutput_Find();
+
+    Actions[Action_Hash] = false;
+
+    if (!FileMap2)
+    {
+        FileMap2 = FileMap;
+        if (OpenStyle != filemap::method::mmap && OpenName)
+        {
+            FileMap = new filemap;
+            FileMap->Open_ReadMode(*OpenName, OpenStyle, 0, 256 * 1024 * 1024);
+            Buffer = *FileMap;
+        }
+    }
 }
 
 //---------------------------------------------------------------------------

Source/Lib/Utils/FileIO/FileIO.cpp

@@ -11,8 +11,11 @@
 #include "Lib/Utils/FileIO/FileIO.h"
 #include <iostream>
 #include <sstream>
+#include <fstream>
 #if defined(_WIN32) || defined(_WINDOWS)
     #include "windows.h"
+    #include <stdio.h>
+    #include <fcntl.h>
     #include <io.h> // File existence
     #include <direct.h> // Directory creation
     #define access _access_s
@@ -29,10 +32,106 @@
 //---------------------------------------------------------------------------
 
 //---------------------------------------------------------------------------
-int filemap::Open_ReadMode(const char* FileName)
+struct private_buffered
+{
+    union f
+    {
+        FILE*           File;
+        ifstream*       Ifstream;
+        int             Int;
+        #if defined(_WIN32) || defined(_WINDOWS)
+        HANDLE          Handle;
+        #endif
+    };
+    f                   F;
+    size_t              Data_Shift = 0;
+    size_t              MaxSize = 0;
+};
+
+//---------------------------------------------------------------------------
+int filemap::Open_ReadMode(const char* FileName, method NewStyle, size_t Begin, size_t End)
 {
     Close();
 
+    if (NewStyle != method::mmap)
+    {
+        Method = NewStyle;
+        private_buffered* P = new private_buffered;
+        P->MaxSize = End - Begin;
+        size_t FileSize;
+
+        switch (Method)
+        {
+        default: // case style::fstream:
+        {
+            auto F = new ifstream(FileName, ios::binary);
+            F->seekg(0, F->end);
+            FileSize = F->tellg();
+            F->seekg(Begin, F->beg);
+            P->F.Ifstream = F;
+            break;
+        }
+        case method::fopen:
+        {
+            struct stat Fstat;
+            if (stat(FileName, &Fstat))
+                return 1;
+            FileSize = Fstat.st_size;
+            auto F = fopen(FileName, "rb");
+            P->F.File = F;
+            break;
+        }
+        case method::open:
+        {
+            struct stat Fstat;
+            if (stat(FileName, &Fstat))
+                return 1;
+            FileSize = Fstat.st_size;
+            #if defined(_WIN32) || defined(_WINDOWS)
+            auto F = _open(FileName, _O_BINARY | _O_RDONLY | _O_SEQUENTIAL, _S_IREAD);
+            #else //defined(_WIN32) || defined(_WINDOWS)
+            auto F = open(FileName, O_RDONLY);
+            #endif //defined(_WIN32) || defined(_WINDOWS)
+            if (F == -1)
+                return 1;
+            P->F.Int = F;
+            break;
+        }
+        #if defined(_WIN32) || defined(_WINDOWS)
+        case method::createfile:
+        {
+            DWORD FileSizeHigh;
+            auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+            auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh);
+            if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value)
+                && (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode
+            {
+                FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow;
+            }
+            else
+                return 1;
+            if (Begin)
+            {
+                LARGE_INTEGER GoTo;
+                GoTo.QuadPart = Begin;
+                if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0))
+                    return 1;
+                P->Data_Shift = Begin;
+            }
+            P->F.Handle = NewFile;
+            break;
+        }
+        #endif //defined(_WIN32) || defined(_WINDOWS)
+        }
+
+        auto Buffer = new uint8_t[P->MaxSize];
+        P->Data_Shift -= P->MaxSize;
+        AssignBase(Buffer - P->Data_Shift, FileSize);
+        Private2 = (decltype(Private2))P;
+
+        return Remap(Begin, End);
+    }
+
     size_t NewSize;
 #if defined(_WIN32) || defined(_WINDOWS)
     auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
@@ -99,12 +198,63 @@ inline int munmap_const(const void* addr, size_t length)
 #pragma GCC diagnostic pop
 #endif
 #endif
-int filemap::Remap()
+int filemap::Remap(size_t Begin, size_t End)
 {
     // Special case for 0-byte files
     if (Empty())
         return 0;
 
+    if (Method != method::mmap)
+    {
+        auto P = (private_buffered*)Private2;
+        auto Buffer = Data() + P->Data_Shift;
+        auto Buffer_MaxSize = P->MaxSize;
+        Begin -= P->Data_Shift;
+        if (!End)
+            End = Size();
+        End -= P->Data_Shift;
+        auto Buffer_Middle = Buffer + Begin;
+        auto Buffer_Middle_Size = Buffer_MaxSize - Begin;
+        memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size);
+        P->Data_Shift += Begin;
+        AssignKeepSizeBase(Buffer - P->Data_Shift);
+        Buffer += Buffer_Middle_Size;
+        Buffer_MaxSize -= Buffer_Middle_Size;
+
+        switch (Method)
+        {
+        default: // case style::fstream:
+        {
+            auto F = P->F.Ifstream;
+            F->read((char*)Buffer, Buffer_MaxSize);
+            break;
+        }
+        case method::fopen:
+        {
+            auto F = P->F.File;
+            if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1)
+                return 1;
+            break;
+        }
+        case method::open:
+        {
+            auto F = P->F.Int;
+            read(F, (void*)Buffer, Buffer_MaxSize);
+            break;
+        }
+        #if defined(_WIN32) || defined(_WINDOWS)
+        case method::createfile:
+        {
+            auto F = P->F.Handle;
+            ReadFile(F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0);
+            break;
+        }
+        #endif //defined(_WIN32) || defined(_WINDOWS)
+        }
+
+        return 0;
+    }
+
     // Close previous map
     if (Data())
     {

Source/Lib/Utils/FileIO/FileIO.h

@@ -23,19 +23,30 @@ class filemap : public buffer_view
                                 ~filemap() { Close(); }
 
     // Actions
-    int                         Open_ReadMode(const char* FileName);
-    int                         Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); }
+    enum class method
+    {
+        mmap,
+        fstream,
+        fopen,
+        open,
+        #if defined(_WIN32) || defined(_WINDOWS)
+        createfile,
+        #endif //defined(_WIN32) || defined(_WINDOWS)
+    };
+    int                         Open_ReadMode(const char* FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {});
+    int                         Open_ReadMode(const string& FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {}) { return Open_ReadMode(FileName.c_str(), NewMethod, Begin, End); }
     bool                        IsOpen() { return Private == (decltype(Private))-1 ? false : true; }
-    int                         Remap();
+    int                         Remap(size_t Begin = 0, size_t End = 0);
     int                         Close();
 
 private:
     #if defined(_WIN32) || defined(_WINDOWS)
     void*                       Private = (void*)-1;
-    void*                       Private2 = (void*)-1;
     #else //defined(_WIN32) || defined(_WINDOWS)
     int                         Private = (int)-1;
     #endif //defined(_WIN32) || defined(_WINDOWS)
+    void*                       Private2 = (void*)-1;
+    method                      Method = {};
 };
 
 class file

Source/Lib/Utils/FileIO/Input_Base.cpp

@@ -38,6 +38,7 @@ input_base::~input_base()
 bool input_base::Parse(filemap* FileMap_Source, const buffer_view& Buffer_Source, size_t FileSize_Source)
 {
     FileMap = FileMap_Source;
+    FileMap2 = nullptr;
     FileSize = FileSize_Source == (size_t)-1 ? Buffer_Source.Size() : FileSize_Source;
     Buffer = Buffer_Source;
     HashComputed = false;