Merge pull request #100 from xiaoyifang/feature/concurrent-fseek

feat:mdx lock free when loadArticle

chunkedstorage.cc

@@ -4,6 +4,7 @@
 #include "chunkedstorage.hh"
 #include <zlib.h>
 #include <string.h>
+#include <QDataStream>
 
 namespace ChunkedStorage {
 
@@ -134,25 +135,34 @@ char * Reader::getBlock( uint32_t address, vector< char > & chunk )
 
   // Read and decompress the chunk
   {
-    file.seek( offsets[ chunkIdx ] );
+    // file.seek( offsets[ chunkIdx ] );
 
-    uint32_t uncompressedSize = file.read< uint32_t >();
+    auto bytes = file.map( offsets[ chunkIdx ], 8 );
-    uint32_t compressedSize = file.read< uint32_t >();
+    auto qBytes = QByteArray::fromRawData( reinterpret_cast< char * >(bytes), 8 );
+    QDataStream in( qBytes );
+    in.setByteOrder( QDataStream::LittleEndian );
 
+    uint32_t uncompressedSize;
+    uint32_t compressedSize;
+    // = file.read< uint32_t >();
+
+    in >> uncompressedSize >> compressedSize;
+
+    file.unmap( bytes );
     chunk.resize( uncompressedSize );
 
-    vector< unsigned char > compressedData( compressedSize );
+    // vector< unsigned char > compressedData( compressedSize );
+    auto chunkDataBytes = file.map( offsets[ chunkIdx ] + 8, compressedSize );
 
-    file.read( &compressedData.front(), compressedData.size() );
+    // file.read( &compressedData.front(), compressedData.size() );
 
     unsigned long decompressedLength = chunk.size();
 
-    if ( uncompress( (unsigned char *)&chunk.front(),
+    if( uncompress( (unsigned char *)&chunk.front(), &decompressedLength, chunkDataBytes, compressedSize ) != Z_OK
-                     &decompressedLength,
+        || decompressedLength != chunk.size() )
-                     &compressedData.front(),
-                     compressedData.size() ) != Z_OK ||
-         decompressedLength != chunk.size() )
       throw exFailedToDecompressChunk();
+
+    file.unmap( chunkDataBytes );
   }
 
   size_t offsetInChunk = address & 0xffFF;

file.cc

@@ -256,6 +256,20 @@ void Class::seek( qint64 offset )
     throw exSeekError();
 }
 
+uchar * Class::map( qint64 offset, qint64 size )
+{
+  if( writeBuffer )
+    flushWriteBuffer();
+
+  return f.map( offset, size );
+}
+
+bool Class::unmap( uchar * address )
+{
+  return f.unmap( address );
+}
+
+
 void Class::seekCur( qint64 offset ) 
 {
   if ( writeBuffer )

file.hh

@@ -95,6 +95,7 @@ public:
 
   /// Seeks in the file, relative to its beginning.
   void seek( qint64 offset ) ;
+  uchar * map( qint64 offset, qint64 size );
   /// Seeks in the file, relative to the current position.
   void seekCur( qint64 offset ) ;
   /// Seeks in the file, relative to the end of file.
@@ -117,6 +118,7 @@ public:
   void close() ;
 
   ~Class() noexcept;
+  bool unmap( uchar * address );
 
 private:
 

ftshelpers.cc

@@ -679,7 +679,7 @@ void FTSResultsRequest::indexSearch( BtreeIndexing::BtreeIndex & ftsIndex,
       vector< char > chunk;
       char * linksPtr;
       {
-        Mutex::Lock _( dict.getFtsMutex() );
+        // Mutex::Lock _( dict.getFtsMutex() );
         linksPtr = chunks->getBlock( links[ x ].articleOffset, chunk );
       }
 
@@ -785,7 +785,7 @@ void FTSResultsRequest::combinedIndexSearch( BtreeIndexing::BtreeIndex & ftsInde
         vector< char > chunk;
         char * linksPtr;
         {
-          Mutex::Lock _( dict.getFtsMutex() );
+          // Mutex::Lock _( dict.getFtsMutex() );
           linksPtr = chunks->getBlock( links[ x ].articleOffset, chunk );
         }
 
@@ -844,7 +844,7 @@ void FTSResultsRequest::combinedIndexSearch( BtreeIndexing::BtreeIndex & ftsInde
           vector< char > chunk;
           char * linksPtr;
           {
-            Mutex::Lock _( dict.getFtsMutex() );
+            // Mutex::Lock _( dict.getFtsMutex() );
             linksPtr = chunks->getBlock( links[ x ].articleOffset, chunk );
           }
 
@@ -935,7 +935,7 @@ void FTSResultsRequest::fullIndexSearch( BtreeIndexing::BtreeIndex & ftsIndex,
         vector< char > chunk;
         char * linksPtr;
         {
-          Mutex::Lock _( dict.getFtsMutex() );
+          // Mutex::Lock _( dict.getFtsMutex() );
           linksPtr = chunks->getBlock( links[ x ].articleOffset, chunk );
         }
 

mdx.cc

@@ -163,7 +163,7 @@ public:
 
     MdictParser::RecordInfo indexEntry;
     vector< char > chunk;
-    Mutex::Lock _( idxMutex );
+    // Mutex::Lock _( idxMutex );
     const char * indexEntryPtr = chunks.getBlock( links[ 0 ].articleOffset, chunk );
     memcpy( &indexEntry, indexEntryPtr, sizeof( indexEntry ) );
 
@@ -867,7 +867,7 @@ const QString & MdxDictionary::getDescription()
   }
   else
   {
-    Mutex::Lock _( idxMutex );
+    // Mutex::Lock _( idxMutex );
     vector< char > chunk;
     char * dictDescription = chunks.getBlock( idxHeader.descriptionAddress, chunk );
     string str( dictDescription );
@@ -900,7 +900,7 @@ void MdxDictionary::loadIcon() noexcept
 void MdxDictionary::loadArticle( uint32_t offset, string & articleText, bool noFilter )
 {
   vector< char > chunk;
-  Mutex::Lock _( idxMutex );
+  // Mutex::Lock _( idxMutex );
 
   // Load record info from index
   MdictParser::RecordInfo recordInfo;