2013-05-08 13:39:47 +00:00
|
|
|
/* This file is (c) 2013 Timon Wong <timon86.wang AT gmail DOT com>
|
2013-04-23 12:07:05 +00:00
|
|
|
* Part of GoldenDict. Licensed under GPLv3 or later, see the LICENSE file */
|
|
|
|
|
|
|
|
#include "mdx.hh"
|
|
|
|
#include "btreeidx.hh"
|
|
|
|
#include "folding.hh"
|
|
|
|
#include "utf8.hh"
|
|
|
|
#include "file.hh"
|
|
|
|
#include "wstring.hh"
|
|
|
|
#include "wstring_qt.hh"
|
|
|
|
#include "chunkedstorage.hh"
|
2013-11-16 18:34:09 +00:00
|
|
|
#include "gddebug.hh"
|
2013-04-23 12:07:05 +00:00
|
|
|
#include "langcoder.hh"
|
|
|
|
#include "fsencoding.hh"
|
|
|
|
#include "audiolink.hh"
|
2013-04-28 08:26:04 +00:00
|
|
|
#include "ex.hh"
|
2013-04-23 12:07:05 +00:00
|
|
|
#include "mdictparser.hh"
|
2013-05-23 14:01:50 +00:00
|
|
|
#include "filetype.hh"
|
2014-04-16 16:18:28 +00:00
|
|
|
#include "ftshelpers.hh"
|
|
|
|
#include "htmlescape.hh"
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-06-17 15:30:20 +00:00
|
|
|
#include <algorithm>
|
2013-04-23 12:07:05 +00:00
|
|
|
#include <map>
|
|
|
|
#include <set>
|
|
|
|
#include <list>
|
|
|
|
#include <ctype.h>
|
2013-04-24 11:35:03 +00:00
|
|
|
#include <stdlib.h>
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
#ifdef _MSC_VER
|
|
|
|
#include <stub_msvc.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include <QDir>
|
|
|
|
#include <QString>
|
|
|
|
#include <QSemaphore>
|
|
|
|
#include <QThreadPool>
|
|
|
|
#include <QAtomicInt>
|
|
|
|
#include <QTextDocument>
|
|
|
|
#include <QCryptographicHash>
|
|
|
|
|
2018-02-27 16:42:21 +00:00
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
#include <QRegularExpression>
|
|
|
|
#endif
|
|
|
|
|
2013-05-30 13:24:21 +00:00
|
|
|
#include "qt4x5.hh"
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
namespace Mdx
|
|
|
|
{
|
|
|
|
|
|
|
|
using std::map;
|
|
|
|
using std::multimap;
|
|
|
|
using std::set;
|
|
|
|
using gd::wstring;
|
|
|
|
using gd::wchar;
|
|
|
|
using std::list;
|
|
|
|
using std::pair;
|
|
|
|
using std::string;
|
|
|
|
|
|
|
|
using BtreeIndexing::WordArticleLink;
|
|
|
|
using BtreeIndexing::IndexedWords;
|
|
|
|
using BtreeIndexing::IndexInfo;
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
using namespace Mdict;
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
enum
|
|
|
|
{
|
|
|
|
kSignature = 0x4349444d, // MDIC
|
2018-04-08 14:57:10 +00:00
|
|
|
kCurrentFormatVersion = 11 + BtreeIndexing::FormatVersion + Folding::Version
|
2013-04-23 12:07:05 +00:00
|
|
|
};
|
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
DEF_EX( exCorruptDictionary, "dictionary file was tampered or corrupted", std::exception )
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
struct IdxHeader
|
|
|
|
{
|
|
|
|
uint32_t signature; // First comes the signature, MDIC
|
|
|
|
uint32_t formatVersion; // File format version, currently 1.
|
|
|
|
uint32_t parserVersion; // Version of the parser used to parse the MDIC file.
|
|
|
|
// Version of the folding algorithm used when building
|
|
|
|
// index. If it's different from the current one,
|
|
|
|
// the file is to be rebuilt.
|
|
|
|
uint32_t foldingVersion;
|
|
|
|
|
|
|
|
uint32_t articleCount; // Total number of articles, for informative purposes only
|
|
|
|
uint32_t wordCount; // Total number of words, for informative purposes only
|
|
|
|
|
|
|
|
uint32_t isRightToLeft; // Right to left
|
|
|
|
uint32_t chunksOffset; // The offset to chunks' storage
|
|
|
|
|
|
|
|
uint32_t descriptionAddress; // Address of the dictionary description in the chunks' storage
|
|
|
|
uint32_t descriptionSize; // Size of the description in the chunks' storage, 0 = no description
|
|
|
|
|
|
|
|
uint32_t styleSheetAddress;
|
|
|
|
uint32_t styleSheetCount;
|
|
|
|
|
|
|
|
uint32_t indexBtreeMaxElements; // Two fields from IndexInfo
|
|
|
|
uint32_t indexRootOffset;
|
|
|
|
|
|
|
|
uint32_t langFrom; // Source language
|
|
|
|
uint32_t langTo; // Target language
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
uint32_t mddIndexInfosOffset; // address of IndexInfos for resource files (.mdd)
|
|
|
|
uint32_t mddIndexInfosCount; // count of IndexInfos for resource files
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
#ifndef _MSC_VER
|
|
|
|
__attribute__( ( packed ) )
|
|
|
|
#endif
|
|
|
|
;
|
|
|
|
|
|
|
|
// A helper method to read resources from .mdd file
|
|
|
|
class IndexedMdd: public BtreeIndexing::BtreeIndex
|
|
|
|
{
|
|
|
|
Mutex & idxMutex;
|
2014-04-16 16:18:28 +00:00
|
|
|
Mutex fileMutex;
|
2013-04-23 12:07:05 +00:00
|
|
|
ChunkedStorage::Reader & chunks;
|
|
|
|
QFile mddFile;
|
|
|
|
bool isFileOpen;
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
IndexedMdd( Mutex & idxMutex, ChunkedStorage::Reader & chunks ):
|
|
|
|
idxMutex( idxMutex ),
|
|
|
|
chunks( chunks ),
|
|
|
|
isFileOpen( false )
|
|
|
|
{}
|
|
|
|
|
|
|
|
/// Opens the index. The values are those previously returned by buildIndex().
|
|
|
|
using BtreeIndexing::BtreeIndex::openIndex;
|
|
|
|
|
|
|
|
/// Opens the mdd file itself. Returns true if succeeded, false otherwise.
|
|
|
|
bool open( const char * fileName )
|
|
|
|
{
|
|
|
|
mddFile.setFileName( QString::fromUtf8( fileName ) );
|
|
|
|
isFileOpen = mddFile.open( QFile::ReadOnly );
|
|
|
|
return isFileOpen;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns true if the mdd is open, false otherwise.
|
|
|
|
inline bool isOpen() const
|
|
|
|
{
|
|
|
|
return isFileOpen;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Checks whether the given file exists in the mdd file or not.
|
|
|
|
/// Note that this function is thread-safe, since it does not access mdd file.
|
|
|
|
bool hasFile( gd::wstring const & name )
|
|
|
|
{
|
|
|
|
if ( !isFileOpen )
|
|
|
|
return false;
|
|
|
|
vector< WordArticleLink > links = findArticles( name );
|
|
|
|
return !links.empty();
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Attempts loading the given file into the given vector. Returns true on
|
|
|
|
/// success, false otherwise.
|
|
|
|
bool loadFile( gd::wstring const & name, std::vector< char > & result )
|
|
|
|
{
|
|
|
|
if ( !isFileOpen )
|
|
|
|
return false;
|
|
|
|
|
|
|
|
vector< WordArticleLink > links = findArticles( name );
|
|
|
|
if ( links.empty() )
|
|
|
|
return false;
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
MdictParser::RecordInfo indexEntry;
|
|
|
|
vector< char > chunk;
|
|
|
|
Mutex::Lock _( idxMutex );
|
|
|
|
const char * indexEntryPtr = chunks.getBlock( links[ 0 ].articleOffset, chunk );
|
|
|
|
memcpy( &indexEntry, indexEntryPtr, sizeof( indexEntry ) );
|
|
|
|
|
|
|
|
ScopedMemMap compressed( mddFile, indexEntry.compressedBlockPos, indexEntry.compressedBlockSize );
|
|
|
|
if ( !compressed.startAddress() )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-04-28 08:26:04 +00:00
|
|
|
return false;
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
QByteArray decompressed;
|
2013-04-28 08:26:04 +00:00
|
|
|
if ( !MdictParser::parseCompressedBlock( indexEntry.compressedBlockSize, ( char * )compressed.startAddress(),
|
2013-04-23 12:07:05 +00:00
|
|
|
indexEntry.decompressedBlockSize, decompressed ) )
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
result.resize( indexEntry.recordSize );
|
|
|
|
memcpy( &result.front(), decompressed.constData() + indexEntry.recordOffset, indexEntry.recordSize );
|
2013-04-23 12:07:05 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
class MdxDictionary: public BtreeIndexing::BtreeDictionary
|
|
|
|
{
|
|
|
|
Mutex idxMutex;
|
|
|
|
File::Class idx;
|
|
|
|
IdxHeader idxHeader;
|
|
|
|
string dictionaryName;
|
2013-04-28 08:26:04 +00:00
|
|
|
string encoding;
|
2013-04-23 12:07:05 +00:00
|
|
|
ChunkedStorage::Reader chunks;
|
2013-04-28 08:26:04 +00:00
|
|
|
QFile dictFile;
|
2013-05-08 13:39:47 +00:00
|
|
|
vector< sptr< IndexedMdd > > mddResources;
|
2013-04-23 12:07:05 +00:00
|
|
|
MdictParser::StyleSheets styleSheets;
|
|
|
|
|
|
|
|
QAtomicInt deferredInitDone;
|
|
|
|
Mutex deferredInitMutex;
|
|
|
|
bool deferredInitRunnableStarted;
|
|
|
|
QSemaphore deferredInitRunnableExited;
|
|
|
|
|
|
|
|
string initError;
|
2021-11-12 18:26:46 +00:00
|
|
|
QString cacheDirName;
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
MdxDictionary( string const & id, string const & indexFile, vector<string> const & dictionaryFiles );
|
|
|
|
|
|
|
|
~MdxDictionary();
|
|
|
|
|
|
|
|
virtual void deferredInit();
|
|
|
|
|
|
|
|
virtual string getName() throw()
|
|
|
|
{
|
|
|
|
return dictionaryName;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual map< Dictionary::Property, string > getProperties() throw()
|
|
|
|
{
|
|
|
|
return map< Dictionary::Property, string >();
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual unsigned long getArticleCount() throw()
|
|
|
|
{
|
|
|
|
return idxHeader.articleCount;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual unsigned long getWordCount() throw()
|
|
|
|
{
|
|
|
|
return idxHeader.wordCount;
|
|
|
|
}
|
|
|
|
|
|
|
|
inline virtual quint32 getLangFrom() const
|
|
|
|
{
|
|
|
|
return idxHeader.langFrom;
|
|
|
|
}
|
|
|
|
|
|
|
|
inline virtual quint32 getLangTo() const
|
|
|
|
{
|
|
|
|
return idxHeader.langTo;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual sptr< Dictionary::DataRequest > getArticle( wstring const & word,
|
|
|
|
vector< wstring > const & alts,
|
2018-06-13 16:00:42 +00:00
|
|
|
wstring const &,
|
|
|
|
bool ignoreDiacritics ) THROW_SPEC( std::exception );
|
2018-05-21 15:32:04 +00:00
|
|
|
virtual sptr< Dictionary::DataRequest > getResource( string const & name ) THROW_SPEC( std::exception );
|
2013-04-23 12:07:05 +00:00
|
|
|
virtual QString const & getDescription();
|
|
|
|
|
2014-04-16 16:18:28 +00:00
|
|
|
virtual sptr< Dictionary::DataRequest > getSearchResults( QString const & searchString,
|
|
|
|
int searchMode, bool matchCase,
|
|
|
|
int distanceBetweenWords,
|
2017-07-25 15:28:29 +00:00
|
|
|
int maxResults,
|
2018-04-10 14:49:52 +00:00
|
|
|
bool ignoreWordsOrder,
|
|
|
|
bool ignoreDiacritics );
|
2014-04-16 16:18:28 +00:00
|
|
|
virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text );
|
|
|
|
|
|
|
|
virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration );
|
|
|
|
|
2014-04-17 14:31:51 +00:00
|
|
|
virtual void setFTSParameters( Config::FullTextSearch const & fts )
|
|
|
|
{
|
|
|
|
if( ensureInitDone().size() )
|
|
|
|
return;
|
|
|
|
|
|
|
|
can_FTS = fts.enabled
|
|
|
|
&& !fts.disabledTypes.contains( "MDICT", Qt::CaseInsensitive )
|
|
|
|
&& ( fts.maxDictionarySize == 0 || getArticleCount() <= fts.maxDictionarySize );
|
|
|
|
}
|
2021-11-12 18:26:46 +00:00
|
|
|
|
|
|
|
QString getCachedFileName( QString name );
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
protected:
|
|
|
|
|
|
|
|
virtual void loadIcon() throw();
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
|
|
|
virtual string const & ensureInitDone();
|
|
|
|
void doDeferredInit();
|
|
|
|
|
|
|
|
/// Loads an article with the given offset, filling the given strings.
|
2014-04-29 13:55:48 +00:00
|
|
|
void loadArticle( uint32_t offset, string & articleText, bool noFilter = false );
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
/// Process resource links (images, audios, etc)
|
2013-05-22 03:15:47 +00:00
|
|
|
QString & filterResource( QString const & articleId, QString & article );
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2021-11-12 18:26:46 +00:00
|
|
|
void removeDirectory( QString const & directory );
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
friend class MdxHeadwordsRequest;
|
|
|
|
friend class MdxArticleRequest;
|
|
|
|
friend class MddResourceRequest;
|
|
|
|
friend class MdxDeferredInitRunnable;
|
|
|
|
};
|
|
|
|
|
|
|
|
MdxDictionary::MdxDictionary( string const & id, string const & indexFile,
|
|
|
|
vector<string> const & dictionaryFiles ):
|
|
|
|
BtreeDictionary( id, dictionaryFiles ),
|
|
|
|
idx( indexFile, "rb" ),
|
|
|
|
idxHeader( idx.read< IdxHeader >() ),
|
|
|
|
chunks( idx, idxHeader.chunksOffset ),
|
|
|
|
deferredInitRunnableStarted( false )
|
|
|
|
{
|
|
|
|
// Read the dictionary's name
|
2013-04-28 08:26:04 +00:00
|
|
|
idx.seek( sizeof( idxHeader ) );
|
2013-04-23 12:07:05 +00:00
|
|
|
size_t len = idx.read< uint32_t >();
|
2013-04-28 08:26:04 +00:00
|
|
|
vector< char > buf( len );
|
2019-01-17 14:53:13 +00:00
|
|
|
if( len > 0 )
|
|
|
|
{
|
|
|
|
idx.read( &buf.front(), len );
|
|
|
|
dictionaryName = string( &buf.front(), len );
|
|
|
|
}
|
2013-04-28 08:26:04 +00:00
|
|
|
|
|
|
|
// then read the dictionary's encoding
|
|
|
|
len = idx.read< uint32_t >();
|
2019-01-17 14:53:13 +00:00
|
|
|
if( len > 0 )
|
|
|
|
{
|
|
|
|
buf.resize( len );
|
|
|
|
idx.read( &buf.front(), len );
|
|
|
|
encoding = string( &buf.front(), len );
|
|
|
|
}
|
2013-04-28 08:26:04 +00:00
|
|
|
|
|
|
|
dictFile.setFileName( QString::fromUtf8( dictionaryFiles[ 0 ].c_str() ) );
|
|
|
|
dictFile.open( QIODevice::ReadOnly );
|
2014-04-16 16:18:28 +00:00
|
|
|
|
|
|
|
// Full-text search parameters
|
|
|
|
|
|
|
|
can_FTS = true;
|
|
|
|
|
2014-05-08 12:38:00 +00:00
|
|
|
ftsIdxName = indexFile + "_FTS";
|
2014-04-16 16:18:28 +00:00
|
|
|
|
2014-05-08 12:38:00 +00:00
|
|
|
if( !Dictionary::needToRebuildIndex( dictionaryFiles, ftsIdxName )
|
2014-11-22 14:22:04 +00:00
|
|
|
&& !FtsHelpers::ftsIndexIsOldOrBad( ftsIdxName, this ) )
|
2014-05-08 12:38:00 +00:00
|
|
|
FTS_index_completed.ref();
|
2021-11-12 18:26:46 +00:00
|
|
|
|
|
|
|
cacheDirName = QDir::tempPath() + QDir::separator()
|
|
|
|
+ QString::fromUtf8( getId().c_str() )
|
|
|
|
+ ".cache";
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
MdxDictionary::~MdxDictionary()
|
|
|
|
{
|
|
|
|
Mutex::Lock _( deferredInitMutex );
|
|
|
|
|
|
|
|
// Wait for init runnable to complete if it was ever started
|
|
|
|
if ( deferredInitRunnableStarted )
|
|
|
|
deferredInitRunnableExited.acquire();
|
2013-04-28 08:26:04 +00:00
|
|
|
|
|
|
|
dictFile.close();
|
2021-11-12 18:26:46 +00:00
|
|
|
|
|
|
|
removeDirectory( cacheDirName );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
//////// MdxDictionary::deferredInit()
|
|
|
|
|
|
|
|
class MdxDeferredInitRunnable: public QRunnable
|
|
|
|
{
|
|
|
|
MdxDictionary & dictionary;
|
|
|
|
QSemaphore & hasExited;
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
MdxDeferredInitRunnable( MdxDictionary & dictionary_,
|
|
|
|
QSemaphore & hasExited_ ):
|
|
|
|
dictionary( dictionary_ ), hasExited( hasExited_ )
|
|
|
|
{}
|
|
|
|
|
|
|
|
~MdxDeferredInitRunnable()
|
|
|
|
{
|
|
|
|
hasExited.release();
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual void run()
|
|
|
|
{
|
|
|
|
dictionary.doDeferredInit();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
void MdxDictionary::deferredInit()
|
|
|
|
{
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( !Qt4x5::AtomicInt::loadAcquire( deferredInitDone ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
Mutex::Lock _( deferredInitMutex );
|
|
|
|
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( Qt4x5::AtomicInt::loadAcquire( deferredInitDone ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
if ( !deferredInitRunnableStarted )
|
|
|
|
{
|
|
|
|
QThreadPool::globalInstance()->start(
|
|
|
|
new MdxDeferredInitRunnable( *this, deferredInitRunnableExited ),
|
|
|
|
-1000 );
|
|
|
|
deferredInitRunnableStarted = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
string const & MdxDictionary::ensureInitDone()
|
|
|
|
{
|
|
|
|
doDeferredInit();
|
|
|
|
return initError;
|
|
|
|
}
|
|
|
|
|
|
|
|
void MdxDictionary::doDeferredInit()
|
|
|
|
{
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( !Qt4x5::AtomicInt::loadAcquire( deferredInitDone ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
Mutex::Lock _( deferredInitMutex );
|
|
|
|
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( Qt4x5::AtomicInt::loadAcquire( deferredInitDone ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
// Do deferred init
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
// Retrieve stylesheets
|
|
|
|
idx.seek( idxHeader.styleSheetAddress );
|
|
|
|
for ( uint32_t i = 0; i < idxHeader.styleSheetCount; i++ )
|
|
|
|
{
|
2013-06-04 16:17:55 +00:00
|
|
|
qint32 key = idx.read< qint32 >();
|
2013-04-23 12:07:05 +00:00
|
|
|
vector< char > buf;
|
2013-06-04 16:17:55 +00:00
|
|
|
quint32 sz;
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-06-04 16:17:55 +00:00
|
|
|
sz = idx.read< quint32 >();
|
2013-04-23 12:07:05 +00:00
|
|
|
buf.resize( sz );
|
|
|
|
idx.read( &buf.front(), sz );
|
|
|
|
QString styleBegin = QString::fromUtf8( buf.data() );
|
|
|
|
|
2013-06-04 16:17:55 +00:00
|
|
|
sz = idx.read< quint32 >();
|
2013-04-23 12:07:05 +00:00
|
|
|
buf.resize( sz );
|
|
|
|
idx.read( &buf.front(), sz );
|
|
|
|
QString styleEnd = QString::fromUtf8( buf.data() );
|
|
|
|
|
|
|
|
styleSheets[ key ] = pair<QString, QString>( styleBegin, styleEnd );
|
|
|
|
}
|
|
|
|
|
|
|
|
// Initialize the index
|
|
|
|
openIndex( IndexInfo( idxHeader.indexBtreeMaxElements,
|
|
|
|
idxHeader.indexRootOffset ), idx, idxMutex );
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
vector< string > mddFileNames;
|
|
|
|
vector< IndexInfo > mddIndexInfos;
|
|
|
|
idx.seek( idxHeader.mddIndexInfosOffset );
|
|
|
|
for ( uint32_t i = 0; i < idxHeader.mddIndexInfosCount; i++ )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-06-04 16:17:55 +00:00
|
|
|
quint32 sz = idx.read< quint32 >();
|
2013-05-08 13:39:47 +00:00
|
|
|
vector< char > buf( sz );
|
|
|
|
idx.read( &buf.front(), sz );
|
|
|
|
uint32_t btreeMaxElements = idx.read<uint32_t>();
|
|
|
|
uint32_t rootOffset = idx.read<uint32_t>();
|
|
|
|
mddFileNames.push_back( string( &buf.front() ) );
|
|
|
|
mddIndexInfos.push_back( IndexInfo( btreeMaxElements, rootOffset ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
vector< string > const dictFiles = getDictionaryFilenames();
|
|
|
|
for ( uint32_t i = 1; i < dictFiles.size() && i < mddFileNames.size() + 1; i++ )
|
|
|
|
{
|
2013-06-04 16:17:55 +00:00
|
|
|
QFileInfo fi( QString::fromUtf8( dictFiles[ i ].c_str() ) );
|
|
|
|
QString mddFileName = QString::fromUtf8( mddFileNames[ i - 1 ].c_str() );
|
|
|
|
|
|
|
|
if ( fi.fileName() != mddFileName || !fi.exists() )
|
2013-05-08 13:39:47 +00:00
|
|
|
continue;
|
|
|
|
|
2013-05-11 05:41:26 +00:00
|
|
|
sptr< IndexedMdd > mdd = new IndexedMdd( idxMutex, chunks );
|
2013-05-08 13:39:47 +00:00
|
|
|
mdd->openIndex( mddIndexInfos[ i - 1 ], idx, idxMutex );
|
|
|
|
mdd->open( dictFiles[ i ].c_str() );
|
|
|
|
mddResources.push_back( mdd );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
catch ( std::exception & e )
|
|
|
|
{
|
|
|
|
initError = e.what();
|
|
|
|
}
|
|
|
|
catch ( ... )
|
|
|
|
{
|
|
|
|
initError = "Unknown error";
|
|
|
|
}
|
|
|
|
|
|
|
|
deferredInitDone.ref();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-04-16 16:18:28 +00:00
|
|
|
void MdxDictionary::makeFTSIndex( QAtomicInt & isCancelled, bool firstIteration )
|
|
|
|
{
|
|
|
|
if( !( Dictionary::needToRebuildIndex( getDictionaryFilenames(), ftsIdxName )
|
2014-11-22 14:22:04 +00:00
|
|
|
|| FtsHelpers::ftsIndexIsOldOrBad( ftsIdxName, this ) ) )
|
2014-04-16 16:18:28 +00:00
|
|
|
FTS_index_completed.ref();
|
|
|
|
|
|
|
|
if( haveFTSIndex() )
|
|
|
|
return;
|
|
|
|
|
|
|
|
if( ensureInitDone().size() )
|
|
|
|
return;
|
|
|
|
|
|
|
|
if( firstIteration && getArticleCount() > FTS::MaxDictionarySizeForFastSearch )
|
|
|
|
return;
|
|
|
|
|
|
|
|
gdDebug( "MDict: Building the full-text index for dictionary: %s\n",
|
|
|
|
getName().c_str() );
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
FtsHelpers::makeFTSIndex( this, isCancelled );
|
2014-04-17 14:31:51 +00:00
|
|
|
FTS_index_completed.ref();
|
2014-04-16 16:18:28 +00:00
|
|
|
}
|
|
|
|
catch( std::exception &ex )
|
|
|
|
{
|
|
|
|
gdWarning( "MDict: Failed building full-text search index for \"%s\", reason: %s\n", getName().c_str(), ex.what() );
|
|
|
|
QFile::remove( FsEncoding::decode( ftsIdxName.c_str() ) );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void MdxDictionary::getArticleText( uint32_t articleAddress, QString & headword, QString & text )
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
headword.clear();
|
|
|
|
string articleText;
|
|
|
|
|
2014-04-29 13:55:48 +00:00
|
|
|
loadArticle( articleAddress, articleText, true );
|
2014-04-16 16:18:28 +00:00
|
|
|
text = Html::unescape( QString::fromUtf8( articleText.data(), articleText.size() ) );
|
|
|
|
}
|
|
|
|
catch( std::exception &ex )
|
|
|
|
{
|
|
|
|
gdWarning( "MDict: Failed retrieving article from \"%s\", reason: %s\n", getName().c_str(), ex.what() );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
sptr< Dictionary::DataRequest > MdxDictionary::getSearchResults( QString const & searchString,
|
|
|
|
int searchMode, bool matchCase,
|
|
|
|
int distanceBetweenWords,
|
2017-07-25 15:28:29 +00:00
|
|
|
int maxResults,
|
2018-04-10 14:49:52 +00:00
|
|
|
bool ignoreWordsOrder,
|
|
|
|
bool ignoreDiacritics )
|
2014-04-16 16:18:28 +00:00
|
|
|
{
|
2018-04-10 14:49:52 +00:00
|
|
|
return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics );
|
2014-04-16 16:18:28 +00:00
|
|
|
}
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
/// MdxDictionary::getArticle
|
|
|
|
|
|
|
|
class MdxArticleRequest;
|
|
|
|
|
|
|
|
class MdxArticleRequestRunnable: public QRunnable
|
|
|
|
{
|
|
|
|
MdxArticleRequest & r;
|
|
|
|
QSemaphore & hasExited;
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
MdxArticleRequestRunnable( MdxArticleRequest & r_,
|
|
|
|
QSemaphore & hasExited_ ):
|
|
|
|
r( r_ ),
|
|
|
|
hasExited( hasExited_ )
|
|
|
|
{}
|
|
|
|
|
|
|
|
~MdxArticleRequestRunnable()
|
|
|
|
{
|
|
|
|
hasExited.release();
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual void run();
|
|
|
|
};
|
|
|
|
|
|
|
|
class MdxArticleRequest: public Dictionary::DataRequest
|
|
|
|
{
|
|
|
|
friend class MdxArticleRequestRunnable;
|
|
|
|
|
|
|
|
wstring word;
|
|
|
|
vector< wstring > alts;
|
|
|
|
MdxDictionary & dict;
|
2018-06-13 16:00:42 +00:00
|
|
|
bool ignoreDiacritics;
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
QAtomicInt isCancelled;
|
|
|
|
QSemaphore hasExited;
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
MdxArticleRequest( wstring const & word_,
|
|
|
|
vector< wstring > const & alts_,
|
2018-06-13 16:00:42 +00:00
|
|
|
MdxDictionary & dict_,
|
|
|
|
bool ignoreDiacritics_ ):
|
2013-04-23 12:07:05 +00:00
|
|
|
word( word_ ),
|
|
|
|
alts( alts_ ),
|
2018-06-13 16:00:42 +00:00
|
|
|
dict( dict_ ),
|
|
|
|
ignoreDiacritics( ignoreDiacritics_ )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
QThreadPool::globalInstance()->start( new MdxArticleRequestRunnable( *this, hasExited ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
void run();
|
|
|
|
|
|
|
|
virtual void cancel()
|
|
|
|
{
|
|
|
|
isCancelled.ref();
|
|
|
|
}
|
|
|
|
|
|
|
|
~MdxArticleRequest()
|
|
|
|
{
|
|
|
|
isCancelled.ref();
|
|
|
|
hasExited.acquire();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
void MdxArticleRequestRunnable::run()
|
|
|
|
{
|
|
|
|
r.run();
|
|
|
|
}
|
|
|
|
|
|
|
|
void MdxArticleRequest::run()
|
|
|
|
{
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( Qt4x5::AtomicInt::loadAcquire( isCancelled ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ( dict.ensureInitDone().size() )
|
|
|
|
{
|
|
|
|
setErrorString( QString::fromUtf8( dict.ensureInitDone().c_str() ) );
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-06-13 16:00:42 +00:00
|
|
|
vector< WordArticleLink > chain = dict.findArticles( word, ignoreDiacritics );
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
for ( unsigned x = 0; x < alts.size(); ++x )
|
|
|
|
{
|
|
|
|
/// Make an additional query for each alt
|
2018-06-13 16:00:42 +00:00
|
|
|
vector< WordArticleLink > altChain = dict.findArticles( alts[ x ], ignoreDiacritics );
|
2013-04-23 12:07:05 +00:00
|
|
|
chain.insert( chain.end(), altChain.begin(), altChain.end() );
|
|
|
|
}
|
|
|
|
|
|
|
|
// Some synonims make it that the articles appear several times. We combat this
|
|
|
|
// by only allowing them to appear once.
|
|
|
|
set< uint32_t > articlesIncluded;
|
|
|
|
// Sometimes the articles are physically duplicated. We store hashes of
|
|
|
|
// the bodies to account for this.
|
|
|
|
set< QByteArray > articleBodiesIncluded;
|
|
|
|
string articleText;
|
|
|
|
|
|
|
|
for ( unsigned x = 0; x < chain.size(); ++x )
|
|
|
|
{
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( Qt4x5::AtomicInt::loadAcquire( isCancelled ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ( articlesIncluded.find( chain[ x ].articleOffset ) != articlesIncluded.end() )
|
|
|
|
continue; // We already have this article in the body.
|
|
|
|
|
|
|
|
// Grab that article
|
|
|
|
string articleBody;
|
2013-05-22 03:15:47 +00:00
|
|
|
bool hasError = false;
|
|
|
|
QString errorMessage;
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
try
|
|
|
|
{
|
|
|
|
dict.loadArticle( chain[ x ].articleOffset, articleBody );
|
|
|
|
}
|
|
|
|
catch ( exCorruptDictionary & )
|
|
|
|
{
|
2013-05-22 11:56:28 +00:00
|
|
|
errorMessage = tr( "Dictionary file was tampered or corrupted" );
|
2013-05-22 03:15:47 +00:00
|
|
|
hasError = true;
|
|
|
|
}
|
|
|
|
catch ( std::exception & e )
|
|
|
|
{
|
|
|
|
errorMessage = e.what();
|
|
|
|
hasError = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ( hasError )
|
|
|
|
{
|
|
|
|
setErrorString( tr( "Failed loading article from %1, reason: %2" )
|
|
|
|
.arg( QString::fromUtf8( dict.getDictionaryFilenames()[ 0 ].c_str() ) )
|
|
|
|
.arg( errorMessage ) );
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
if ( articlesIncluded.find( chain[ x ].articleOffset ) != articlesIncluded.end() )
|
|
|
|
continue; // We already have this article in the body.
|
|
|
|
|
|
|
|
QCryptographicHash hash( QCryptographicHash::Md5 );
|
|
|
|
hash.addData( articleBody.data(), articleBody.size() );
|
|
|
|
if ( !articleBodiesIncluded.insert( hash.result() ).second )
|
|
|
|
continue; // Already had this body
|
|
|
|
|
|
|
|
// Handle internal redirects
|
|
|
|
if ( strncmp( articleBody.c_str(), "@@@LINK=", 8 ) == 0 )
|
|
|
|
{
|
|
|
|
wstring target = Utf8::decode( articleBody.c_str() + 8 );
|
|
|
|
target = Folding::trimWhitespace( target );
|
|
|
|
// Make an additional query for this redirection
|
|
|
|
vector< WordArticleLink > altChain = dict.findArticles( target );
|
|
|
|
chain.insert( chain.end(), altChain.begin(), altChain.end() );
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2013-04-25 20:41:10 +00:00
|
|
|
// See Issue #271: A mechanism to clean-up invalid HTML cards.
|
|
|
|
string cleaner = "</font>""</font>""</font>""</font>""</font>""</font>"
|
|
|
|
"</font>""</font>""</font>""</font>""</font>""</font>"
|
|
|
|
"</b></b></b></b></b></b></b></b>"
|
2013-04-27 13:10:43 +00:00
|
|
|
"</i></i></i></i></i></i></i></i>"
|
|
|
|
"</a></a></a></a></a></a></a></a>";
|
2013-04-25 20:41:10 +00:00
|
|
|
articleText += "<div class=\"mdict\">" + articleBody + cleaner + "</div>\n";
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
2019-11-19 16:20:44 +00:00
|
|
|
if ( !articleText.empty() )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
Mutex::Lock _( dataMutex );
|
|
|
|
data.insert( data.end(), articleText.begin(), articleText.end() );
|
2019-11-19 16:20:44 +00:00
|
|
|
hasAnyData = true;
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
finish();
|
|
|
|
}
|
|
|
|
|
|
|
|
sptr<Dictionary::DataRequest> MdxDictionary::getArticle( const wstring & word, const vector<wstring> & alts,
|
2018-06-13 16:00:42 +00:00
|
|
|
const wstring &, bool ignoreDiacritics ) THROW_SPEC( std::exception )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2018-06-13 16:00:42 +00:00
|
|
|
return new MdxArticleRequest( word, alts, *this, ignoreDiacritics );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// MdxDictionary::getResource
|
|
|
|
|
|
|
|
class MddResourceRequest;
|
|
|
|
|
|
|
|
class MddResourceRequestRunnable: public QRunnable
|
|
|
|
{
|
|
|
|
MddResourceRequest & r;
|
|
|
|
QSemaphore & hasExited;
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
MddResourceRequestRunnable( MddResourceRequest & r_,
|
|
|
|
QSemaphore & hasExited_ ): r( r_ ),
|
|
|
|
hasExited( hasExited_ )
|
|
|
|
{}
|
|
|
|
|
|
|
|
~MddResourceRequestRunnable()
|
|
|
|
{
|
|
|
|
hasExited.release();
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual void run();
|
|
|
|
};
|
|
|
|
|
|
|
|
class MddResourceRequest: public Dictionary::DataRequest
|
|
|
|
{
|
|
|
|
friend class MddResourceRequestRunnable;
|
|
|
|
|
|
|
|
MdxDictionary & dict;
|
|
|
|
wstring resourceName;
|
|
|
|
QAtomicInt isCancelled;
|
|
|
|
QSemaphore hasExited;
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
MddResourceRequest( MdxDictionary & dict_,
|
|
|
|
string const & resourceName_ ):
|
|
|
|
dict( dict_ ),
|
|
|
|
resourceName( Utf8::decode( resourceName_ ) )
|
|
|
|
{
|
|
|
|
QThreadPool::globalInstance()->start( new MddResourceRequestRunnable( *this, hasExited ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
void run(); // Run from another thread by MddResourceRequestRunnable
|
|
|
|
|
|
|
|
virtual void cancel()
|
|
|
|
{
|
|
|
|
isCancelled.ref();
|
|
|
|
}
|
|
|
|
|
|
|
|
~MddResourceRequest()
|
|
|
|
{
|
|
|
|
isCancelled.ref();
|
|
|
|
hasExited.acquire();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
void MddResourceRequestRunnable::run()
|
|
|
|
{
|
|
|
|
r.run();
|
|
|
|
}
|
|
|
|
|
|
|
|
void MddResourceRequest::run()
|
|
|
|
{
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( Qt4x5::AtomicInt::loadAcquire( isCancelled ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ( dict.ensureInitDone().size() )
|
|
|
|
{
|
|
|
|
setErrorString( QString::fromUtf8( dict.ensureInitDone().c_str() ) );
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// In order to prevent recursive internal redirection...
|
|
|
|
set< QByteArray > resourceIncluded;
|
|
|
|
|
|
|
|
for ( ;; )
|
|
|
|
{
|
|
|
|
// Some runnables linger enough that they are cancelled before they start
|
2013-05-30 13:24:21 +00:00
|
|
|
if ( Qt4x5::AtomicInt::loadAcquire( isCancelled ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
finish();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
string u8ResourceName = Utf8::encode( resourceName );
|
|
|
|
QCryptographicHash hash( QCryptographicHash::Md5 );
|
|
|
|
hash.addData( u8ResourceName.data(), u8ResourceName.size() );
|
|
|
|
if ( !resourceIncluded.insert( hash.result() ).second )
|
|
|
|
continue;
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
// Convert to the Windows separator
|
|
|
|
std::replace( resourceName.begin(), resourceName.end(), '/', '\\' );
|
|
|
|
if ( resourceName[ 0 ] != '\\' )
|
|
|
|
{
|
|
|
|
resourceName.insert( 0, 1, '\\' );
|
|
|
|
}
|
|
|
|
|
|
|
|
Mutex::Lock _( dataMutex );
|
|
|
|
data.clear();
|
2013-05-08 13:39:47 +00:00
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
// local file takes precedence
|
|
|
|
string fn = FsEncoding::dirname( dict.getDictionaryFilenames()[ 0 ] ) +
|
|
|
|
FsEncoding::separator() + u8ResourceName;
|
|
|
|
File::loadFromFile( fn, data );
|
|
|
|
}
|
|
|
|
catch ( File::exCantOpen & )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
for ( vector< sptr< IndexedMdd > >::const_iterator i = dict.mddResources.begin();
|
2022-02-16 13:47:03 +00:00
|
|
|
i != dict.mddResources.end(); ++i )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
sptr< IndexedMdd > mddResource = *i;
|
|
|
|
if ( mddResource->loadFile( resourceName, data ) )
|
|
|
|
break;
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
2013-05-08 13:39:47 +00:00
|
|
|
}
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
// Check if this file has a redirection
|
|
|
|
// Always encoded in UTF16-LE
|
|
|
|
// L"@@@LINK="
|
|
|
|
static const char pattern[16] =
|
|
|
|
{
|
|
|
|
'@', '\0', '@', '\0', '@', '\0', 'L', '\0', 'I', '\0', 'N', '\0', 'K', '\0', '=', '\0'
|
|
|
|
};
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
if ( data.size() > sizeof( pattern ) )
|
|
|
|
{
|
|
|
|
if ( memcmp( &data.front(), pattern, sizeof( pattern ) ) == 0 )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
data.push_back( '\0' );
|
|
|
|
data.push_back( '\0' );
|
|
|
|
QString target = MdictParser::toUtf16( "UTF-16LE", &data.front() + sizeof( pattern ),
|
|
|
|
data.size() - sizeof( pattern ) );
|
|
|
|
resourceName = gd::toWString( target.trimmed() );
|
|
|
|
continue;
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
if ( data.size() > 0 )
|
2013-05-27 06:12:46 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
hasAnyData = true;
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-06-04 16:17:55 +00:00
|
|
|
if ( Filetype::isNameOfCSS( u8ResourceName ) )
|
2013-05-23 14:01:50 +00:00
|
|
|
{
|
|
|
|
QString css = QString::fromUtf8( data.data(), data.size() );
|
2016-04-01 22:50:19 +00:00
|
|
|
|
2018-02-27 16:42:21 +00:00
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
QRegularExpression links( "url\\(\\s*(['\"]?)([^'\"]*)(['\"]?)\\s*\\)",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
#else
|
2016-04-01 22:50:19 +00:00
|
|
|
QRegExp links( "url\\(\\s*(['\"]?)([^'\"]*)(['\"]?)\\s*\\)", Qt::CaseInsensitive, QRegExp::RegExp );
|
2018-02-27 16:42:21 +00:00
|
|
|
#endif
|
2016-04-01 22:50:19 +00:00
|
|
|
QString id = QString::fromUtf8( dict.getId().c_str() );
|
|
|
|
int pos = 0;
|
2018-02-27 16:42:21 +00:00
|
|
|
|
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
QString newCSS;
|
|
|
|
QRegularExpressionMatchIterator it = links.globalMatch( css );
|
|
|
|
while ( it.hasNext() )
|
|
|
|
{
|
|
|
|
QRegularExpressionMatch match = it.next();
|
|
|
|
newCSS += css.midRef( pos, match.capturedStart() - pos );
|
|
|
|
pos = match.capturedEnd();
|
|
|
|
QString url = match.captured( 2 );
|
|
|
|
#else
|
2016-04-01 22:50:19 +00:00
|
|
|
for( ; ; )
|
|
|
|
{
|
|
|
|
pos = links.indexIn( css, pos );
|
|
|
|
if( pos < 0 )
|
|
|
|
break;
|
|
|
|
QString url = links.cap( 2 );
|
2018-02-27 16:42:21 +00:00
|
|
|
#endif
|
2016-04-01 22:50:19 +00:00
|
|
|
|
2016-04-14 15:51:00 +00:00
|
|
|
if( url.indexOf( ":/" ) >= 0 || url.indexOf( "data:" ) >= 0)
|
2016-04-01 22:50:19 +00:00
|
|
|
{
|
2016-04-14 15:51:00 +00:00
|
|
|
// External link or base64-encoded data
|
2018-02-27 16:42:21 +00:00
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
newCSS += match.captured();
|
|
|
|
#else
|
2016-04-01 22:50:19 +00:00
|
|
|
pos += links.cap().size();
|
2018-02-27 16:42:21 +00:00
|
|
|
#endif
|
2016-04-01 22:50:19 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2018-02-27 16:42:21 +00:00
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
QString newUrl = QString( "url(" ) + match.captured( 1 ) + "bres://"
|
|
|
|
+ id + "/" + url + match.captured( 3 ) + ")";
|
|
|
|
newCSS += newUrl;
|
|
|
|
#else
|
2016-04-01 22:50:19 +00:00
|
|
|
QString newUrl = QString( "url(" ) + links.cap( 1 ) + "bres://"
|
|
|
|
+ id + "/" + url + links.cap( 3 ) + ")";
|
|
|
|
css.replace( pos, links.cap().size(), newUrl );
|
|
|
|
pos += newUrl.size();
|
2018-02-27 16:42:21 +00:00
|
|
|
#endif
|
2016-04-01 22:50:19 +00:00
|
|
|
}
|
2018-02-27 16:42:21 +00:00
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
if( pos )
|
|
|
|
{
|
|
|
|
newCSS += css.midRef( pos );
|
|
|
|
css = newCSS;
|
|
|
|
newCSS.clear();
|
|
|
|
}
|
|
|
|
#endif
|
2013-05-28 07:00:36 +00:00
|
|
|
dict.isolateCSS( css, ".mdict" );
|
2013-05-23 14:01:50 +00:00
|
|
|
QByteArray bytes = css.toUtf8();
|
|
|
|
data.resize( bytes.size() );
|
|
|
|
memcpy( &data.front(), bytes.constData(), bytes.size() );
|
|
|
|
}
|
2013-05-22 17:42:41 +00:00
|
|
|
}
|
2013-04-23 12:07:05 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
finish();
|
|
|
|
}
|
|
|
|
|
2018-05-21 15:32:04 +00:00
|
|
|
sptr<Dictionary::DataRequest> MdxDictionary::getResource( const string & name ) THROW_SPEC( std::exception )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
return new MddResourceRequest( *this, name );
|
|
|
|
}
|
|
|
|
|
|
|
|
const QString & MdxDictionary::getDescription()
|
|
|
|
{
|
|
|
|
if ( !dictionaryDescription.isEmpty() )
|
|
|
|
return dictionaryDescription;
|
|
|
|
|
|
|
|
if ( idxHeader.descriptionSize == 0 )
|
|
|
|
{
|
|
|
|
dictionaryDescription = "NONE";
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2014-04-20 20:02:41 +00:00
|
|
|
Mutex::Lock _( idxMutex );
|
2013-04-23 12:07:05 +00:00
|
|
|
vector< char > chunk;
|
|
|
|
char * dictDescription = chunks.getBlock( idxHeader.descriptionAddress, chunk );
|
|
|
|
string str( dictDescription );
|
|
|
|
dictionaryDescription = QString::fromUtf8( str.c_str(), str.size() );
|
|
|
|
}
|
|
|
|
|
|
|
|
return dictionaryDescription;
|
|
|
|
}
|
|
|
|
|
|
|
|
void MdxDictionary::loadIcon() throw()
|
|
|
|
{
|
|
|
|
if ( dictionaryIconLoaded )
|
|
|
|
return;
|
|
|
|
|
|
|
|
QString fileName =
|
|
|
|
QDir::fromNativeSeparators( FsEncoding::decode( getDictionaryFilenames()[ 0 ].c_str() ) );
|
|
|
|
|
|
|
|
// Remove the extension
|
|
|
|
fileName.chop( 3 );
|
|
|
|
|
|
|
|
if ( !loadIconFromFile( fileName ) )
|
|
|
|
{
|
|
|
|
// Use default icons
|
|
|
|
dictionaryIcon = dictionaryNativeIcon = QIcon( ":/icons/mdict.png" );
|
|
|
|
}
|
|
|
|
|
|
|
|
dictionaryIconLoaded = true;
|
|
|
|
}
|
|
|
|
|
2014-04-29 13:55:48 +00:00
|
|
|
void MdxDictionary::loadArticle( uint32_t offset, string & articleText, bool noFilter )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
vector< char > chunk;
|
|
|
|
Mutex::Lock _( idxMutex );
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
// Load record info from index
|
|
|
|
MdictParser::RecordInfo recordInfo;
|
|
|
|
char * pRecordInfo = chunks.getBlock( offset, chunk );
|
|
|
|
memcpy( &recordInfo, pRecordInfo, sizeof( recordInfo ) );
|
2013-04-24 11:35:03 +00:00
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
// Make a sub unique id for this article
|
2013-04-25 13:52:23 +00:00
|
|
|
QString articleId;
|
2013-04-28 08:26:04 +00:00
|
|
|
articleId.setNum( ( quint64 )pRecordInfo, 16 );
|
2013-04-24 11:35:03 +00:00
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
ScopedMemMap compressed( dictFile, recordInfo.compressedBlockPos, recordInfo.compressedBlockSize );
|
|
|
|
if ( !compressed.startAddress() )
|
|
|
|
throw exCorruptDictionary();
|
2013-04-28 08:26:04 +00:00
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
QByteArray decompressed;
|
|
|
|
if ( !MdictParser::parseCompressedBlock( recordInfo.compressedBlockSize, ( char * )compressed.startAddress(),
|
|
|
|
recordInfo.decompressedBlockSize, decompressed ) )
|
|
|
|
throw exCorruptDictionary();
|
2013-04-28 08:26:04 +00:00
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
QString article = MdictParser::toUtf16( encoding.c_str(),
|
|
|
|
decompressed.constData() + recordInfo.recordOffset,
|
|
|
|
recordInfo.recordSize );
|
2013-04-28 08:26:04 +00:00
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
article = MdictParser::substituteStylesheet( article, styleSheets );
|
2018-02-27 16:42:21 +00:00
|
|
|
|
2014-04-29 13:55:48 +00:00
|
|
|
if( !noFilter )
|
|
|
|
article = filterResource( articleId, article );
|
2016-05-11 15:56:37 +00:00
|
|
|
|
|
|
|
// Check for unclosed <span> and <div>
|
|
|
|
|
|
|
|
int openTags = article.count( QRegExp( "<\\s*span\\b", Qt::CaseInsensitive ) );
|
|
|
|
int closedTags = article.count( QRegExp( "<\\s*/span\\s*>", Qt::CaseInsensitive ) );
|
|
|
|
while( openTags > closedTags )
|
|
|
|
{
|
|
|
|
article += "</span>";
|
|
|
|
closedTags += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
openTags = article.count( QRegExp( "<\\s*div\\b", Qt::CaseInsensitive ) );
|
|
|
|
closedTags = article.count( QRegExp( "<\\s*/div\\s*>", Qt::CaseInsensitive ) );
|
|
|
|
while( openTags > closedTags )
|
|
|
|
{
|
|
|
|
article += "</div>";
|
|
|
|
closedTags += 1;
|
|
|
|
}
|
|
|
|
|
2013-05-22 03:15:47 +00:00
|
|
|
articleText = string( article.toUtf8().constData() );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
2018-02-27 16:42:21 +00:00
|
|
|
#if QT_VERSION >= QT_VERSION_CHECK( 5, 0, 0 )
|
|
|
|
QString & MdxDictionary::filterResource( QString const & articleId, QString & article )
|
|
|
|
{
|
|
|
|
QString id = QString::fromStdString( getId() );
|
|
|
|
QString uniquePrefix = QString::fromLatin1( "g" ) + id + "_" + articleId + "_";
|
|
|
|
|
2021-11-12 18:26:46 +00:00
|
|
|
QRegularExpression allLinksRe( "(?:<\\s*(a(?:rea)?|img|link|script|source)(?:\\s+[^>]+|\\s*)>)",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-07-19 15:18:50 +00:00
|
|
|
QRegularExpression wordCrossLink( "([\\s\"']href\\s*=)\\s*([\"'])entry://([^>#]*?)((?:#[^>]*?)?)\\2",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression anchorIdRe( "([\\s\"'](?:name|id)\\s*=)\\s*([\"'])\\s*(?=\\S)",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression anchorIdRe2( "([\\s\"'](?:name|id)\\s*=)\\s*(?=[^\"'])([^\\s\">]+)",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression anchorLinkRe( "([\\s\"']href\\s*=\\s*[\"'])entry://#",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression audioRe( "([\\s\"']href\\s*=)\\s*([\"'])sound://([^\">]+)\\2",
|
|
|
|
QRegularExpression::CaseInsensitiveOption
|
|
|
|
| QRegularExpression::InvertedGreedinessOption );
|
|
|
|
QRegularExpression stylesRe( "([\\s\"']href\\s*=)\\s*([\"'])(?!\\s*\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\">]+)\\2",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression stylesRe2( "([\\s\"']href\\s*=)\\s*(?![\\s\"']|\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\\s\">]+)",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression inlineScriptRe( "<\\s*script(?:(?=\\s)(?:(?![\\s\"']src\\s*=)[^>])+|\\s*)>",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression closeScriptTagRe( "<\\s*/script\\s*>",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression srcRe( "([\\s\"']src\\s*=)\\s*([\"'])(?!\\s*\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\">]+)\\2",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpression srcRe2( "([\\s\"']src\\s*=)\\s*(?![\\s\"']|\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\\s\">]+)",
|
2018-02-28 14:15:27 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2018-02-27 16:42:21 +00:00
|
|
|
|
|
|
|
QString articleNewText;
|
|
|
|
int linkPos = 0;
|
|
|
|
QRegularExpressionMatchIterator it = allLinksRe.globalMatch( article );
|
|
|
|
while( it.hasNext() )
|
|
|
|
{
|
|
|
|
QRegularExpressionMatch allLinksMatch = it.next();
|
|
|
|
|
|
|
|
if( allLinksMatch.capturedEnd() < linkPos )
|
|
|
|
continue;
|
|
|
|
|
|
|
|
articleNewText += article.midRef( linkPos, allLinksMatch.capturedStart() - linkPos );
|
|
|
|
linkPos = allLinksMatch.capturedEnd();
|
|
|
|
|
|
|
|
QString linkTxt = allLinksMatch.captured();
|
|
|
|
QString linkType = allLinksMatch.captured( 1 ).toLower();
|
|
|
|
QString newLink;
|
|
|
|
|
|
|
|
if( !linkType.isEmpty() && linkType.at( 0 ) == 'a' )
|
|
|
|
{
|
|
|
|
QRegularExpressionMatch match = anchorIdRe.match( linkTxt );
|
|
|
|
if( match.hasMatch() )
|
|
|
|
{
|
|
|
|
QString newText = match.captured( 1 ) + match.captured( 2 ) + uniquePrefix;
|
|
|
|
newLink = linkTxt.replace( match.capturedStart(), match.capturedLength(), newText );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
newLink = linkTxt.replace( anchorIdRe2, "\\1\"" + uniquePrefix + "\\2\"" );
|
|
|
|
|
|
|
|
newLink = newLink.replace( anchorLinkRe, "\\1#" + uniquePrefix );
|
|
|
|
|
|
|
|
match = audioRe.match( newLink );
|
|
|
|
if( match.hasMatch() )
|
|
|
|
{
|
|
|
|
// sounds and audio link script
|
|
|
|
QString newTxt = match.captured( 1 ) + match.captured( 2 )
|
|
|
|
+ "gdau://" + id + "/"
|
|
|
|
+ match.captured( 3 ) + match.captured( 2 );
|
|
|
|
newLink = QString::fromUtf8( addAudioLink( "\"gdau://" + getId() + "/" + match.captured( 3 ).toUtf8().data() + "\"", getId() ).c_str() )
|
|
|
|
+ newLink.replace( match.capturedStart(), match.capturedLength(), newTxt );
|
|
|
|
}
|
|
|
|
|
|
|
|
match = wordCrossLink.match( newLink );
|
|
|
|
if( match.hasMatch() )
|
|
|
|
{
|
|
|
|
QString newTxt = match.captured( 1 ) + match.captured( 2 )
|
|
|
|
+ "gdlookup://localhost/"
|
|
|
|
+ match.captured( 3 );
|
|
|
|
|
2018-07-04 15:10:22 +00:00
|
|
|
if( match.lastCapturedIndex() >= 4 && !match.captured( 4 ).isEmpty() )
|
2018-02-27 16:42:21 +00:00
|
|
|
newTxt += QString( "?gdanchor=" ) + uniquePrefix + match.captured( 4 ).mid( 1 );
|
|
|
|
|
|
|
|
newTxt += match.captured( 2 );
|
|
|
|
newLink.replace( match.capturedStart(), match.capturedLength(), newTxt );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
if( linkType.compare( "link" ) == 0 )
|
|
|
|
{
|
|
|
|
// stylesheets
|
|
|
|
QRegularExpressionMatch match = stylesRe.match( linkTxt );
|
|
|
|
if( match.hasMatch() )
|
|
|
|
{
|
|
|
|
QString newText = match.captured( 1 ) + match.captured( 2 )
|
|
|
|
+ "bres://" + id + "/"
|
|
|
|
+ match.captured( 3 ) + match.captured( 2 );
|
|
|
|
newLink = linkTxt.replace( match.capturedStart(), match.capturedLength(), newText );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
newLink = linkTxt.replace( stylesRe2,
|
|
|
|
"\\1\"bres://" + id + "/\\2\"" );
|
|
|
|
}
|
|
|
|
else
|
2021-11-12 18:26:46 +00:00
|
|
|
if( linkType.compare( "script" ) == 0 || linkType.compare( "img" ) == 0
|
|
|
|
|| linkType.compare( "source" ) == 0 )
|
2018-02-27 16:42:21 +00:00
|
|
|
{
|
|
|
|
// javascripts and images
|
2018-04-08 14:48:46 +00:00
|
|
|
QRegularExpressionMatch match = inlineScriptRe.match( linkTxt );
|
2021-11-12 18:26:46 +00:00
|
|
|
if( linkType.at( 1 ) == 'c' // "script" tag
|
2018-04-08 14:48:46 +00:00
|
|
|
&& match.hasMatch() && match.capturedLength() == linkTxt.length() )
|
2018-02-27 16:42:21 +00:00
|
|
|
{
|
|
|
|
// skip inline scripts
|
|
|
|
articleNewText += linkTxt;
|
2018-04-08 14:48:46 +00:00
|
|
|
match = closeScriptTagRe.match( article, linkPos );
|
|
|
|
if( match.hasMatch() )
|
2018-02-27 16:42:21 +00:00
|
|
|
{
|
2018-04-08 14:48:46 +00:00
|
|
|
articleNewText += article.midRef( linkPos, match.capturedEnd() - linkPos );
|
|
|
|
linkPos = match.capturedEnd();
|
2018-02-27 16:42:21 +00:00
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
match = srcRe.match( linkTxt );
|
|
|
|
if( match.hasMatch() )
|
|
|
|
{
|
2021-11-12 18:26:46 +00:00
|
|
|
QString newText;
|
|
|
|
if( linkType.at( 1 ) == 'o' ) // "source" tag
|
|
|
|
{
|
|
|
|
QString filename = match.captured( 3 );
|
|
|
|
QString newName = getCachedFileName( filename );
|
|
|
|
newName.replace( '\\', '/' );
|
|
|
|
newText = match.captured( 1 ) + match.captured( 2 )
|
|
|
|
+ "file:///" + newName + match.captured( 2 );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
newText = match.captured( 1 ) + match.captured( 2 )
|
|
|
|
+ "bres://" + id + "/"
|
|
|
|
+ match.captured( 3 ) + match.captured( 2 );
|
|
|
|
}
|
2018-02-27 16:42:21 +00:00
|
|
|
newLink = linkTxt.replace( match.capturedStart(), match.capturedLength(), newText );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
newLink = linkTxt.replace( srcRe2,
|
|
|
|
"\\1\"bres://" + id + "/\\2\"" );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if( !newLink.isEmpty() )
|
|
|
|
{
|
|
|
|
articleNewText += newLink;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
articleNewText += allLinksMatch.captured();
|
|
|
|
}
|
|
|
|
if( linkPos )
|
|
|
|
{
|
|
|
|
articleNewText += article.midRef( linkPos );
|
|
|
|
article = articleNewText;
|
|
|
|
}
|
|
|
|
|
|
|
|
return article;
|
|
|
|
}
|
|
|
|
#else
|
2013-05-22 03:15:47 +00:00
|
|
|
QString & MdxDictionary::filterResource( QString const & articleId, QString & article )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-04-28 08:49:15 +00:00
|
|
|
QString id = QString::fromStdString( getId() );
|
2016-05-11 14:41:13 +00:00
|
|
|
QString uniquePrefix = QString::fromLatin1( "g" ) + id + "_" + articleId + "_";
|
2013-04-24 11:35:03 +00:00
|
|
|
|
2021-11-12 18:26:46 +00:00
|
|
|
QRegExp allLinksRe( "(?:<\\s*(a(?:rea)?|img|link|script|source)(?:\\s+[^>]+|\\s*)>)", Qt::CaseInsensitive );
|
2016-11-13 13:33:22 +00:00
|
|
|
QRegExp wordCrossLink( "([\\s\"']href\\s*=)\\s*([\"'])entry://([^>#]*)((?:#[^>]*)?)\\2", Qt::CaseInsensitive );
|
|
|
|
wordCrossLink.setMinimal( true );
|
|
|
|
|
|
|
|
QRegExp anchorIdRe( "([\\s\"'](?:name|id)\\s*=)\\s*([\"'])\\s*(?=\\S)", Qt::CaseInsensitive );
|
|
|
|
QRegExp anchorIdRe2( "([\\s\"'](?:name|id)\\s*=)\\s*(?=[^\"'])([^\\s\">]+)", Qt::CaseInsensitive );
|
|
|
|
QRegExp anchorLinkRe( "([\\s\"']href\\s*=\\s*[\"'])entry://#", Qt::CaseInsensitive );
|
|
|
|
QRegExp audioRe( "([\\s\"']href\\s*=)\\s*([\"'])sound://([^\">]+)\\2", Qt::CaseInsensitive );
|
|
|
|
audioRe.setMinimal( true );
|
|
|
|
|
|
|
|
QRegExp stylesRe( "([\\s\"']href\\s*=)\\s*([\"'])(?!\\s*\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\">]+)\\2",
|
2016-11-07 15:40:53 +00:00
|
|
|
Qt::CaseInsensitive, QRegExp::RegExp2 );
|
2016-11-13 13:33:22 +00:00
|
|
|
stylesRe.setMinimal( true );
|
|
|
|
QRegExp stylesRe2( "([\\s\"']href\\s*=)\\s*(?![\\s\"']|\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\\s\">]+)",
|
2016-11-07 15:40:53 +00:00
|
|
|
Qt::CaseInsensitive, QRegExp::RegExp2 );
|
2016-11-13 13:33:22 +00:00
|
|
|
QRegExp inlineScriptRe( "<\\s*script(?:(?=\\s)(?:(?![\\s\"']src\\s*=)[^>])+|\\s*)>", Qt::CaseInsensitive, QRegExp::RegExp2 );
|
2016-11-07 15:40:53 +00:00
|
|
|
QRegExp closeScriptTagRe( "<\\s*/script\\s*>", Qt::CaseInsensitive, QRegExp::RegExp2 );
|
2016-11-13 13:33:22 +00:00
|
|
|
QRegExp srcRe( "([\\s\"']src\\s*=)\\s*([\"'])(?!\\s*\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\">]+)\\2",
|
2016-11-07 15:40:53 +00:00
|
|
|
Qt::CaseInsensitive, QRegExp::RegExp2 );
|
2016-11-13 13:33:22 +00:00
|
|
|
srcRe.setMinimal( true );
|
|
|
|
QRegExp srcRe2( "([\\s\"']src\\s*=)\\s*(?![\\s\"']|\\b(?:(?:bres|https?|ftp)://|(?:data|javascript):))(?:file://)?[\\x00-\\x1f\\x7f]*\\.*/?([^\\s\">]+)",
|
2016-11-07 15:40:53 +00:00
|
|
|
Qt::CaseInsensitive, QRegExp::RegExp2 );
|
|
|
|
|
|
|
|
int linkPos = 0;
|
|
|
|
while( linkPos >= 0 )
|
2015-10-28 19:56:58 +00:00
|
|
|
{
|
2016-11-13 13:33:22 +00:00
|
|
|
linkPos = allLinksRe.indexIn( article, linkPos );
|
2016-11-07 15:40:53 +00:00
|
|
|
if( linkPos < 0 )
|
2015-10-28 19:56:58 +00:00
|
|
|
break;
|
|
|
|
|
2016-11-07 15:40:53 +00:00
|
|
|
QString linkTxt = allLinksRe.cap( 0 );
|
2016-11-13 13:33:22 +00:00
|
|
|
QString linkType = allLinksRe.cap( 1 ).toLower();
|
2016-11-07 15:40:53 +00:00
|
|
|
QString newLink;
|
2015-10-30 19:21:35 +00:00
|
|
|
|
2016-11-13 13:33:22 +00:00
|
|
|
if( !linkType.isEmpty() && linkType.at( 0 ) == 'a' )
|
2016-11-07 15:40:53 +00:00
|
|
|
{
|
2016-11-13 13:33:22 +00:00
|
|
|
int pos = anchorIdRe.indexIn( linkTxt );
|
|
|
|
if( pos >= 0 )
|
|
|
|
{
|
|
|
|
QString newText = anchorIdRe.cap( 1 ) + anchorIdRe.cap( 2 ) + uniquePrefix;
|
|
|
|
newLink = linkTxt.replace( pos, anchorIdRe.cap().length(), newText );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
newLink = linkTxt.replace( anchorIdRe2, "\\1\"" + uniquePrefix + "\\2\"" );
|
|
|
|
|
|
|
|
newLink = newLink.replace( anchorLinkRe, "\\1#" + uniquePrefix );
|
|
|
|
|
|
|
|
pos = audioRe.indexIn( newLink );
|
|
|
|
if( pos >= 0 )
|
|
|
|
{
|
|
|
|
// sounds and audio link script
|
|
|
|
QString newTxt = audioRe.cap( 1 ) + audioRe.cap( 2 )
|
|
|
|
+ "gdau://" + id + "/"
|
|
|
|
+ audioRe.cap( 3 ) + audioRe.cap( 2 );
|
|
|
|
newLink = QString::fromUtf8( addAudioLink( "\"gdau://" + getId() + "/" + audioRe.cap( 3 ).toUtf8().data() + "\"", getId() ).c_str() )
|
|
|
|
+ newLink.replace( pos, audioRe.cap().length(), newTxt );
|
|
|
|
}
|
|
|
|
|
|
|
|
pos = wordCrossLink.indexIn( newLink );
|
2016-11-07 15:40:53 +00:00
|
|
|
if( pos >= 0 )
|
|
|
|
{
|
2016-11-13 13:33:22 +00:00
|
|
|
QString newTxt = wordCrossLink.cap( 1 ) + wordCrossLink.cap( 2 )
|
2016-11-07 15:40:53 +00:00
|
|
|
+ "gdlookup://localhost/"
|
2016-11-13 13:33:22 +00:00
|
|
|
+ wordCrossLink.cap( 3 );
|
2015-10-28 19:56:58 +00:00
|
|
|
|
2016-11-13 13:33:22 +00:00
|
|
|
if( !wordCrossLink.cap( 4 ).isEmpty() )
|
|
|
|
newTxt += QString( "?gdanchor=" ) + uniquePrefix + wordCrossLink.cap( 4 ).mid( 1 );
|
2016-03-09 20:06:53 +00:00
|
|
|
|
2016-11-13 13:33:22 +00:00
|
|
|
newTxt += wordCrossLink.cap( 2 );
|
|
|
|
newLink.replace( pos, wordCrossLink.cap( 0 ).length(), newTxt );
|
2016-11-07 15:40:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
2016-11-13 13:33:22 +00:00
|
|
|
if( linkType.compare( "link" ) == 0 )
|
2016-03-09 20:06:53 +00:00
|
|
|
{
|
2016-11-07 15:40:53 +00:00
|
|
|
// stylesheets
|
2016-11-13 13:33:22 +00:00
|
|
|
int pos = stylesRe.indexIn( linkTxt );
|
|
|
|
if( pos >= 0 )
|
|
|
|
{
|
|
|
|
QString newText = stylesRe.cap( 1 ) + stylesRe.cap( 2 )
|
|
|
|
+ "bres://" + id + "/"
|
|
|
|
+ stylesRe.cap( 3 ) + stylesRe.cap( 2 );
|
|
|
|
newLink = linkTxt.replace( pos, stylesRe.cap().length(), newText );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
newLink = linkTxt.replace( stylesRe2,
|
|
|
|
"\\1\"bres://" + id + "/\\2\"" );
|
2016-03-09 20:06:53 +00:00
|
|
|
}
|
2016-11-07 15:40:53 +00:00
|
|
|
else
|
2021-11-12 18:26:46 +00:00
|
|
|
if( linkType.compare( "script" ) == 0 || linkType.compare( "img" ) == 0
|
|
|
|
|| linkType.compare( "source" ) == 0 )
|
2016-11-07 15:40:53 +00:00
|
|
|
{
|
2016-11-13 13:33:22 +00:00
|
|
|
// javascripts and images
|
|
|
|
if( linkType.at( 0 ) == 's' && inlineScriptRe.exactMatch( linkTxt ) )
|
2016-11-07 15:40:53 +00:00
|
|
|
{
|
|
|
|
// skip inline scripts
|
2016-11-13 13:33:22 +00:00
|
|
|
linkPos += linkTxt.length();
|
|
|
|
int pos = closeScriptTagRe.indexIn( article, linkPos );
|
|
|
|
if( pos > 0 )
|
|
|
|
linkPos = pos + closeScriptTagRe.cap().length();
|
2016-11-07 15:40:53 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2016-11-13 13:33:22 +00:00
|
|
|
int pos = srcRe.indexIn( linkTxt );
|
|
|
|
if( pos >= 0 )
|
|
|
|
{
|
2021-11-12 18:26:46 +00:00
|
|
|
QString newText;
|
|
|
|
if( linkType.at( 1 ) == 'o' ) // "source" tag
|
|
|
|
{
|
|
|
|
QString filename = srcRe.cap( 3 );
|
|
|
|
QString newName = getCachedFileName( filename );
|
|
|
|
newName.replace( '\\', '/' );
|
|
|
|
newText = srcRe.cap( 1 ) + srcRe.cap( 2 )
|
|
|
|
+ "file:///" + newName + srcRe.cap( 2 );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
newText = srcRe.cap( 1 ) + srcRe.cap( 2 )
|
|
|
|
+ "bres://" + id + "/"
|
|
|
|
+ srcRe.cap( 3 ) + srcRe.cap( 2 );
|
|
|
|
}
|
2016-11-13 13:33:22 +00:00
|
|
|
newLink = linkTxt.replace( pos, srcRe.cap().length(), newText );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
newLink = linkTxt.replace( srcRe2,
|
|
|
|
"\\1\"bres://" + id + "/\\2\"" );
|
2016-11-07 15:40:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if( !newLink.isEmpty() )
|
|
|
|
{
|
|
|
|
article.replace( linkPos, allLinksRe.cap().length(), newLink );
|
|
|
|
linkPos += newLink.length();
|
|
|
|
}
|
|
|
|
else
|
|
|
|
linkPos += allLinksRe.cap().length();
|
2016-03-09 20:06:53 +00:00
|
|
|
}
|
|
|
|
|
2015-10-28 19:56:58 +00:00
|
|
|
return article;
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
2018-02-27 16:42:21 +00:00
|
|
|
#endif
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2021-11-12 18:26:46 +00:00
|
|
|
QString MdxDictionary::getCachedFileName( QString filename )
|
|
|
|
{
|
|
|
|
QDir dir;
|
|
|
|
QFileInfo info( cacheDirName );
|
|
|
|
if( !info.exists() || !info.isDir() )
|
|
|
|
{
|
|
|
|
if( !dir.mkdir( cacheDirName ) )
|
|
|
|
{
|
|
|
|
gdWarning( "Mdx: can't create cache directory \"%s\"", cacheDirName.toUtf8().data() );
|
|
|
|
return QString();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create subfolders if needed
|
|
|
|
|
|
|
|
QString name = filename;
|
|
|
|
name.replace( '/', '\\' );
|
|
|
|
QStringList list = name.split( '\\' );
|
|
|
|
int subFolders = list.size() - 1;
|
|
|
|
if( subFolders > 0 )
|
|
|
|
{
|
|
|
|
QString dirName = cacheDirName;
|
|
|
|
for( int i = 0; i < subFolders; i++ )
|
|
|
|
{
|
|
|
|
dirName += QDir::separator() + list.at( i );
|
|
|
|
QFileInfo dirInfo( dirName );
|
|
|
|
if( !dirInfo.exists() )
|
|
|
|
{
|
|
|
|
if( !dir.mkdir( dirName ) )
|
|
|
|
{
|
|
|
|
gdWarning( "Mdx: can't create cache directory \"%s\"", dirName.toUtf8().data() );
|
|
|
|
return QString();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
QString fullName = cacheDirName + QDir::separator() + filename;
|
|
|
|
|
|
|
|
info.setFile( fullName );
|
|
|
|
if( !info.exists() )
|
|
|
|
{
|
|
|
|
QFile f( fullName );
|
|
|
|
if( f.open( QFile::WriteOnly ) )
|
|
|
|
{
|
|
|
|
gd::wstring resourceName = FsEncoding::decode( filename.toStdString() );
|
|
|
|
vector< char > data;
|
|
|
|
|
|
|
|
// In order to prevent recursive internal redirection...
|
|
|
|
set< QByteArray > resourceIncluded;
|
|
|
|
|
|
|
|
for ( ;; )
|
|
|
|
{
|
|
|
|
|
|
|
|
string u8ResourceName = Utf8::encode( resourceName );
|
|
|
|
QCryptographicHash hash( QCryptographicHash::Md5 );
|
|
|
|
hash.addData( u8ResourceName.data(), u8ResourceName.size() );
|
|
|
|
if ( !resourceIncluded.insert( hash.result() ).second )
|
|
|
|
continue;
|
|
|
|
|
|
|
|
// Convert to the Windows separator
|
|
|
|
std::replace( resourceName.begin(), resourceName.end(), '/', '\\' );
|
|
|
|
if ( resourceName[ 0 ] != '\\' )
|
|
|
|
{
|
|
|
|
resourceName.insert( 0, 1, '\\' );
|
|
|
|
}
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
// local file takes precedence
|
|
|
|
string fn = FsEncoding::dirname( getDictionaryFilenames()[ 0 ] ) +
|
|
|
|
FsEncoding::separator() + u8ResourceName;
|
|
|
|
File::loadFromFile( fn, data );
|
|
|
|
}
|
|
|
|
catch ( File::exCantOpen & )
|
|
|
|
{
|
|
|
|
for ( vector< sptr< IndexedMdd > >::const_iterator i = mddResources.begin();
|
2022-02-16 13:47:03 +00:00
|
|
|
i != mddResources.end(); ++i )
|
2021-11-12 18:26:46 +00:00
|
|
|
{
|
|
|
|
sptr< IndexedMdd > mddResource = *i;
|
|
|
|
if ( mddResource->loadFile( resourceName, data ) )
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if this file has a redirection
|
|
|
|
// Always encoded in UTF16-LE
|
|
|
|
// L"@@@LINK="
|
|
|
|
static const char pattern[16] =
|
|
|
|
{
|
|
|
|
'@', '\0', '@', '\0', '@', '\0', 'L', '\0', 'I', '\0', 'N', '\0', 'K', '\0', '=', '\0'
|
|
|
|
};
|
|
|
|
|
|
|
|
if ( data.size() > sizeof( pattern ) )
|
|
|
|
{
|
|
|
|
if ( memcmp( &data.front(), pattern, sizeof( pattern ) ) == 0 )
|
|
|
|
{
|
|
|
|
data.push_back( '\0' );
|
|
|
|
data.push_back( '\0' );
|
|
|
|
QString target = MdictParser::toUtf16( "UTF-16LE", &data.front() + sizeof( pattern ),
|
|
|
|
data.size() - sizeof( pattern ) );
|
|
|
|
resourceName = gd::toWString( target.trimmed() );
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
qint64 n = 0;
|
|
|
|
if( !data.empty() )
|
|
|
|
n = f.write( data.data(), data.size() );
|
|
|
|
|
|
|
|
f.close();
|
|
|
|
|
|
|
|
if( n < (qint64)data.size() )
|
|
|
|
{
|
|
|
|
gdWarning( "Mdx: file \"%s\" writing error: \"%s\"", fullName.toUtf8().data(),
|
|
|
|
f.errorString().toUtf8().data() );
|
|
|
|
return QString();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
gdWarning( "Mdx: file \"%s\" creating error: \"%s\"", fullName.toUtf8().data(),
|
|
|
|
f.errorString().toUtf8().data() );
|
|
|
|
return QString();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return fullName;
|
|
|
|
}
|
|
|
|
|
|
|
|
void MdxDictionary::removeDirectory( QString const & directory )
|
|
|
|
{
|
|
|
|
QDir dir( directory );
|
|
|
|
Q_FOREACH( QFileInfo info, dir.entryInfoList( QDir::NoDotAndDotDot
|
|
|
|
| QDir::AllDirs
|
|
|
|
| QDir::Files,
|
|
|
|
QDir::DirsFirst))
|
|
|
|
{
|
|
|
|
if( info.isDir() )
|
|
|
|
removeDirectory( info.absoluteFilePath() );
|
|
|
|
else
|
|
|
|
QFile::remove( info.absoluteFilePath() );
|
|
|
|
}
|
|
|
|
|
|
|
|
dir.rmdir( directory );
|
|
|
|
}
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
static void addEntryToIndex( QString const & word, uint32_t offset, IndexedWords & indexedWords )
|
|
|
|
{
|
|
|
|
// Strip any leading or trailing whitespaces
|
|
|
|
QString wordTrimmed = word.trimmed();
|
|
|
|
indexedWords.addWord( gd::toWString( wordTrimmed ), offset );
|
|
|
|
}
|
|
|
|
|
|
|
|
static void addEntryToIndexSingle( QString const & word, uint32_t offset, IndexedWords & indexedWords )
|
|
|
|
{
|
|
|
|
// Strip any leading or trailing whitespaces
|
|
|
|
QString wordTrimmed = word.trimmed();
|
|
|
|
indexedWords.addSingleWord( gd::toWString( wordTrimmed ), offset );
|
|
|
|
}
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
class ArticleHandler: public MdictParser::RecordHandler
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
public:
|
|
|
|
ArticleHandler( ChunkedStorage::Writer & chunks, IndexedWords & indexedWords ) :
|
|
|
|
chunks( chunks ),
|
2013-04-28 08:26:04 +00:00
|
|
|
indexedWords( indexedWords )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
virtual void handleRecord( QString const & headWord, MdictParser::RecordInfo const & recordInfo )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-04-28 08:26:04 +00:00
|
|
|
// Save the article's record info
|
2013-04-23 12:07:05 +00:00
|
|
|
uint32_t articleAddress = chunks.startNewBlock();
|
2013-04-28 08:26:04 +00:00
|
|
|
chunks.addToBlock( &recordInfo, sizeof( recordInfo ) );
|
2013-04-23 12:07:05 +00:00
|
|
|
// Add entries to the index
|
|
|
|
addEntryToIndex( headWord, articleAddress, indexedWords );
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
ChunkedStorage::Writer & chunks;
|
|
|
|
IndexedWords & indexedWords;
|
|
|
|
};
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
class ResourceHandler: public MdictParser::RecordHandler
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
public:
|
|
|
|
ResourceHandler( ChunkedStorage::Writer & chunks, IndexedWords & indexedWords ):
|
|
|
|
chunks( chunks ),
|
|
|
|
indexedWords( indexedWords )
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
virtual void handleRecord( QString const & fileName, MdictParser::RecordInfo const & recordInfo )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
uint32_t resourceInfoAddress = chunks.startNewBlock();
|
2013-04-28 08:26:04 +00:00
|
|
|
chunks.addToBlock( &recordInfo, sizeof( recordInfo ) );
|
2013-04-23 12:07:05 +00:00
|
|
|
// Add entries to the index
|
|
|
|
addEntryToIndexSingle( fileName, resourceInfoAddress, indexedWords );
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
ChunkedStorage::Writer & chunks;
|
|
|
|
IndexedWords & indexedWords;
|
|
|
|
};
|
|
|
|
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
static bool indexIsOldOrBad( vector< string > const & dictFiles, string const & indexFile )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
File::Class idx( indexFile, "rb" );
|
|
|
|
IdxHeader header;
|
|
|
|
|
|
|
|
return idx.readRecords( &header, sizeof( header ), 1 ) != 1 ||
|
|
|
|
header.signature != kSignature ||
|
|
|
|
header.formatVersion != kCurrentFormatVersion ||
|
|
|
|
header.parserVersion != MdictParser::kParserVersion ||
|
|
|
|
header.foldingVersion != Folding::Version ||
|
2013-05-08 13:39:47 +00:00
|
|
|
header.mddIndexInfosCount != dictFiles.size() - 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void findResourceFiles( string const & mdx, vector< string > & dictFiles )
|
|
|
|
{
|
|
|
|
string base( mdx, 0, mdx.size() - 4 );
|
|
|
|
// Check if there' is any file end with .mdd, which is the resource file for the dictionary
|
|
|
|
string resFile;
|
|
|
|
if ( File::tryPossibleName( base + ".mdd", resFile ) )
|
|
|
|
{
|
|
|
|
dictFiles.push_back( resFile );
|
|
|
|
// Find complementary .mdd file (volumes), like follows:
|
|
|
|
// demo.mdx <- main dictionary file
|
|
|
|
// demo.mdd <- main resource file ( 1st volume )
|
|
|
|
// demo.1.mdd <- 2nd volume
|
|
|
|
// ...
|
|
|
|
// demo.n.mdd <- nth volume
|
|
|
|
QString baseU8 = QString::fromUtf8( base.c_str() );
|
|
|
|
int vol = 1;
|
|
|
|
while ( File::tryPossibleName( string( QString( "%1.%2.mdd" ).arg( baseU8 ).arg( vol )
|
|
|
|
.toUtf8().constBegin() ), resFile ) )
|
|
|
|
{
|
|
|
|
dictFiles.push_back( resFile );
|
|
|
|
vol++;
|
|
|
|
}
|
|
|
|
}
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
vector< sptr< Dictionary::Class > > makeDictionaries( vector< string > const & fileNames,
|
|
|
|
string const & indicesDir,
|
2018-05-21 15:32:04 +00:00
|
|
|
Dictionary::Initializing & initializing ) THROW_SPEC( std::exception )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
vector< sptr< Dictionary::Class > > dictionaries;
|
|
|
|
|
2022-02-16 13:47:03 +00:00
|
|
|
for ( vector< string >::const_iterator i = fileNames.begin(); i != fileNames.end(); ++i )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
// Skip files with the extensions different to .mdx to speed up the
|
|
|
|
// scanning
|
|
|
|
if ( i->size() < 4 || strcasecmp( i->c_str() + ( i->size() - 4 ), ".mdx" ) != 0 )
|
|
|
|
continue;
|
|
|
|
|
|
|
|
vector< string > dictFiles( 1, *i );
|
2013-05-08 13:39:47 +00:00
|
|
|
findResourceFiles( *i, dictFiles );
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
string dictId = Dictionary::makeDictionaryId( dictFiles );
|
|
|
|
string indexFile = indicesDir + dictId;
|
|
|
|
|
|
|
|
if ( Dictionary::needToRebuildIndex( dictFiles, indexFile ) ||
|
2013-05-08 13:39:47 +00:00
|
|
|
indexIsOldOrBad( dictFiles, indexFile ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
// Building the index
|
2013-09-20 14:25:44 +00:00
|
|
|
|
2013-11-16 18:34:09 +00:00
|
|
|
gdDebug( "MDict: Building the index for dictionary: %s\n", i->c_str() );
|
2013-09-20 14:25:44 +00:00
|
|
|
|
2013-05-11 05:41:26 +00:00
|
|
|
MdictParser parser;
|
2013-05-08 13:39:47 +00:00
|
|
|
list< sptr< MdictParser > > mddParsers;
|
2013-04-23 12:07:05 +00:00
|
|
|
|
2013-05-11 05:41:26 +00:00
|
|
|
if ( !parser.open( i->c_str() ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
continue;
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
string title = string( parser.title().toUtf8().constData() );
|
|
|
|
initializing.indexingDictionary( title );
|
|
|
|
|
|
|
|
for ( vector< string >::const_iterator mddIter = dictFiles.begin() + 1;
|
2022-02-16 13:47:03 +00:00
|
|
|
mddIter != dictFiles.end(); ++mddIter )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
if ( File::exists( *mddIter ) )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-11 05:41:26 +00:00
|
|
|
sptr< MdictParser > mddParser = new MdictParser();
|
|
|
|
if ( !mddParser->open( mddIter->c_str() ) )
|
2013-05-08 13:39:47 +00:00
|
|
|
{
|
2017-06-22 15:02:04 +00:00
|
|
|
gdWarning( "Broken mdd (resource) file: %s\n", mddIter->c_str() );
|
2013-05-08 13:39:47 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
mddParsers.push_back( mddParser );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
File::Class idx( indexFile, "wb" );
|
|
|
|
IdxHeader idxHeader;
|
|
|
|
memset( &idxHeader, 0, sizeof( idxHeader ) );
|
|
|
|
// We write a dummy header first. At the end of the process the header
|
|
|
|
// will be rewritten with the right values.
|
|
|
|
idx.write( idxHeader );
|
2013-04-28 08:26:04 +00:00
|
|
|
|
|
|
|
// Write the title first
|
2013-04-23 12:07:05 +00:00
|
|
|
idx.write< uint32_t >( title.size() );
|
|
|
|
idx.write( title.data(), title.size() );
|
|
|
|
|
2013-04-28 08:26:04 +00:00
|
|
|
// then the encoding
|
|
|
|
{
|
|
|
|
string encoding = string( parser.encoding().toUtf8().constData() );
|
|
|
|
idx.write< uint32_t >( encoding.size() );
|
|
|
|
idx.write( encoding.data(), encoding.size() );
|
|
|
|
}
|
|
|
|
|
2013-04-23 12:07:05 +00:00
|
|
|
// This is our index data that we accumulate during the loading process.
|
|
|
|
// For each new word encountered, we emit the article's body to the file
|
|
|
|
// immediately, inserting the word itself and its offset in this map.
|
|
|
|
// This map maps folded words to the original words and the corresponding
|
|
|
|
// articles' offsets.
|
|
|
|
IndexedWords indexedWords;
|
|
|
|
ChunkedStorage::Writer chunks( idx );
|
|
|
|
|
|
|
|
idxHeader.isRightToLeft = parser.isRightToLeft();
|
|
|
|
|
|
|
|
// Save dictionary description if there's one
|
|
|
|
{
|
|
|
|
string description = string( parser.description().toUtf8().constData() );
|
|
|
|
idxHeader.descriptionAddress = chunks.startNewBlock();
|
|
|
|
chunks.addToBlock( description.c_str(), description.size() + 1 );
|
2013-04-28 08:26:04 +00:00
|
|
|
idxHeader.descriptionSize = description.size() + 1;
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ArticleHandler articleHandler( chunks, indexedWords );
|
|
|
|
MdictParser::HeadWordIndex headWordIndex;
|
|
|
|
|
|
|
|
// enumerating word and its definition
|
|
|
|
while ( parser.readNextHeadWordIndex( headWordIndex ) )
|
|
|
|
{
|
|
|
|
parser.readRecordBlock( headWordIndex, articleHandler );
|
|
|
|
}
|
|
|
|
|
|
|
|
// enumerating resources if there's any
|
2013-05-08 13:39:47 +00:00
|
|
|
vector< sptr< IndexedWords > > mddIndices;
|
|
|
|
vector< string > mddFileNames;
|
|
|
|
while ( !mddParsers.empty() )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
sptr< MdictParser > mddParser = mddParsers.front();
|
2013-05-11 05:41:26 +00:00
|
|
|
sptr< IndexedWords > mddIndexedWords = new IndexedWords();
|
2018-04-08 14:57:10 +00:00
|
|
|
MdictParser::HeadWordIndex resourcesIndex;
|
2013-04-23 12:07:05 +00:00
|
|
|
ResourceHandler resourceHandler( chunks, *mddIndexedWords );
|
|
|
|
|
|
|
|
while ( mddParser->readNextHeadWordIndex( headWordIndex ) )
|
|
|
|
{
|
2018-04-08 14:57:10 +00:00
|
|
|
resourcesIndex.insert( resourcesIndex.end(), headWordIndex.begin(), headWordIndex.end() );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
2018-04-08 14:57:10 +00:00
|
|
|
mddParser->readRecordBlock( resourcesIndex, resourceHandler );
|
2013-05-08 13:39:47 +00:00
|
|
|
|
|
|
|
mddIndices.push_back( mddIndexedWords );
|
2013-06-04 16:17:55 +00:00
|
|
|
// Save filename for .mdd files only
|
|
|
|
QFileInfo fi( mddParser->filename() );
|
|
|
|
mddFileNames.push_back( string( fi.fileName().toUtf8().constData() ) );
|
2013-05-08 13:39:47 +00:00
|
|
|
mddParsers.pop_front();
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Finish with the chunks
|
|
|
|
idxHeader.chunksOffset = chunks.finish();
|
|
|
|
|
2014-05-10 21:02:31 +00:00
|
|
|
GD_DPRINTF( "Writing index...\n" );
|
2013-04-23 12:07:05 +00:00
|
|
|
|
|
|
|
// Good. Now build the index
|
|
|
|
IndexInfo idxInfo = BtreeIndexing::buildIndex( indexedWords, idx );
|
|
|
|
idxHeader.indexBtreeMaxElements = idxInfo.btreeMaxElements;
|
|
|
|
idxHeader.indexRootOffset = idxInfo.rootOffset;
|
|
|
|
|
|
|
|
// Save dictionary stylesheets
|
|
|
|
{
|
|
|
|
MdictParser::StyleSheets const & styleSheets = parser.styleSheets();
|
|
|
|
idxHeader.styleSheetAddress = idx.tell();
|
|
|
|
idxHeader.styleSheetCount = styleSheets.size();
|
|
|
|
|
|
|
|
for ( MdictParser::StyleSheets::const_iterator iter = styleSheets.begin();
|
2022-02-16 13:47:03 +00:00
|
|
|
iter != styleSheets.end(); ++iter )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
|
|
|
string styleBegin( iter->second.first.toUtf8().constData() );
|
|
|
|
string styleEnd( iter->second.second.toUtf8().constData() );
|
|
|
|
|
|
|
|
// key
|
2013-06-04 16:17:55 +00:00
|
|
|
idx.write<qint32>( iter->first );
|
2013-04-23 12:07:05 +00:00
|
|
|
// styleBegin
|
2013-06-04 16:17:55 +00:00
|
|
|
idx.write<quint32>( ( quint32 )styleBegin.size() + 1 );
|
2013-04-23 12:07:05 +00:00
|
|
|
idx.write( styleBegin.c_str(), styleBegin.size() + 1 );
|
|
|
|
// styleEnd
|
2013-06-04 16:17:55 +00:00
|
|
|
idx.write<quint32>( ( quint32 )styleEnd.size() + 1 );
|
2013-04-23 12:07:05 +00:00
|
|
|
idx.write( styleEnd.c_str(), styleEnd.size() + 1 );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// read languages
|
|
|
|
QPair<quint32, quint32> langs = LangCoder::findIdsForFilename( QString::fromStdString( *i ) );
|
|
|
|
|
|
|
|
// if no languages found, try dictionary's name
|
|
|
|
if ( langs.first == 0 || langs.second == 0 )
|
|
|
|
{
|
|
|
|
langs = LangCoder::findIdsForFilename( parser.title() );
|
|
|
|
}
|
|
|
|
|
|
|
|
idxHeader.langFrom = langs.first;
|
|
|
|
idxHeader.langTo = langs.second;
|
|
|
|
|
2013-05-08 13:39:47 +00:00
|
|
|
// Build index info for each mdd file
|
|
|
|
vector< IndexInfo > mddIndexInfos;
|
|
|
|
for ( vector< sptr< IndexedWords > >::const_iterator mddIndexIter = mddIndices.begin();
|
2022-02-16 13:47:03 +00:00
|
|
|
mddIndexIter != mddIndices.end(); ++mddIndexIter )
|
2013-04-23 12:07:05 +00:00
|
|
|
{
|
2013-05-08 13:39:47 +00:00
|
|
|
IndexInfo resourceIdxInfo = BtreeIndexing::buildIndex( *( *mddIndexIter ), idx );
|
|
|
|
mddIndexInfos.push_back( resourceIdxInfo );
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save address of IndexInfos for resource files
|
|
|
|
idxHeader.mddIndexInfosOffset = idx.tell();
|
|
|
|
idxHeader.mddIndexInfosCount = mddIndexInfos.size();
|
|
|
|
for ( uint32_t mi = 0; mi < mddIndexInfos.size(); mi++ )
|
|
|
|
{
|
|
|
|
const string & mddfile = mddFileNames[ mi ];
|
|
|
|
|
2013-06-04 16:17:55 +00:00
|
|
|
idx.write<quint32>( ( quint32 )mddfile.size() + 1 );
|
2013-05-08 13:39:47 +00:00
|
|
|
idx.write( mddfile.c_str(), mddfile.size() + 1 );
|
|
|
|
idx.write<uint32_t>( mddIndexInfos[ mi ].btreeMaxElements );
|
|
|
|
idx.write<uint32_t>( mddIndexInfos[ mi ].rootOffset );
|
2013-04-23 12:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// That concludes it. Update the header.
|
|
|
|
idxHeader.signature = kSignature;
|
|
|
|
idxHeader.formatVersion = kCurrentFormatVersion;
|
|
|
|
idxHeader.parserVersion = MdictParser::kParserVersion;
|
|
|
|
idxHeader.foldingVersion = Folding::Version;
|
2013-04-28 08:26:04 +00:00
|
|
|
idxHeader.articleCount = parser.wordCount();
|
2013-04-23 12:07:05 +00:00
|
|
|
idxHeader.wordCount = parser.wordCount();
|
|
|
|
|
|
|
|
idx.rewind();
|
|
|
|
idx.write( &idxHeader, sizeof( idxHeader ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
dictionaries.push_back( new MdxDictionary( dictId, indexFile, dictFiles ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
return dictionaries;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|