2012-02-20 21:47:14 +00:00
|
|
|
/* This file is (c) 2008-2012 Konstantin Isakov <ikm@goldendict.org>
|
2009-04-09 18:50:49 +00:00
|
|
|
* Part of GoldenDict. Licensed under GPLv3 or later, see the LICENSE file */
|
|
|
|
|
|
|
|
#include "dictdfiles.hh"
|
|
|
|
#include "btreeidx.hh"
|
|
|
|
#include "folding.hh"
|
|
|
|
#include "utf8.hh"
|
2023-04-17 20:12:27 +00:00
|
|
|
#include "dictzip.hh"
|
2009-04-09 18:50:49 +00:00
|
|
|
#include "htmlescape.hh"
|
2023-04-28 16:09:45 +00:00
|
|
|
|
2009-05-05 22:45:02 +00:00
|
|
|
#include "langcoder.hh"
|
2009-04-09 18:50:49 +00:00
|
|
|
#include <map>
|
|
|
|
#include <set>
|
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
|
|
|
#include <list>
|
|
|
|
#include <wctype.h>
|
|
|
|
#include <stdlib.h>
|
2013-11-16 18:34:09 +00:00
|
|
|
#include "gddebug.hh"
|
2014-04-16 16:18:28 +00:00
|
|
|
#include "ftshelpers.hh"
|
2016-04-12 16:48:10 +00:00
|
|
|
#include <QUrl>
|
2009-04-09 18:50:49 +00:00
|
|
|
|
2013-09-20 14:25:44 +00:00
|
|
|
|
2018-02-21 14:43:35 +00:00
|
|
|
#include <QRegularExpression>
|
|
|
|
|
2009-04-29 23:18:26 +00:00
|
|
|
#ifdef _MSC_VER
|
|
|
|
#include <stub_msvc.h>
|
|
|
|
#endif
|
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
namespace DictdFiles {
|
|
|
|
|
|
|
|
using std::map;
|
|
|
|
using std::multimap;
|
|
|
|
using std::pair;
|
|
|
|
using std::set;
|
|
|
|
using std::string;
|
2009-04-18 17:20:12 +00:00
|
|
|
using gd::wstring;
|
2009-04-09 18:50:49 +00:00
|
|
|
using std::vector;
|
|
|
|
using std::list;
|
|
|
|
|
|
|
|
using BtreeIndexing::WordArticleLink;
|
|
|
|
using BtreeIndexing::IndexedWords;
|
2009-04-14 16:35:47 +00:00
|
|
|
using BtreeIndexing::IndexInfo;
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
DEF_EX_STR( exCantReadFile, "Can't read file", Dictionary::Ex )
|
|
|
|
DEF_EX( exFailedToReadLineFromIndex, "Failed to read line from index file", Dictionary::Ex )
|
|
|
|
DEF_EX( exMalformedIndexFileLine, "Malformed index file line encountered", Dictionary::Ex )
|
|
|
|
DEF_EX( exInvalidBase64, "Invalid base64 sequence encountered", Dictionary::Ex )
|
2014-04-24 18:50:47 +00:00
|
|
|
DEF_EX_STR( exDictzipError, "DICTZIP error", Dictionary::Ex )
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
enum
|
|
|
|
{
|
|
|
|
Signature = 0x58444344, // DCDX on little-endian, XDCD on big-endian
|
2013-06-16 17:47:04 +00:00
|
|
|
CurrentFormatVersion = 5 + BtreeIndexing::FormatVersion + Folding::Version
|
2009-04-09 18:50:49 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct IdxHeader
|
|
|
|
{
|
|
|
|
uint32_t signature; // First comes the signature, DCDX
|
|
|
|
uint32_t formatVersion; // File format version (CurrentFormatVersion)
|
|
|
|
uint32_t wordCount; // Total number of words
|
2013-06-16 17:47:04 +00:00
|
|
|
uint32_t articleCount; // Total number of articles
|
2009-04-14 16:35:47 +00:00
|
|
|
uint32_t indexBtreeMaxElements; // Two fields from IndexInfo
|
|
|
|
uint32_t indexRootOffset;
|
2009-05-05 22:45:02 +00:00
|
|
|
uint32_t langFrom; // Source language
|
|
|
|
uint32_t langTo; // Target language
|
|
|
|
}
|
2009-04-29 23:18:26 +00:00
|
|
|
#ifndef _MSC_VER
|
|
|
|
__attribute__((packed))
|
|
|
|
#endif
|
|
|
|
;
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
bool indexIsOldOrBad( string const & indexFile )
|
|
|
|
{
|
|
|
|
File::Class idx( indexFile, "rb" );
|
|
|
|
|
|
|
|
IdxHeader header;
|
|
|
|
|
|
|
|
return idx.readRecords( &header, sizeof( header ), 1 ) != 1 ||
|
|
|
|
header.signature != Signature ||
|
|
|
|
header.formatVersion != CurrentFormatVersion;
|
|
|
|
}
|
|
|
|
|
|
|
|
class DictdDictionary: public BtreeIndexing::BtreeDictionary
|
|
|
|
{
|
2023-05-29 13:56:04 +00:00
|
|
|
QMutex idxMutex;
|
2009-04-09 18:50:49 +00:00
|
|
|
File::Class idx, indexFile; // The later is .index file
|
|
|
|
IdxHeader idxHeader;
|
|
|
|
dictData * dz;
|
2023-05-29 13:56:04 +00:00
|
|
|
QMutex indexFileMutex, dzMutex;
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
DictdDictionary( string const & id, string const & indexFile,
|
|
|
|
vector< string > const & dictionaryFiles );
|
|
|
|
|
|
|
|
~DictdDictionary();
|
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
string getName() noexcept override
|
2013-03-13 09:45:11 +00:00
|
|
|
{ return dictionaryName; }
|
2009-04-09 18:50:49 +00:00
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
map< Dictionary::Property, string > getProperties() noexcept override
|
2009-04-09 18:50:49 +00:00
|
|
|
{ return map< Dictionary::Property, string >(); }
|
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
unsigned long getArticleCount() noexcept override
|
2013-06-16 17:47:04 +00:00
|
|
|
{ return idxHeader.articleCount; }
|
2009-04-09 18:50:49 +00:00
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
unsigned long getWordCount() noexcept override
|
2009-04-09 18:50:49 +00:00
|
|
|
{ return idxHeader.wordCount; }
|
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
void loadIcon() noexcept override;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
inline quint32 getLangFrom() const override
|
2009-05-05 22:45:02 +00:00
|
|
|
{ return idxHeader.langFrom; }
|
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
inline quint32 getLangTo() const override
|
2009-05-05 22:45:02 +00:00
|
|
|
{ return idxHeader.langTo; }
|
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
sptr< Dictionary::DataRequest > getArticle( wstring const &,
|
2009-05-29 19:48:50 +00:00
|
|
|
vector< wstring > const & alts,
|
2018-06-13 16:00:42 +00:00
|
|
|
wstring const &,
|
2022-12-29 07:07:40 +00:00
|
|
|
bool ignoreDiacritics ) override
|
2022-01-09 08:35:07 +00:00
|
|
|
;
|
2014-03-19 13:59:39 +00:00
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
QString const& getDescription() override;
|
2014-04-16 16:18:28 +00:00
|
|
|
|
2023-05-30 23:42:31 +00:00
|
|
|
sptr< Dictionary::DataRequest >
|
|
|
|
getSearchResults( QString const & searchString, int searchMode, bool matchCase, bool ignoreDiacritics ) override;
|
2022-12-29 07:07:40 +00:00
|
|
|
void getArticleText( uint32_t articleAddress, QString & headword, QString & text ) override;
|
2014-04-16 16:18:28 +00:00
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ) override;
|
2014-04-16 16:18:28 +00:00
|
|
|
|
2022-12-29 07:07:40 +00:00
|
|
|
void setFTSParameters( Config::FullTextSearch const & fts ) override
|
2014-04-17 14:31:51 +00:00
|
|
|
{
|
|
|
|
can_FTS = fts.enabled
|
|
|
|
&& !fts.disabledTypes.contains( "DICTD", Qt::CaseInsensitive )
|
|
|
|
&& ( fts.maxDictionarySize == 0 || getArticleCount() <= fts.maxDictionarySize );
|
|
|
|
}
|
2009-04-09 18:50:49 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
DictdDictionary::DictdDictionary( string const & id,
|
|
|
|
string const & indexFile,
|
|
|
|
vector< string > const & dictionaryFiles ):
|
|
|
|
BtreeDictionary( id, dictionaryFiles ),
|
|
|
|
idx( indexFile, "rb" ),
|
|
|
|
indexFile( dictionaryFiles[ 0 ], "rb" ),
|
|
|
|
idxHeader( idx.read< IdxHeader >() )
|
|
|
|
{
|
2013-03-13 09:45:11 +00:00
|
|
|
|
|
|
|
// Read the dictionary name
|
|
|
|
idx.seek( sizeof( idxHeader ) );
|
|
|
|
|
|
|
|
vector< char > dName( idx.read< uint32_t >() );
|
2019-01-17 14:53:13 +00:00
|
|
|
if( dName.size() > 0 )
|
|
|
|
{
|
|
|
|
idx.read( &dName.front(), dName.size() );
|
|
|
|
dictionaryName = string( &dName.front(), dName.size() );
|
|
|
|
}
|
2013-03-13 09:45:11 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
// Open the .dict file
|
|
|
|
|
2014-04-24 18:50:47 +00:00
|
|
|
DZ_ERRORS error;
|
|
|
|
dz = dict_data_open( dictionaryFiles[ 1 ].c_str(), &error, 0 );
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
if ( !dz )
|
2014-04-24 18:50:47 +00:00
|
|
|
throw exDictzipError( string( dz_error_str( error ) )
|
|
|
|
+ "(" + getDictionaryFilenames()[ 1 ] + ")" );
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
// Initialize the index
|
|
|
|
|
2009-04-14 16:35:47 +00:00
|
|
|
openIndex( IndexInfo( idxHeader.indexBtreeMaxElements,
|
|
|
|
idxHeader.indexRootOffset ),
|
|
|
|
idx, idxMutex );
|
2014-04-16 16:18:28 +00:00
|
|
|
|
|
|
|
// Full-text search parameters
|
|
|
|
|
|
|
|
can_FTS = true;
|
|
|
|
|
2022-10-06 03:04:48 +00:00
|
|
|
ftsIdxName = indexFile + Dictionary::getFtsSuffix();
|
2014-04-16 16:18:28 +00:00
|
|
|
|
2023-06-03 00:29:19 +00:00
|
|
|
if ( !Dictionary::needToRebuildIndex( dictionaryFiles, ftsIdxName ) && !FtsHelpers::ftsIndexIsOldOrBad( this ) )
|
2014-05-08 12:38:00 +00:00
|
|
|
FTS_index_completed.ref();
|
2009-04-09 18:50:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
DictdDictionary::~DictdDictionary()
|
|
|
|
{
|
|
|
|
if ( dz )
|
|
|
|
dict_data_close( dz );
|
|
|
|
}
|
|
|
|
|
|
|
|
string nameFromFileName( string const & indexFileName )
|
|
|
|
{
|
|
|
|
if ( indexFileName.empty() )
|
|
|
|
return string();
|
|
|
|
|
2023-04-28 16:09:45 +00:00
|
|
|
char const * sep = strrchr( indexFileName.c_str(), Utils::Fs::separator() );
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
if ( !sep )
|
|
|
|
sep = indexFileName.c_str();
|
|
|
|
|
|
|
|
char const * dot = strrchr( sep, '.' );
|
|
|
|
|
|
|
|
if ( !dot )
|
|
|
|
dot = indexFileName.c_str() + indexFileName.size();
|
|
|
|
|
2023-04-13 10:08:32 +00:00
|
|
|
return string( sep + 1, dot - sep - 1 );
|
2009-04-09 18:50:49 +00:00
|
|
|
}
|
|
|
|
|
2022-06-03 13:28:41 +00:00
|
|
|
void DictdDictionary::loadIcon() noexcept
|
2013-01-31 23:53:45 +00:00
|
|
|
{
|
|
|
|
if ( dictionaryIconLoaded )
|
|
|
|
return;
|
|
|
|
|
2023-04-13 10:08:32 +00:00
|
|
|
QString fileName = QDir::fromNativeSeparators( QString::fromStdString( getDictionaryFilenames()[ 0 ] ) );
|
2013-01-31 23:53:45 +00:00
|
|
|
|
|
|
|
// Remove the extension
|
|
|
|
fileName.chop( 5 );
|
|
|
|
|
|
|
|
if( !loadIconFromFile( fileName ) )
|
|
|
|
{
|
|
|
|
// Load failed -- use default icons
|
2023-06-19 02:34:08 +00:00
|
|
|
dictionaryIcon = QIcon(":/icons/icon32_dictd.png");
|
2013-01-31 23:53:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
dictionaryIconLoaded = true;
|
|
|
|
}
|
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
uint32_t decodeBase64( string const & str )
|
|
|
|
{
|
|
|
|
static char const digits[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
|
|
|
|
|
|
uint32_t number = 0;
|
|
|
|
|
|
|
|
for( char const * next = str.c_str(); *next; ++next )
|
|
|
|
{
|
|
|
|
char const * d = strchr( digits, *next );
|
|
|
|
|
|
|
|
if ( !d )
|
|
|
|
throw exInvalidBase64();
|
|
|
|
|
|
|
|
number = number * 64 + ( d - digits );
|
|
|
|
}
|
|
|
|
|
|
|
|
return number;
|
|
|
|
}
|
|
|
|
|
|
|
|
sptr< Dictionary::DataRequest > DictdDictionary::getArticle( wstring const & word,
|
2009-05-29 19:48:50 +00:00
|
|
|
vector< wstring > const & alts,
|
2018-06-13 16:00:42 +00:00
|
|
|
wstring const &,
|
|
|
|
bool ignoreDiacritics )
|
2022-01-09 08:35:07 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
2018-06-13 16:00:42 +00:00
|
|
|
vector< WordArticleLink > chain = findArticles( word, ignoreDiacritics );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
for( unsigned x = 0; x < alts.size(); ++x )
|
|
|
|
{
|
|
|
|
/// Make an additional query for each alt
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2018-06-13 16:00:42 +00:00
|
|
|
vector< WordArticleLink > altChain = findArticles( alts[ x ], ignoreDiacritics );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
chain.insert( chain.end(), altChain.begin(), altChain.end() );
|
|
|
|
}
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
multimap< wstring, string > mainArticles, alternateArticles;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
set< uint32_t > articlesIncluded; // Some synonyms make it that the articles
|
|
|
|
// appear several times. We combat this
|
|
|
|
// by only allowing them to appear once.
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
wstring wordCaseFolded = Folding::applySimpleCaseOnly( word );
|
2018-06-13 16:00:42 +00:00
|
|
|
if( ignoreDiacritics )
|
|
|
|
wordCaseFolded = Folding::applyDiacriticsOnly( wordCaseFolded );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
char buf[ 16384 ];
|
|
|
|
|
|
|
|
for( unsigned x = 0; x < chain.size(); ++x )
|
|
|
|
{
|
|
|
|
if ( articlesIncluded.find( chain[ x ].articleOffset ) != articlesIncluded.end() )
|
|
|
|
continue; // We already have this article in the body.
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
// Now load that article
|
|
|
|
|
2014-04-16 16:18:28 +00:00
|
|
|
{
|
2023-05-29 13:56:04 +00:00
|
|
|
QMutexLocker _( &indexFileMutex );
|
2014-04-16 16:18:28 +00:00
|
|
|
indexFile.seek( chain[ x ].articleOffset );
|
2009-04-09 18:50:49 +00:00
|
|
|
|
2014-04-16 16:18:28 +00:00
|
|
|
if ( !indexFile.gets( buf, sizeof( buf ), true ) )
|
|
|
|
throw exFailedToReadLineFromIndex();
|
|
|
|
}
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
char * tab1 = strchr( buf, '\t' );
|
|
|
|
|
|
|
|
if ( !tab1 )
|
|
|
|
throw exMalformedIndexFileLine();
|
|
|
|
|
|
|
|
char * tab2 = strchr( tab1 + 1, '\t' );
|
|
|
|
|
|
|
|
if ( !tab2 )
|
|
|
|
throw exMalformedIndexFileLine();
|
|
|
|
|
|
|
|
// After tab1 should be article offset, after tab2 -- article size
|
|
|
|
|
|
|
|
uint32_t articleOffset = decodeBase64( string( tab1 + 1, tab2 - tab1 - 1 ) );
|
2013-06-16 17:47:04 +00:00
|
|
|
|
|
|
|
char * tab3 = strchr( tab2 + 1, '\t');
|
|
|
|
|
|
|
|
uint32_t articleSize;
|
|
|
|
if ( tab3 )
|
|
|
|
{
|
|
|
|
articleSize = decodeBase64( string( tab2 + 1, tab3 - tab2 - 1 ) );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
articleSize = decodeBase64( tab2 + 1 );
|
|
|
|
}
|
|
|
|
|
2013-03-15 12:27:32 +00:00
|
|
|
string articleText;
|
2009-04-09 18:50:49 +00:00
|
|
|
|
2014-04-16 16:18:28 +00:00
|
|
|
char * articleBody;
|
|
|
|
{
|
2023-05-29 13:56:04 +00:00
|
|
|
QMutexLocker _( &dzMutex );
|
2014-04-16 16:18:28 +00:00
|
|
|
articleBody = dict_data_read_( dz, articleOffset, articleSize, 0, 0 );
|
|
|
|
}
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
if ( !articleBody )
|
2013-03-15 12:27:32 +00:00
|
|
|
{
|
|
|
|
articleText = string( "<div class=\"dictd_article\">DICTZIP error: " )
|
|
|
|
+ dict_error_str( dz ) + "</div>";
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2022-12-24 22:01:50 +00:00
|
|
|
static QRegularExpression phonetic( R"(\\([^\\]+)\\)",
|
2018-02-21 14:43:35 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption ); // phonetics: \stuff\ ...
|
2022-12-24 22:01:50 +00:00
|
|
|
static QRegularExpression refs( R"(\{([^\{\}]+)\})",
|
2018-02-21 14:43:35 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption ); // links: {stuff}
|
|
|
|
static QRegularExpression links( "<a href=\"gdlookup://localhost/([^\"]*)\">",
|
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
|
|
|
static QRegularExpression tags( "<[^>]*>",
|
|
|
|
QRegularExpression::CaseInsensitiveOption );
|
2021-09-22 04:01:17 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
|
2013-07-10 13:48:09 +00:00
|
|
|
articleText = string( "<div class=\"dictd_article\"" );
|
|
|
|
if( isToLanguageRTL() )
|
|
|
|
articleText += " dir=\"rtl\"";
|
|
|
|
articleText += ">";
|
|
|
|
|
2013-07-13 11:24:58 +00:00
|
|
|
string convertedText = Html::preformat( articleBody, isToLanguageRTL() );
|
2013-07-10 13:48:09 +00:00
|
|
|
free( articleBody );
|
|
|
|
|
2016-04-12 16:48:10 +00:00
|
|
|
QString articleString = QString::fromUtf8( convertedText.c_str() )
|
2022-12-24 22:01:50 +00:00
|
|
|
.replace(phonetic, R"(<span class="dictd_phonetic">\1</span>)")
|
|
|
|
.replace(refs, R"(<a href="gdlookup://localhost/\1">\1</a>)");
|
2016-04-12 16:48:10 +00:00
|
|
|
convertedText.erase();
|
|
|
|
|
|
|
|
int pos = 0;
|
2021-09-22 04:01:17 +00:00
|
|
|
|
2018-02-27 16:42:21 +00:00
|
|
|
QString articleNewString;
|
|
|
|
QRegularExpressionMatchIterator it = links.globalMatch( articleString );
|
|
|
|
while( it.hasNext() )
|
2018-02-21 14:43:35 +00:00
|
|
|
{
|
2018-02-27 16:42:21 +00:00
|
|
|
QRegularExpressionMatch match = it.next();
|
2022-02-27 05:17:37 +00:00
|
|
|
articleNewString += articleString.mid( pos, match.capturedStart() - pos );
|
2018-02-27 16:42:21 +00:00
|
|
|
pos = match.capturedEnd();
|
2018-02-21 14:43:35 +00:00
|
|
|
|
|
|
|
QString link = match.captured( 1 );
|
|
|
|
link.replace( tags, " " );
|
|
|
|
link.replace( " ", " " );
|
2018-02-27 16:42:21 +00:00
|
|
|
|
|
|
|
QString newLink = match.captured();
|
|
|
|
newLink.replace( 30, match.capturedLength( 1 ),
|
|
|
|
QString::fromUtf8( QUrl::toPercentEncoding( link.simplified() ) ) );
|
|
|
|
articleNewString += newLink;
|
|
|
|
}
|
|
|
|
if( pos )
|
|
|
|
{
|
2022-02-27 05:17:37 +00:00
|
|
|
articleNewString += articleString.mid( pos );
|
2018-02-27 16:42:21 +00:00
|
|
|
articleString = articleNewString;
|
|
|
|
articleNewString.clear();
|
2018-02-21 14:43:35 +00:00
|
|
|
}
|
2016-04-12 16:48:10 +00:00
|
|
|
|
|
|
|
|
|
|
|
articleString += "</div>";
|
|
|
|
|
|
|
|
articleText += articleString.toUtf8().data();
|
2013-03-15 12:27:32 +00:00
|
|
|
}
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
// Ok. Now, does it go to main articles, or to alternate ones? We list
|
|
|
|
// main ones first, and alternates after.
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
// We do the case-folded comparison here.
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
wstring headwordStripped =
|
2023-04-29 02:35:56 +00:00
|
|
|
Folding::applySimpleCaseOnly( chain[ x ].word );
|
2018-06-13 16:00:42 +00:00
|
|
|
if( ignoreDiacritics )
|
|
|
|
headwordStripped = Folding::applyDiacriticsOnly( headwordStripped );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
|
|
|
multimap< wstring, string > & mapToUse =
|
2009-04-09 18:50:49 +00:00
|
|
|
( wordCaseFolded == headwordStripped ) ?
|
|
|
|
mainArticles : alternateArticles;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2023-04-29 02:35:56 +00:00
|
|
|
mapToUse.insert( pair(
|
|
|
|
Folding::applySimpleCaseOnly( chain[ x ].word ),
|
2009-04-09 18:50:49 +00:00
|
|
|
articleText ) );
|
|
|
|
|
|
|
|
articlesIncluded.insert( chain[ x ].articleOffset );
|
|
|
|
}
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
if ( mainArticles.empty() && alternateArticles.empty() )
|
2022-11-29 03:54:31 +00:00
|
|
|
return std::make_shared<Dictionary::DataRequestInstant>( false );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
string result;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
multimap< wstring, string >::const_iterator i;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
for( i = mainArticles.begin(); i != mainArticles.end(); ++i )
|
|
|
|
result += i->second;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
for( i = alternateArticles.begin(); i != alternateArticles.end(); ++i )
|
|
|
|
result += i->second;
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
sptr< Dictionary::DataRequestInstant > ret =
|
2022-11-29 03:54:31 +00:00
|
|
|
std::make_shared<Dictionary::DataRequestInstant>( true );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
ret->getData().resize( result.size() );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
memcpy( &(ret->getData().front()), result.data(), result.size() );
|
2009-04-21 20:09:02 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
catch( std::exception & e )
|
|
|
|
{
|
2022-11-29 03:54:31 +00:00
|
|
|
return std::make_shared<Dictionary::DataRequestInstant>( QString( e.what() ) );
|
2009-04-09 18:50:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-19 13:59:39 +00:00
|
|
|
QString const& DictdDictionary::getDescription()
|
|
|
|
{
|
|
|
|
if( !dictionaryDescription.isEmpty() )
|
|
|
|
return dictionaryDescription;
|
|
|
|
|
2022-02-17 14:01:09 +00:00
|
|
|
sptr< Dictionary::DataRequest > req =
|
|
|
|
getArticle( U"00databaseinfo" , vector< wstring >(), wstring(), false );
|
2014-03-19 13:59:39 +00:00
|
|
|
|
2023-04-27 15:47:35 +00:00
|
|
|
if ( req->dataSize() > 0 ) {
|
|
|
|
dictionaryDescription =
|
|
|
|
Html::unescape( QString::fromUtf8( req->getFullData().data(), req->getFullData().size() ),
|
|
|
|
Html::HtmlOption::Keep );
|
|
|
|
}
|
|
|
|
else {
|
2014-03-19 13:59:39 +00:00
|
|
|
dictionaryDescription = "NONE";
|
2023-04-27 15:47:35 +00:00
|
|
|
}
|
2014-03-19 13:59:39 +00:00
|
|
|
|
|
|
|
return dictionaryDescription;
|
|
|
|
}
|
|
|
|
|
2014-04-16 16:18:28 +00:00
|
|
|
void DictdDictionary::makeFTSIndex( QAtomicInt & isCancelled, bool firstIteration )
|
|
|
|
{
|
2023-06-03 00:29:19 +00:00
|
|
|
if ( !( Dictionary::needToRebuildIndex( getDictionaryFilenames(), ftsIdxName )
|
|
|
|
|| FtsHelpers::ftsIndexIsOldOrBad( this ) ) )
|
2014-04-16 16:18:28 +00:00
|
|
|
FTS_index_completed.ref();
|
|
|
|
|
2023-06-03 00:29:19 +00:00
|
|
|
if ( haveFTSIndex() )
|
2014-04-16 16:18:28 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
if( ensureInitDone().size() )
|
|
|
|
return;
|
|
|
|
|
|
|
|
if( firstIteration && getArticleCount() > FTS::MaxDictionarySizeForFastSearch )
|
|
|
|
return;
|
|
|
|
|
|
|
|
gdDebug( "DictD: Building the full-text index for dictionary: %s\n",
|
|
|
|
getName().c_str() );
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
FtsHelpers::makeFTSIndex( this, isCancelled );
|
2014-04-17 14:31:51 +00:00
|
|
|
FTS_index_completed.ref();
|
2014-04-16 16:18:28 +00:00
|
|
|
}
|
|
|
|
catch( std::exception &ex )
|
|
|
|
{
|
|
|
|
gdWarning( "DictD: Failed building full-text search index for \"%s\", reason: %s\n", getName().c_str(), ex.what() );
|
2023-04-13 10:08:32 +00:00
|
|
|
QFile::remove( QString::fromStdString( ftsIdxName ) );
|
2014-04-16 16:18:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void DictdDictionary::getArticleText( uint32_t articleAddress, QString & headword, QString & text )
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
char buf[ 16384 ];
|
|
|
|
{
|
2023-05-29 13:56:04 +00:00
|
|
|
QMutexLocker _( &indexFileMutex );
|
2014-04-16 16:18:28 +00:00
|
|
|
indexFile.seek( articleAddress );
|
|
|
|
|
|
|
|
if ( !indexFile.gets( buf, sizeof( buf ), true ) )
|
|
|
|
throw exFailedToReadLineFromIndex();
|
|
|
|
}
|
|
|
|
|
|
|
|
char * tab1 = strchr( buf, '\t' );
|
|
|
|
|
|
|
|
if ( !tab1 )
|
|
|
|
throw exMalformedIndexFileLine();
|
|
|
|
|
|
|
|
headword = QString::fromUtf8( buf, tab1 - buf );
|
|
|
|
|
|
|
|
char * tab2 = strchr( tab1 + 1, '\t' );
|
|
|
|
|
|
|
|
if ( !tab2 )
|
|
|
|
throw exMalformedIndexFileLine();
|
|
|
|
|
|
|
|
// After tab1 should be article offset, after tab2 -- article size
|
|
|
|
|
|
|
|
uint32_t articleOffset = decodeBase64( string( tab1 + 1, tab2 - tab1 - 1 ) );
|
|
|
|
|
|
|
|
char * tab3 = strchr( tab2 + 1, '\t');
|
|
|
|
|
|
|
|
uint32_t articleSize;
|
|
|
|
if ( tab3 )
|
|
|
|
{
|
|
|
|
articleSize = decodeBase64( string( tab2 + 1, tab3 - tab2 - 1 ) );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
articleSize = decodeBase64( tab2 + 1 );
|
|
|
|
}
|
|
|
|
|
|
|
|
string articleText;
|
|
|
|
|
2014-04-20 20:02:41 +00:00
|
|
|
char * articleBody;
|
|
|
|
{
|
2023-05-29 13:56:04 +00:00
|
|
|
QMutexLocker _( &dzMutex );
|
2014-04-20 20:02:41 +00:00
|
|
|
articleBody = dict_data_read_( dz, articleOffset, articleSize, 0, 0 );
|
|
|
|
}
|
2014-04-16 16:18:28 +00:00
|
|
|
|
|
|
|
if ( !articleBody )
|
|
|
|
{
|
|
|
|
articleText = dict_error_str( dz );
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2022-12-24 22:01:50 +00:00
|
|
|
static QRegularExpression phonetic( R"(\\([^\\]+)\\)",
|
2018-02-21 14:43:35 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption ); // phonetics: \stuff\ ...
|
2022-12-24 22:01:50 +00:00
|
|
|
static QRegularExpression refs( R"(\{([^\{\}]+)\})",
|
2018-02-21 14:43:35 +00:00
|
|
|
QRegularExpression::CaseInsensitiveOption ); // links: {stuff}
|
2014-04-16 16:18:28 +00:00
|
|
|
|
|
|
|
string convertedText = Html::preformat( articleBody, isToLanguageRTL() );
|
|
|
|
free( articleBody );
|
|
|
|
|
|
|
|
text = QString::fromUtf8( convertedText.data(), convertedText.size() )
|
2022-12-24 22:01:50 +00:00
|
|
|
.replace(phonetic, R"(<span class="dictd_phonetic">\1</span>)")
|
|
|
|
.replace(refs, R"(<a href="gdlookup://localhost/\1">\1</a>)");
|
2014-04-16 16:18:28 +00:00
|
|
|
|
|
|
|
text = Html::unescape( text );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
catch( std::exception &ex )
|
|
|
|
{
|
|
|
|
gdWarning( "DictD: Failed retrieving article from \"%s\", reason: %s\n", getName().c_str(), ex.what() );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-30 23:42:31 +00:00
|
|
|
sptr< Dictionary::DataRequest >
|
|
|
|
DictdDictionary::getSearchResults( QString const & searchString, int searchMode, bool matchCase, bool ignoreDiacritics )
|
2014-04-16 16:18:28 +00:00
|
|
|
{
|
2023-05-30 23:42:31 +00:00
|
|
|
return std::make_shared< FtsHelpers::FTSResultsRequest >( *this,
|
|
|
|
searchString,
|
|
|
|
searchMode,
|
|
|
|
matchCase,
|
|
|
|
ignoreDiacritics );
|
2014-04-16 16:18:28 +00:00
|
|
|
}
|
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
} // anonymous namespace
|
|
|
|
|
|
|
|
vector< sptr< Dictionary::Class > > makeDictionaries(
|
|
|
|
vector< string > const & fileNames,
|
|
|
|
string const & indicesDir,
|
|
|
|
Dictionary::Initializing & initializing )
|
2022-01-09 08:35:07 +00:00
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
{
|
|
|
|
vector< sptr< Dictionary::Class > > dictionaries;
|
|
|
|
|
|
|
|
for( vector< string >::const_iterator i = fileNames.begin(); i != fileNames.end();
|
|
|
|
++i )
|
|
|
|
{
|
|
|
|
// Only allow .index suffixes
|
|
|
|
|
|
|
|
if ( i->size() < 6 ||
|
|
|
|
strcasecmp( i->c_str() + ( i->size() - 6 ), ".index" ) != 0 )
|
|
|
|
continue;
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
vector< string > dictFiles( 1, *i );
|
|
|
|
|
|
|
|
// Check if there is an 'abrv' file present
|
|
|
|
string baseName( *i, 0, i->size() - 5 );
|
|
|
|
|
|
|
|
dictFiles.push_back( string() );
|
|
|
|
|
2012-01-30 13:11:41 +00:00
|
|
|
if ( !File::tryPossibleName( baseName + "dict", dictFiles[ 1 ] ) &&
|
|
|
|
!File::tryPossibleName( baseName + "dict.dz", dictFiles[ 1 ] ) )
|
2009-04-09 18:50:49 +00:00
|
|
|
{
|
|
|
|
// No corresponding .dict file, skipping
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
string dictId = Dictionary::makeDictionaryId( dictFiles );
|
|
|
|
|
|
|
|
string indexFile = indicesDir + dictId;
|
|
|
|
|
|
|
|
if ( Dictionary::needToRebuildIndex( dictFiles, indexFile ) ||
|
|
|
|
indexIsOldOrBad( indexFile ) )
|
|
|
|
{
|
|
|
|
// Building the index
|
2013-03-13 09:45:11 +00:00
|
|
|
string dictionaryName = nameFromFileName( dictFiles[ 0 ] );
|
2013-09-20 14:25:44 +00:00
|
|
|
|
2013-11-16 18:34:09 +00:00
|
|
|
gdDebug( "DictD: Building the index for dictionary: %s\n", dictionaryName.c_str() );
|
2013-09-20 14:25:44 +00:00
|
|
|
|
2013-03-13 09:45:11 +00:00
|
|
|
initializing.indexingDictionary( dictionaryName );
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
File::Class idx( indexFile, "wb" );
|
|
|
|
|
|
|
|
IdxHeader idxHeader;
|
|
|
|
|
|
|
|
memset( &idxHeader, 0, sizeof( idxHeader ) );
|
|
|
|
|
|
|
|
// We write a dummy header first. At the end of the process the header
|
|
|
|
// will be rewritten with the right values.
|
|
|
|
|
|
|
|
idx.write( idxHeader );
|
|
|
|
|
|
|
|
IndexedWords indexedWords;
|
|
|
|
|
2009-04-11 14:02:31 +00:00
|
|
|
File::Class indexFile( dictFiles[ 0 ], "rb" );
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
// Read words from index until none's left.
|
|
|
|
|
|
|
|
char buf[ 16384 ];
|
|
|
|
|
|
|
|
do
|
|
|
|
{
|
|
|
|
uint32_t curOffset = indexFile.tell();
|
|
|
|
|
|
|
|
if ( !indexFile.gets( buf, sizeof( buf ), true ) )
|
|
|
|
break;
|
|
|
|
|
2013-06-16 17:47:04 +00:00
|
|
|
// Check that there are exactly two or three tabs in the record.
|
|
|
|
char * tab1 = strchr( buf, '\t' );
|
|
|
|
if ( tab1 )
|
2013-03-13 09:45:11 +00:00
|
|
|
{
|
2013-06-16 17:47:04 +00:00
|
|
|
char * tab2 = strchr( tab1 + 1, '\t' );
|
|
|
|
if ( tab2 )
|
2013-03-13 09:45:11 +00:00
|
|
|
{
|
2013-06-16 17:47:04 +00:00
|
|
|
char * tab3 = strchr( tab2 + 1, '\t');
|
|
|
|
if ( tab3 )
|
|
|
|
{
|
|
|
|
char * tab4 = strchr( tab3 + 1, '\t');
|
|
|
|
if ( tab4 )
|
|
|
|
{
|
2014-05-10 21:02:31 +00:00
|
|
|
GD_DPRINTF( "Warning: too many tabs present, skipping: %s\n", buf );
|
2013-06-16 17:47:04 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Handle the forth entry, if it exists. From dictfmt man:
|
|
|
|
// When --index-keep-orig option is used fourth column is created
|
|
|
|
// (if necessary) in .index file.
|
|
|
|
indexedWords.addWord( Utf8::decode( string( tab3 + 1, strlen ( tab3 + 1 ) ) ), curOffset );
|
|
|
|
++idxHeader.wordCount;
|
|
|
|
}
|
|
|
|
indexedWords.addWord( Utf8::decode( string( buf, strchr( buf, '\t' ) - buf ) ), curOffset );
|
|
|
|
++idxHeader.wordCount;
|
|
|
|
++idxHeader.articleCount;
|
|
|
|
|
|
|
|
// Check for proper dictionary name
|
|
|
|
if ( !strncmp( buf, "00databaseshort", 15 ) || !strncmp( buf, "00-database-short", 17 ) )
|
2013-03-13 09:45:11 +00:00
|
|
|
{
|
|
|
|
// After tab1 should be article offset, after tab2 -- article size
|
|
|
|
uint32_t articleOffset = decodeBase64( string( tab1 + 1, tab2 - tab1 - 1 ) );
|
|
|
|
uint32_t articleSize = decodeBase64( tab2 + 1 );
|
2014-04-24 18:50:47 +00:00
|
|
|
|
|
|
|
DZ_ERRORS error;
|
|
|
|
dictData * dz = dict_data_open( dictFiles[ 1 ].c_str(), &error, 0 );
|
2013-03-13 09:45:11 +00:00
|
|
|
|
|
|
|
if ( dz )
|
|
|
|
{
|
|
|
|
char * articleBody = dict_data_read_( dz, articleOffset, articleSize, 0, 0 );
|
|
|
|
if ( articleBody )
|
|
|
|
{
|
2018-10-26 17:18:58 +00:00
|
|
|
char * eol;
|
|
|
|
if ( !strncmp( articleBody, "00databaseshort", 15 ) || !strncmp( articleBody, "00-database-short", 17 ) )
|
|
|
|
eol = strchr( articleBody, '\n' ); // skip the first line (headword itself)
|
|
|
|
else
|
|
|
|
eol = articleBody; // No headword itself
|
2013-03-13 09:45:11 +00:00
|
|
|
if ( eol )
|
|
|
|
{
|
2016-04-15 14:44:53 +00:00
|
|
|
while( *eol && Utf8::isspace( *eol ) ) ++eol; // skip spaces
|
2013-03-13 09:45:11 +00:00
|
|
|
|
|
|
|
// use only the single line for the dictionary title
|
|
|
|
char * endEol = strchr( eol, '\n' );
|
|
|
|
if ( endEol )
|
|
|
|
*endEol = 0;
|
|
|
|
|
2014-05-10 21:02:31 +00:00
|
|
|
GD_DPRINTF( "DICT NAME: '%s'\n", eol );
|
2013-03-13 09:45:11 +00:00
|
|
|
dictionaryName = eol;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
dict_data_close( dz );
|
|
|
|
}
|
2014-04-24 18:50:47 +00:00
|
|
|
else
|
|
|
|
throw exDictzipError( string( dz_error_str( error ) )
|
|
|
|
+ "(" + dictFiles[ 1 ] + ")" );
|
2013-03-13 09:45:11 +00:00
|
|
|
}
|
|
|
|
}
|
2013-06-16 17:47:04 +00:00
|
|
|
else
|
|
|
|
{
|
2014-05-10 21:02:31 +00:00
|
|
|
GD_DPRINTF( "Warning: only a single tab present, skipping: %s\n", buf );
|
2013-06-16 17:47:04 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2014-05-10 21:02:31 +00:00
|
|
|
GD_DPRINTF( "Warning: no tabs present, skipping: %s\n", buf );
|
2013-06-16 17:47:04 +00:00
|
|
|
continue;
|
2013-03-13 09:45:11 +00:00
|
|
|
}
|
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
} while( !indexFile.eof() );
|
|
|
|
|
2013-03-13 09:45:11 +00:00
|
|
|
|
|
|
|
// Write dictionary name
|
|
|
|
|
|
|
|
idx.write( (uint32_t) dictionaryName.size() );
|
|
|
|
idx.write( dictionaryName.data(), dictionaryName.size() );
|
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
// Build index
|
|
|
|
|
2009-04-14 16:35:47 +00:00
|
|
|
IndexInfo idxInfo = BtreeIndexing::buildIndex( indexedWords, idx );
|
|
|
|
|
|
|
|
idxHeader.indexBtreeMaxElements = idxInfo.btreeMaxElements;
|
|
|
|
idxHeader.indexRootOffset = idxInfo.rootOffset;
|
2009-04-09 18:50:49 +00:00
|
|
|
|
|
|
|
// That concludes it. Update the header.
|
|
|
|
|
|
|
|
idxHeader.signature = Signature;
|
|
|
|
idxHeader.formatVersion = CurrentFormatVersion;
|
|
|
|
|
2009-05-05 22:45:02 +00:00
|
|
|
// read languages
|
|
|
|
QPair<quint32,quint32> langs =
|
|
|
|
LangCoder::findIdsForFilename( QString::fromStdString( dictFiles[ 0 ] ) );
|
|
|
|
|
|
|
|
// if no languages found, try dictionary's name
|
|
|
|
if ( langs.first == 0 || langs.second == 0 )
|
|
|
|
{
|
|
|
|
langs =
|
|
|
|
LangCoder::findIdsForFilename( QString::fromStdString( nameFromFileName( dictFiles[ 0 ] ) ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
idxHeader.langFrom = langs.first;
|
|
|
|
idxHeader.langTo = langs.second;
|
|
|
|
|
2009-04-09 18:50:49 +00:00
|
|
|
idx.rewind();
|
|
|
|
|
|
|
|
idx.write( &idxHeader, sizeof( idxHeader ) );
|
|
|
|
}
|
|
|
|
|
2022-11-29 03:54:31 +00:00
|
|
|
dictionaries.push_back( std::make_shared<DictdDictionary>( dictId,
|
2009-04-09 18:50:49 +00:00
|
|
|
indexFile,
|
|
|
|
dictFiles ) );
|
|
|
|
}
|
|
|
|
catch( std::exception & e )
|
|
|
|
{
|
2013-11-16 18:34:09 +00:00
|
|
|
gdWarning( "Dictd dictionary \"%s\" reading failed, error: %s\n",
|
|
|
|
i->c_str(), e.what() );
|
2009-04-09 18:50:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return dictionaries;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|