From e333a03d7cc28822a76f7fbea2c8a8158d64ecfb Mon Sep 17 00:00:00 2001 From: Abs62 Date: Tue, 6 Sep 2022 17:40:22 +0300 Subject: [PATCH 1/2] Full-text search: Use separate thread pool for search requests --- aard.cc | 8 +++++--- bgl.cc | 8 +++++--- dictdfiles.cc | 8 +++++--- dictionary.cc | 2 +- dictionary.hh | 4 +++- dsl.cc | 8 +++++--- epwing.cc | 8 +++++--- ftshelpers.cc | 6 ++++++ ftshelpers.hh | 4 ++-- fulltextsearch.cc | 11 +++++++++-- fulltextsearch.hh | 2 ++ gls.cc | 8 +++++--- mdx.cc | 8 +++++--- sdict.cc | 8 +++++--- slob.cc | 8 +++++--- stardict.cc | 8 +++++--- xdxf.cc | 8 +++++--- zim.cc | 8 +++++--- 18 files changed, 83 insertions(+), 42 deletions(-) diff --git a/aard.cc b/aard.cc index 6d5a2871..bd704199 100644 --- a/aard.cc +++ b/aard.cc @@ -274,7 +274,8 @@ class AardDictionary: public BtreeIndexing::BtreeDictionary int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -657,9 +658,10 @@ sptr< Dictionary::DataRequest > AardDictionary::getSearchResults( QString const int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } /// AardDictionary::getArticle() diff --git a/bgl.cc b/bgl.cc index 09921bd8..ab9dd89f 100644 --- a/bgl.cc +++ b/bgl.cc @@ -244,7 +244,8 @@ namespace int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual QString const& getDescription(); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); @@ -1211,9 +1212,10 @@ sptr< Dictionary::DataRequest > BglDictionary::getSearchResults( QString const & int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } diff --git a/dictdfiles.cc b/dictdfiles.cc index d633c23d..f567cf51 100644 --- a/dictdfiles.cc +++ b/dictdfiles.cc @@ -135,7 +135,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -599,9 +600,10 @@ sptr< Dictionary::DataRequest > DictdDictionary::getSearchResults( QString const int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } } // anonymous namespace diff --git a/dictionary.cc b/dictionary.cc index 3d13508c..8afc7600 100644 --- a/dictionary.cc +++ b/dictionary.cc @@ -175,7 +175,7 @@ sptr< DataRequest > Class::getResource( string const & /*name*/ ) return new DataRequestInstant( false ); } -sptr< DataRequest > Class::getSearchResults(const QString &, int, bool, int, int, bool, bool ) +sptr< DataRequest > Class::getSearchResults(const QString &, int, bool, int, int, bool, bool, QThreadPool * ) { return new DataRequestInstant( false ); } diff --git a/dictionary.hh b/dictionary.hh index 93f99258..53cfde32 100644 --- a/dictionary.hh +++ b/dictionary.hh @@ -9,6 +9,7 @@ #include #include #include +#include #include "cpp_features.hh" #include "sptr.hh" #include "ex.hh" @@ -400,7 +401,8 @@ public: int distanceBetweenWords, int maxArticlesPerDictionary, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); // Return dictionary description if presented virtual QString const& getDescription(); diff --git a/dsl.cc b/dsl.cc index 3652544d..1ec5bb6b 100644 --- a/dsl.cc +++ b/dsl.cc @@ -236,7 +236,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual QString const& getDescription(); virtual QString getMainFilename(); @@ -2047,9 +2048,10 @@ sptr< Dictionary::DataRequest > DslDictionary::getSearchResults( QString const & int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } } // anonymous namespace diff --git a/epwing.cc b/epwing.cc index dfc14949..2d73fa14 100644 --- a/epwing.cc +++ b/epwing.cc @@ -130,7 +130,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -796,9 +797,10 @@ sptr< Dictionary::DataRequest > EpwingDictionary::getSearchResults( QString cons int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } int EpwingDictionary::japaneseWriting( gd::wchar ch ) diff --git a/ftshelpers.cc b/ftshelpers.cc index 8d3e4052..0a97a81a 100644 --- a/ftshelpers.cc +++ b/ftshelpers.cc @@ -1149,6 +1149,12 @@ void FTSResultsRequest::fullSearch( QStringList & searchWords, QRegExp & regexp void FTSResultsRequest::run() { + if( Qt4x5::AtomicInt::loadAcquire( isCancelled ) ) + { + finish(); + return; + } + if ( dict.ensureInitDone().size() ) { setErrorString( QString::fromUtf8( dict.ensureInitDone().c_str() ) ); diff --git a/ftshelpers.hh b/ftshelpers.hh index 0c3530aa..18802f13 100644 --- a/ftshelpers.hh +++ b/ftshelpers.hh @@ -129,7 +129,7 @@ public: FTSResultsRequest( BtreeIndexing::BtreeDictionary & dict_, QString const & searchString_, int searchMode_, bool matchCase_, int distanceBetweenWords_, int maxResults_, - bool ignoreWordsOrder_, bool ignoreDiacritics_ ): + bool ignoreWordsOrder_, bool ignoreDiacritics_, QThreadPool * ftsThreadPoolPtr ): dict( dict_ ), searchString( searchString_ ), searchMode( searchMode_ ), @@ -145,7 +145,7 @@ public: searchString = gd::toQString( Folding::applyDiacriticsOnly( gd::toWString( searchString_ ) ) ); foundHeadwords = new QList< FTS::FtsHeadword >; - QThreadPool::globalInstance()->start( + ftsThreadPoolPtr->start( new FTSResultsRequestRunnable( *this, hasExited ), -100 ); } diff --git a/fulltextsearch.cc b/fulltextsearch.cc index 66bdc343..48917c37 100644 --- a/fulltextsearch.cc +++ b/fulltextsearch.cc @@ -138,11 +138,15 @@ FullTextSearchDialog::FullTextSearchDialog( QWidget * parent, group( 0 ), ignoreWordsOrder( cfg_.preferences.fts.ignoreWordsOrder ), ignoreDiacritics( cfg_.preferences.fts.ignoreDiacritics ), + ftsThreadPool( this ), + searchInProgress( false ), ftsIdx( ftsidx ) , helpAction( this ) { ui.setupUi( this ); + ftsThreadPool.setExpiryTimeout( -1 ); + setAttribute( Qt::WA_DeleteOnClose, false ); setWindowFlags( windowFlags() & ~Qt::WindowContextHelpButtonHint ); @@ -429,11 +433,13 @@ void FullTextSearchDialog::accept() distanceBetweenWords, maxResultsPerDict, ignoreWordsOrder, - ignoreDiacritics + ignoreDiacritics, + &ftsThreadPool ); connect( req.get(), SIGNAL( finished() ), this, SLOT( searchReqFinished() ), Qt::QueuedConnection ); + searchInProgress = true; searchReqs.push_back( req ); } @@ -486,8 +492,9 @@ void FullTextSearchDialog::searchReqFinished() else break; } - if ( searchReqs.empty() ) + if ( searchReqs.empty() && searchInProgress ) { + searchInProgress = false; ui.searchProgressBar->hide(); ui.OKButton->setEnabled( true ); QApplication::beep(); diff --git a/fulltextsearch.hh b/fulltextsearch.hh index 9faae720..38f9a85f 100644 --- a/fulltextsearch.hh +++ b/fulltextsearch.hh @@ -177,6 +177,8 @@ class FullTextSearchDialog : public QDialog std::vector< sptr< Dictionary::Class > > activeDicts; bool ignoreWordsOrder; bool ignoreDiacritics; + QThreadPool ftsThreadPool; + bool searchInProgress; std::list< sptr< Dictionary::DataRequest > > searchReqs; diff --git a/gls.cc b/gls.cc index c66a3085..ac87ebe5 100644 --- a/gls.cc +++ b/gls.cc @@ -489,7 +489,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); @@ -1559,9 +1560,10 @@ sptr< Dictionary::DataRequest > GlsDictionary::getSearchResults( QString const & int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } } // anonymous namespace diff --git a/mdx.cc b/mdx.cc index 4a4edfc2..0f815ca4 100644 --- a/mdx.cc +++ b/mdx.cc @@ -257,7 +257,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -536,9 +537,10 @@ sptr< Dictionary::DataRequest > MdxDictionary::getSearchResults( QString const & int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } /// MdxDictionary::getArticle diff --git a/sdict.cc b/sdict.cc index 4bbda718..6c788d01 100644 --- a/sdict.cc +++ b/sdict.cc @@ -174,7 +174,8 @@ class SdictDictionary: public BtreeIndexing::BtreeDictionary int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -483,9 +484,10 @@ sptr< Dictionary::DataRequest > SdictDictionary::getSearchResults( QString const int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } /// SdictDictionary::getArticle() diff --git a/slob.cc b/slob.cc index a2015346..94093669 100644 --- a/slob.cc +++ b/slob.cc @@ -623,7 +623,8 @@ class SlobDictionary: public BtreeIndexing::BtreeDictionary int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); quint64 getArticlePos(uint32_t articleNumber ); @@ -1344,9 +1345,10 @@ sptr< Dictionary::DataRequest > SlobDictionary::getSearchResults( QString const int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString, searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString, searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } diff --git a/stardict.cc b/stardict.cc index 9b713fff..aabece31 100644 --- a/stardict.cc +++ b/stardict.cc @@ -192,7 +192,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -1270,9 +1271,10 @@ sptr< Dictionary::DataRequest > StardictDictionary::getSearchResults( QString co int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } /// StardictDictionary::findHeadwordsForSynonym() diff --git a/xdxf.cc b/xdxf.cc index 8cef3ab9..09cedee6 100644 --- a/xdxf.cc +++ b/xdxf.cc @@ -181,7 +181,8 @@ public: int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); virtual void makeFTSIndex(QAtomicInt & isCancelled, bool firstIteration ); @@ -426,9 +427,10 @@ sptr< Dictionary::DataRequest > XdxfDictionary::getSearchResults( QString const int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } /// XdxfDictionary::getArticle() diff --git a/zim.cc b/zim.cc index 599ea9ea..d0394729 100644 --- a/zim.cc +++ b/zim.cc @@ -708,7 +708,8 @@ class ZimDictionary: public BtreeIndexing::BtreeDictionary int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ); + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ); virtual void getArticleText( uint32_t articleAddress, QString & headword, QString & text ); quint32 getArticleText( uint32_t articleAddress, QString & headword, QString & text, @@ -1288,9 +1289,10 @@ sptr< Dictionary::DataRequest > ZimDictionary::getSearchResults( QString const & int distanceBetweenWords, int maxResults, bool ignoreWordsOrder, - bool ignoreDiacritics ) + bool ignoreDiacritics, + QThreadPool * ftsThreadPoolPtr ) { - return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics ); + return new FtsHelpers::FTSResultsRequest( *this, searchString,searchMode, matchCase, distanceBetweenWords, maxResults, ignoreWordsOrder, ignoreDiacritics, ftsThreadPoolPtr ); } /// ZimDictionary::getArticle() From 3b2df81b3d8372d2d19fb2481b4710a316dd93df Mon Sep 17 00:00:00 2001 From: Abs62 Date: Wed, 7 Sep 2022 17:35:52 +0300 Subject: [PATCH 2/2] Full-text search: Reduce max thread count --- fulltextsearch.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fulltextsearch.cc b/fulltextsearch.cc index 48917c37..0ccaab6e 100644 --- a/fulltextsearch.cc +++ b/fulltextsearch.cc @@ -146,6 +146,9 @@ FullTextSearchDialog::FullTextSearchDialog( QWidget * parent, ui.setupUi( this ); ftsThreadPool.setExpiryTimeout( -1 ); + int threads = ftsThreadPool.maxThreadCount(); + if( threads > 1 ) + ftsThreadPool.setMaxThreadCount( threads - 1 ); setAttribute( Qt::WA_DeleteOnClose, false ); setWindowFlags( windowFlags() & ~Qt::WindowContextHelpButtonHint );