hashirama/goldendict-ng
hashirama/goldendict-ng
0
0
Fork 0
mirror of https://github.com/xiaoyifang/goldendict-ng.git synced 2024-12-18 07:24:07 +00:00
Code Issues Actions 2 Projects Releases Wiki Activity

feat: fulltext qtconcurrent worker return directly when canceled

Browse source 
concurrent index with segmentation lock
This commit is contained in:
Xiao YiFang 2022-06-18 14:48:24 +08:00
parent c8486424f6
commit 6caf4fa58c
2 changed files with 57 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
articleview.cc
View file

@ -548,7 +548,6 @@ void ArticleView::loadFinished( bool result )
return; return;
} }
if( cfg.preferences.autoScrollToTargetArticle ) if( cfg.preferences.autoScrollToTargetArticle )
{ {
QString const scrollTo = Utils::Url::queryItemValue( url, "scrollto" ); QString const scrollTo = Utils::Url::queryItemValue( url, "scrollto" );

75
ftshelpers.cc
View file

@ -220,6 +220,8 @@ bool parseSearchString( QString const & str, QStringList & indexWords,
return true; return true;
} }
//definition;
Mutex lockMutex;
void parseArticleForFts( uint32_t articleAddress, QString & articleText, void parseArticleForFts( uint32_t articleAddress, QString & articleText,
QMap< QString, QVector< uint32_t > > & words, QMap< QString, QVector< uint32_t > > & words,
@ -232,7 +234,7 @@ void parseArticleForFts( uint32_t articleAddress, QString & articleText,
.split( handleRoundBrackets ? RX::Ftx::handleRoundBracket : RX::Ftx::noRoundBracket, .split( handleRoundBrackets ? RX::Ftx::handleRoundBracket : RX::Ftx::noRoundBracket,
Qt::SkipEmptyParts ); Qt::SkipEmptyParts );
QSet< QString > setOfWords; QVector< QString > setOfWords;
setOfWords.reserve( articleWords.size() ); setOfWords.reserve( articleWords.size() );
for( int x = 0; x < articleWords.size(); x++ ) for( int x = 0; x < articleWords.size(); x++ )
@ -253,10 +255,11 @@ void parseArticleForFts( uint32_t articleAddress, QString & articleText,
&& QChar( word[ y + 1 ] ).isLowSurrogate() ) && QChar( word[ y + 1 ] ).isLowSurrogate() )
hieroglyph.append( word[ ++y ] ); hieroglyph.append( word[ ++y ] );
if( !setOfWords.contains( hieroglyph ) ) //if( !setOfWords.contains( hieroglyph ) )
{ {
setOfWords.insert( hieroglyph ); setOfWords.push_back( hieroglyph );
words[ hieroglyph ].push_back( articleAddress ); /*Mutex::Lock _( _mapLock );
words[ hieroglyph ].push_back( articleAddress );*/
} }
hieroglyph.clear(); hieroglyph.clear();
@ -302,20 +305,33 @@ void parseArticleForFts( uint32_t articleAddress, QString & articleText,
for( QStringList::iterator it = list.begin(); it != list.end(); ++it ) for( QStringList::iterator it = list.begin(); it != list.end(); ++it )
{ {
if( !setOfWords.contains( *it ) ) //if( !setOfWords.contains( *it ) )
{ {
setOfWords.insert( *it ); setOfWords.push_back( *it );
words[ *it ].push_back( articleAddress ); /*Mutex::Lock _( _mapLock );
words[ *it ].push_back( articleAddress );*/
} }
} }
} }
else else
if( !setOfWords.contains( word ) ) //if( !setOfWords.contains( word ) )
{ {
setOfWords.insert( word ); setOfWords.push_back( word );
words[ word ].push_back( articleAddress ); /*Mutex::Lock _( _mapLock );
words[ word ].push_back( articleAddress );*/
} }
} }
}
{
Mutex::Lock _( lockMutex );
for( const QString & word : setOfWords )
{
words[ word ].push_back( articleAddress );
}
} }
} }
@ -375,22 +391,43 @@ void makeFTSIndex( BtreeIndexing::BtreeDictionary * dict, QAtomicInt & isCancell
needHandleBrackets = name.endsWith( ".dsl" ) || name.endsWith( "dsl.dz" ); needHandleBrackets = name.endsWith( ".dsl" ) || name.endsWith( "dsl.dz" );
} }
// index articles for full-text search QSemaphore sem( QThread::idealThreadCount() );
for( int i = 0; i < offsets.size(); i++ ) //QFutureSynchronizer< void > synchronizer;
for( auto & address : offsets )
{ {
if( Utils::AtomicInt::loadAcquire( isCancelled ) ) if( Utils::AtomicInt::loadAcquire( isCancelled ) )
throw exUserAbort(); {
//wait the future to be finished.
sem.acquire( QThread::idealThreadCount() );
return;
}
sem.acquire();
QFuture< void > f = QtConcurrent::run(
[ & ]()
{
QSemaphoreReleaser releaser( sem );
if( Utils::AtomicInt::loadAcquire( isCancelled ) )
{
return;
}
QString headword, articleStr; QString headword, articleStr;
dict->getArticleText( offsets.at( i ), headword, articleStr ); dict->getArticleText( address, headword, articleStr );
parseArticleForFts( offsets.at( i ), articleStr, ftsWords, needHandleBrackets ); parseArticleForFts( address, articleStr, ftsWords, needHandleBrackets );
} );
//synchronizer.addFuture( f );
} }
sem.acquire( QThread::idealThreadCount() );
// Free memory // Free memory
offsets.clear(); offsets.clear();
if( Utils::AtomicInt::loadAcquire( isCancelled ) )
throw exUserAbort();
QMap< QString, QVector< uint32_t > >::iterator it = ftsWords.begin(); QMap< QString, QVector< uint32_t > >::iterator it = ftsWords.begin();
while( it != ftsWords.end() ) while( it != ftsWords.end() )
{ {
@ -822,8 +859,8 @@ void FTSResultsRequest::combinedIndexSearch( BtreeIndexing::BtreeIndex & ftsInde
linksPtr = chunks->getBlock( links[ x ].articleOffset, chunk ); linksPtr = chunks->getBlock( links[ x ].articleOffset, chunk );
} }
memcpy( &size, linksPtr, sizeof(uint32_t) ); memcpy( &size, linksPtr, sizeof( uint32_t ) );
linksPtr += sizeof(uint32_t); linksPtr += sizeof( uint32_t );
// across chunks, need further investigation // across chunks, need further investigation
uint32_t max = ( chunk.size() - ( linksPtr - &chunk.front() )) / 4; uint32_t max = ( chunk.size() - ( linksPtr - &chunk.front() )) / 4;
@ -832,7 +869,7 @@ void FTSResultsRequest::combinedIndexSearch( BtreeIndexing::BtreeIndex & ftsInde
for( uint32_t y = 0; y < q_max; y++ ) for( uint32_t y = 0; y < q_max; y++ )
{ {
tmp.insert( *( reinterpret_cast< uint32_t * >( linksPtr ) ) ); tmp.insert( *( reinterpret_cast< uint32_t * >( linksPtr ) ) );
linksPtr += sizeof(uint32_t); linksPtr += sizeof( uint32_t );
} }
} }