diff --git a/article_maker.cc b/article_maker.cc
index e6b04075..db56ca2b 100644
--- a/article_maker.cc
+++ b/article_maker.cc
@@ -126,7 +126,7 @@ std::string ArticleMaker::makeHtmlHeader( QString const & word,
}
}
- result += "
" + Html::escape( Utf8::encode( gd::toWString( word ) ) ) + "";
+ result += "" + Html::escape( word.toStdString()) + "";
// This doesn't seem to be much of influence right now, but we'll keep
// it anyway.
diff --git a/bgl.cc b/bgl.cc
index bec116d6..ec11ba0b 100644
--- a/bgl.cc
+++ b/bgl.cc
@@ -178,20 +178,7 @@ namespace
}
// Convert the word from utf8 to wide chars
-
- if ( wcharBuffer.size() <= word.size() )
- wcharBuffer.resize( word.size() + 1 );
-
- long result = Utf8::decode( word.c_str(), word.size(),
- &wcharBuffer.front() );
-
- if ( result < 0 )
- {
- gdWarning( "Failed to decode utf8 of headword \"%s\", skipping it.", word.c_str() );
- return;
- }
-
- indexedWords.addWord( wstring( &wcharBuffer.front(), result ), articleOffset );
+ indexedWords.addWord( Utf8::decode( word ), articleOffset );
}
diff --git a/btreeidx.cc b/btreeidx.cc
index 16cb687c..645ae4e1 100644
--- a/btreeidx.cc
+++ b/btreeidx.cc
@@ -526,8 +526,8 @@ char const * BtreeIndex::findChainOffsetExactOrPrefix( wstring const & target,
// Lookup the index by traversing the index btree
- vector< wchar > wcharBuffer;
-
+ // vector< wchar > wcharBuffer;
+ wstring w_word;
exactMatch = false;
// Read a node
@@ -615,20 +615,10 @@ char const * BtreeIndex::findChainOffsetExactOrPrefix( wstring const & target,
--closestString;
size_t wordSize = strlen( closestString );
-
- if ( wcharBuffer.size() <= wordSize )
- wcharBuffer.resize( wordSize + 1 );
-
- long result = Utf8::decode( closestString, wordSize, &wcharBuffer.front() );
- if ( result < 0 )
- throw Utf8::exCantDecode( closestString );
-
- wcharBuffer[ result ] = 0;
+ w_word = Utf8::decode( string( closestString, wordSize ) );
- //GD_DPRINTF( "Checking against %s\n", closestString );
-
- compareResult = target.compare( &wcharBuffer.front() );
+ compareResult = target.compare( w_word);
if ( !compareResult )
{
@@ -749,22 +739,12 @@ char const * BtreeIndex::findChainOffsetExactOrPrefix( wstring const & target,
ptr += sizeof( uint32_t );
size_t wordSize = strlen( ptr );
+
+ w_word = Utf8::decode( string( ptr, wordSize ) );
- if ( wcharBuffer.size() <= wordSize )
- wcharBuffer.resize( wordSize + 1 );
-
- //GD_DPRINTF( "checking against word %s, left = %u\n", ptr, leafEntries );
-
- long result = Utf8::decode( ptr, wordSize, &wcharBuffer.front() );
-
- if ( result < 0 )
- throw Utf8::exCantDecode( ptr );
-
- wcharBuffer[ result ] = 0;
-
- wstring foldedWord = Folding::apply( &wcharBuffer.front() );
+ wstring foldedWord = Folding::apply( w_word );
if( foldedWord.empty() )
- foldedWord = Folding::applyWhitespaceOnly( &wcharBuffer.front() );
+ foldedWord = Folding::applyWhitespaceOnly( w_word );
int compareResult = target.compare( foldedWord );
@@ -1071,22 +1051,10 @@ void IndexedWords::addWord( wstring const & word, uint32_t articleOffset, unsign
// Safeguard us against various bugs here. Don't attempt adding words
// which are freakishly huge.
- if ( wordSize > maxHeadwordSize )
+ if( wordSize > maxHeadwordSize )
{
-#define MAX_LOG_WORD_SIZE 500
- string headword;
- if( wordSize <= MAX_LOG_WORD_SIZE )
- headword = Utf8::encode( word );
- else
- {
- std::vector< char > buffer( MAX_LOG_WORD_SIZE * 4 );
- headword = string( &buffer.front(),
- Utf8::encode( wordBegin, MAX_LOG_WORD_SIZE, &buffer.front() ) );
- headword += "...";
- }
- gdWarning( "Skipped too long headword: \"%s\"", headword.c_str() );
+ qWarning() << "Skipped too long headword: " << word.substr( 0, 100 ).c_str() << "size:" << wordSize;
return;
-#undef MAX_LOG_WORD_SIZE
}
// Skip any leading whitespace
@@ -1118,17 +1086,11 @@ void IndexedWords::addWord( wstring const & word, uint32_t articleOffset, unsign
wstring folded = Folding::applyWhitespaceOnly( wstring( wordBegin, wordSize ) );
if( !folded.empty() )
{
- iterator i = insert(
- IndexedWords::value_type(
- string( &utfBuffer.front(),
- Utf8::encode( folded.data(), folded.size(), &utfBuffer.front() ) ),
- vector< WordArticleLink >() ) ).first;
+ iterator i = insert( { Utf8::encode( folded ),
+ vector< WordArticleLink >() } )
+ .first;
- // Try to conserve memory somewhat -- slow insertions are ok
- i->second.reserve( i->second.size() + 1 );
-
- string utfWord( &utfBuffer.front(),
- Utf8::encode( wordBegin, wordSize, &utfBuffer.front() ) );
+ string utfWord=Utf8::encode( wstring(wordBegin, wordSize )) ;
string utfPrefix;
i->second.push_back( WordArticleLink( utfWord, articleOffset, utfPrefix ) );
}
@@ -1142,24 +1104,15 @@ void IndexedWords::addWord( wstring const & word, uint32_t articleOffset, unsign
// Insert this word
wstring folded = Folding::apply( nextChar );
-
- iterator i = insert(
- IndexedWords::value_type(
- string( &utfBuffer.front(),
- Utf8::encode( folded.data(), folded.size(), &utfBuffer.front() ) ),
- vector< WordArticleLink >() ) ).first;
- if ( ( i->second.size() < 1024 ) || ( nextChar == wordBegin ) ) // Don't overpopulate chains with middle matches
+ iterator i = insert( { Utf8::encode( folded ), vector< WordArticleLink >() } ).first;
+
+ if( ( i->second.size() < 1024 ) || ( nextChar == wordBegin ) ) // Don't overpopulate chains with middle matches
{
- // Try to conserve memory somewhat -- slow insertions are ok
- i->second.reserve( i->second.size() + 1 );
-
- string utfWord( &utfBuffer.front(),
- Utf8::encode( nextChar, wordSize - ( nextChar - wordBegin ), &utfBuffer.front() ) );
-
- string utfPrefix( &utfBuffer.front(),
- Utf8::encode( wordBegin, nextChar - wordBegin, &utfBuffer.front() ) );
-
+ string utfWord = Utf8::encode( wstring( nextChar, wordSize - ( nextChar - wordBegin ) ) );
+
+ string utfPrefix = Utf8::encode( wstring( wordBegin, nextChar - wordBegin ) );
+
i->second.push_back( WordArticleLink( utfWord, articleOffset, utfPrefix ) );
}