diff --git a/dsl.cc b/dsl.cc index 141a0159..da94f755 100644 --- a/dsl.cc +++ b/dsl.cc @@ -2182,7 +2182,7 @@ vector< sptr< Dictionary::Class > > makeDictionaries( for( ; ; ) { // Skip any whitespace - if ( !abrvScanner.readNextLineWithoutComments( curString, curOffset ) ) + if ( !abrvScanner.readNextLineWithoutComments( curString, curOffset, true ) ) break; if ( curString.empty() || isDslWs( curString[ 0 ] ) ) continue; @@ -2270,7 +2270,7 @@ vector< sptr< Dictionary::Class > > makeDictionaries( { // Find the main headword - if ( !hasString && !scanner.readNextLineWithoutComments( curString, curOffset ) ) + if ( !hasString && !scanner.readNextLineWithoutComments( curString, curOffset, true) ) break; // Clean end of file hasString = false; @@ -2368,7 +2368,8 @@ vector< sptr< Dictionary::Class > > makeDictionaries( // Skip the article's body for( ; ; ) { - hasString = haveLine ? true : scanner.readNextLineWithoutComments( curString, curOffset ); + //todo ,return headword only? + hasString = haveLine ? true : scanner.readNextLineWithoutComments( curString, curOffset); haveLine = false; if ( !hasString || ( curString.size() && !isDslWs( curString[ 0 ] ) ) ) diff --git a/dsl_details.cc b/dsl_details.cc index 151cb007..d7c5d7d9 100644 --- a/dsl_details.cc +++ b/dsl_details.cc @@ -892,7 +892,6 @@ DslScanner::DslScanner( string const & fileName ) THROW_SPEC( Ex, Iconv::Ex ): iconv.reinit( encoding ); codec=QTextCodec::codecForName(iconv.getEncodingNameFor(encoding)); - // We now can use our own readNextLine() function wstring str; @@ -996,15 +995,15 @@ DslScanner::~DslScanner() throw() gzclose( f ); } -bool DslScanner::readNextLine( wstring & out, size_t & offset ) THROW_SPEC( Ex, +bool DslScanner::readNextLine( wstring & out, size_t & offset, bool only_head_word ) THROW_SPEC( Ex, Iconv::Ex ) { offset = (size_t)( gztell( f ) - readBufferLeft+pos ); - for( ; ; ) + for(;;) { // Check that we have bytes to read - if ( readBufferLeft-pos < 1000 ) + if ( readBufferLeft-pos < 2000 ) { readBufferPtr+=pos; readBufferLeft-=pos; @@ -1024,31 +1023,30 @@ bool DslScanner::readNextLine( wstring & out, size_t & offset ) THROW_SPEC( Ex, readBufferPtr = readBuffer; readBufferLeft += (size_t) result; QByteArray frag = QByteArray::fromRawData(readBuffer, readBufferLeft); - //QTextStream in(frag); fragStream = new QTextStream(frag) ; fragStream->setCodec(codec); } } - - //QByteArray frag=QByteArray::fromRawData(readBuffer,readBufferLeft); if(fragStream->atEnd()) return false; QString line=fragStream->readLine(); pos = fragStream->pos(); -// qint64 pos= fragStream.pos(); -// readBufferPtr+=pos; -// readBufferLeft-=pos; linesRead++; + if(only_head_word &&( line.isEmpty()||line.at(0).isSpace())) + continue; +#ifdef __WIN32 out=line.toStdU32String(); - //frag.remove(0, pos); +#else + out=line.toStdWString(); +#endif return true; } } -bool DslScanner::readNextLineWithoutComments( wstring & out, size_t & offset ) +bool DslScanner::readNextLineWithoutComments( wstring & out, size_t & offset , bool only_headword) THROW_SPEC( Ex, Iconv::Ex ) { wstring str; @@ -1060,7 +1058,7 @@ bool DslScanner::readNextLineWithoutComments( wstring & out, size_t & offset ) do { - bool b = readNextLine( str, currentOffset ); + bool b = readNextLine( str, currentOffset, only_headword ); if( offset == 0 ) offset = currentOffset; @@ -1256,8 +1254,9 @@ void expandOptionalParts( wstring & str, list< wstring > * result, { if( !inside_recurse ) { - expanded.sort(); - result->merge(expanded ); +// expanded.sort(); +// result->merge(expanded ); + result->splice(result->end(),expanded); } return; } @@ -1284,9 +1283,7 @@ void expandOptionalParts( wstring & str, list< wstring > * result, headwords->push_back( str ); if (!inside_recurse) { - result->sort(); - expanded.sort(); - result->merge(expanded); + result->splice(result->end(),expanded); } } diff --git a/dsl_details.hh b/dsl_details.hh index dc65a9d2..4bc3cede 100644 --- a/dsl_details.hh +++ b/dsl_details.hh @@ -126,7 +126,7 @@ class DslScanner wstring dictionaryName; wstring langFrom, langTo; wstring soundDictionary; - char readBuffer[ 3000 ]; + char readBuffer[ 10000 ]; QTextStream* fragStream; char * readBufferPtr; size_t readBufferLeft; @@ -172,10 +172,10 @@ public: /// If end of file is reached, false is returned. /// Reading begins from the first line after the headers (ones which start /// with #). - bool readNextLine( wstring &, size_t & offset ) THROW_SPEC( Ex, Iconv::Ex ); + bool readNextLine( wstring &, size_t & offset, bool only_head_word = false ) THROW_SPEC( Ex, Iconv::Ex ); /// Similar readNextLine but strip all DSL comments {{...}} - bool readNextLineWithoutComments( wstring &, size_t & offset ) THROW_SPEC( Ex, Iconv::Ex ); + bool readNextLineWithoutComments( wstring &, size_t & offset, bool only_headword = false ) THROW_SPEC( Ex, Iconv::Ex ); /// Returns the number of lines read so far from the file. unsigned getLinesRead() const diff --git a/wstring.hh b/wstring.hh index 4d2bb463..9741b522 100644 --- a/wstring.hh +++ b/wstring.hh @@ -43,17 +43,18 @@ namespace gd { - typedef char32_t wchar; - typedef std::u32string wstring; - #ifdef __WIN32 + #ifdef __WIN32 + typedef char32_t wchar; + typedef std::u32string wstring; // GD_NATIVE_TO_WS is used to convert L"" strings to a const pointer to // wchar. wstring __nativeToWs( wchar_t const * ); #define GD_NATIVE_TO_WS( str ) ( gd::__nativeToWs( ( str ) ).c_str() ) #else - + typedef wchar_t wchar; + typedef std::basic_string wstring; #define GD_NATIVE_TO_WS( str ) ( str ) #endif } diff --git a/wstring_qt.cc b/wstring_qt.cc index 99de873a..c7ff8b8d 100644 --- a/wstring_qt.cc +++ b/wstring_qt.cc @@ -7,7 +7,11 @@ namespace gd QString toQString( wstring const & in ) { +#ifdef __WIN32 return QString::fromUcs4( in.c_str() ); +#else + return QString::fromStdWString(in); +#endif } wstring toWString( QString const & in )