mirror of
https://github.com/xiaoyifang/goldendict-ng.git
synced 2024-11-27 19:24:08 +00:00
babde3d53d
wchar_t. This changes nothing on Linux and most other systems, but on Win32 it causes to use normal UCS-4 strings instead of Win32's usual UTF-16.
532 lines
18 KiB
C++
532 lines
18 KiB
C++
/* This file is (c) 2008-2009 Konstantin Isakov <ikm@users.berlios.de>
|
|
* Part of GoldenDict. Licensed under GPLv3 or later, see the LICENSE file */
|
|
|
|
#include "folding.hh"
|
|
|
|
namespace Folding {
|
|
|
|
namespace
|
|
{
|
|
#include "inc_case_folding.hh"
|
|
#include "inc_diacritic_folding.hh"
|
|
}
|
|
|
|
wstring apply( wstring const & in )
|
|
{
|
|
// First, strip diacritics and apply ws/punctuation removal
|
|
|
|
wstring withoutDiacritics;
|
|
|
|
withoutDiacritics.reserve( in.size() );
|
|
|
|
wchar const * nextChar = in.data();
|
|
|
|
size_t consumed;
|
|
|
|
for( size_t left = in.size(); left; )
|
|
{
|
|
wchar ch = foldDiacritic( nextChar, left, consumed );
|
|
|
|
if ( !isWhitespace( ch ) && !isPunct( ch ) )
|
|
withoutDiacritics.push_back( ch );
|
|
|
|
nextChar += consumed;
|
|
left -= consumed;
|
|
}
|
|
|
|
// Now, fold the case
|
|
|
|
wstring caseFolded;
|
|
|
|
caseFolded.reserve( withoutDiacritics.size() * foldCaseMaxOut );
|
|
|
|
nextChar = withoutDiacritics.data();
|
|
|
|
wchar buf[ foldCaseMaxOut ];
|
|
|
|
for( size_t left = withoutDiacritics.size(); left--; )
|
|
caseFolded.append( buf, foldCase( *nextChar++, buf ) );
|
|
|
|
return caseFolded;
|
|
}
|
|
|
|
wstring applySimpleCaseOnly( wstring const & in )
|
|
{
|
|
wchar const * nextChar = in.data();
|
|
|
|
wstring out;
|
|
|
|
out.reserve( in.size() );
|
|
|
|
for( size_t left = in.size(); left--; )
|
|
out.push_back( foldCaseSimple( *nextChar++ ) );
|
|
|
|
return out;
|
|
}
|
|
|
|
wstring applyFullCaseOnly( wstring const & in )
|
|
{
|
|
wstring caseFolded;
|
|
|
|
caseFolded.reserve( in.size() * foldCaseMaxOut );
|
|
|
|
wchar const * nextChar = in.data();
|
|
|
|
wchar buf[ foldCaseMaxOut ];
|
|
|
|
for( size_t left = in.size(); left--; )
|
|
caseFolded.append( buf, foldCase( *nextChar++, buf ) );
|
|
|
|
return caseFolded;
|
|
}
|
|
|
|
wstring applyDiacriticsOnly( wstring const & in )
|
|
{
|
|
wstring withoutDiacritics;
|
|
|
|
withoutDiacritics.reserve( in.size() );
|
|
|
|
wchar const * nextChar = in.data();
|
|
|
|
size_t consumed;
|
|
|
|
for( size_t left = in.size(); left; )
|
|
{
|
|
wchar ch = foldDiacritic( nextChar, left, consumed );
|
|
|
|
withoutDiacritics.push_back( ch );
|
|
|
|
nextChar += consumed;
|
|
left -= consumed;
|
|
}
|
|
|
|
return withoutDiacritics;
|
|
}
|
|
|
|
wstring applyPunctOnly( wstring const & in )
|
|
{
|
|
wchar const * nextChar = in.data();
|
|
|
|
wstring out;
|
|
|
|
out.reserve( in.size() );
|
|
|
|
for( size_t left = in.size(); left--; ++nextChar )
|
|
if ( !isPunct( *nextChar ) )
|
|
out.push_back( *nextChar );
|
|
|
|
return out;
|
|
}
|
|
|
|
wstring applyWhitespaceOnly( wstring const & in )
|
|
{
|
|
wchar const * nextChar = in.data();
|
|
|
|
wstring out;
|
|
|
|
out.reserve( in.size() );
|
|
|
|
for( size_t left = in.size(); left--; ++nextChar )
|
|
if ( !isWhitespace( *nextChar ) )
|
|
out.push_back( *nextChar );
|
|
|
|
return out;
|
|
}
|
|
|
|
wstring applyWhitespaceAndPunctOnly( wstring const & in )
|
|
{
|
|
wchar const * nextChar = in.data();
|
|
|
|
wstring out;
|
|
|
|
out.reserve( in.size() );
|
|
|
|
for( size_t left = in.size(); left--; ++nextChar )
|
|
if ( !isWhitespace( *nextChar ) && !isPunct( *nextChar ) )
|
|
out.push_back( *nextChar );
|
|
|
|
return out;
|
|
}
|
|
|
|
bool isWhitespace( wchar ch )
|
|
{
|
|
switch( ch )
|
|
{
|
|
case '\n':
|
|
case '\r':
|
|
case '\t':
|
|
|
|
case 0x2028: // Zl, LINE SEPARATOR
|
|
|
|
case 0x2029: // Zp, PARAGRAPH SEPARATOR
|
|
|
|
case 0x0020: // Zs, SPACE
|
|
case 0x00A0: // Zs, NO-BREAK SPACE
|
|
case 0x1680: // Zs, OGHAM SPACE MARK
|
|
case 0x180E: // Zs, MONGOLIAN VOWEL SEPARATOR
|
|
case 0x2000: // Zs, EN QUAD
|
|
case 0x2001: // Zs, EM QUAD
|
|
case 0x2002: // Zs, EN SPACE
|
|
case 0x2003: // Zs, EM SPACE
|
|
case 0x2004: // Zs, THREE-PER-EM SPACE
|
|
case 0x2005: // Zs, FOUR-PER-EM SPACE
|
|
case 0x2006: // Zs, SIX-PER-EM SPACE
|
|
case 0x2007: // Zs, FIGURE SPACE
|
|
case 0x2008: // Zs, PUNCTUATION SPACE
|
|
case 0x2009: // Zs, THIN SPACE
|
|
case 0x200A: // Zs, HAIR SPACE
|
|
case 0x202F: // Zs, NARROW NO-BREAK SPACE
|
|
case 0x205F: // Zs, MEDIUM MATHEMATICAL SPACE
|
|
case 0x3000: // Zs, IDEOGRAPHIC SPACE
|
|
return true;
|
|
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool isPunct( wchar ch )
|
|
{
|
|
switch( ch )
|
|
{
|
|
// Pc
|
|
|
|
case 0x005F: // LOW LINE
|
|
case 0x203F: // UNDERTIE
|
|
case 0x2040: // CHARACTER TIE
|
|
case 0x2054: // INVERTED UNDERTIE
|
|
case 0x30FB: // KATAKANA MIDDLE DOT
|
|
case 0xFE33: // PRESENTATION FORM FOR VERTICAL LOW LINE
|
|
case 0xFE34: // PRESENTATION FORM FOR VERTICAL WAVY LOW LINE
|
|
case 0xFE4D: // DASHED LOW LINE
|
|
case 0xFE4E: // CENTRELINE LOW LINE
|
|
case 0xFE4F: // WAVY LOW LINE
|
|
case 0xFF3F: // FULLWIDTH LOW LINE
|
|
case 0xFF65: // HALFWIDTH KATAKANA MIDDLE DOT
|
|
|
|
// Pd
|
|
case 0x002D: // HYPHEN-MINUS
|
|
case 0x058A: // ARMENIAN HYPHEN
|
|
case 0x1806: // MONGOLIAN TODO SOFT HYPHEN
|
|
case 0x2010: // HYPHEN
|
|
case 0x2011: // NON-BREAKING HYPHEN
|
|
case 0x2012: // FIGURE DASH
|
|
case 0x2013: // EN DASH
|
|
case 0x2014: // EM DASH
|
|
case 0x2015: // HORIZONTAL BAR
|
|
case 0x301C: // WAVE DASH
|
|
case 0x3030: // WAVY DASH
|
|
case 0x30A0: // KATAKANA-HIRAGANA DOUBLE HYPHEN
|
|
case 0xFE31: // PRESENTATION FORM FOR VERTICAL EM DASH
|
|
case 0xFE32: // PRESENTATION FORM FOR VERTICAL EN DASH
|
|
case 0xFE58: // SMALL EM DASH
|
|
case 0xFE63: // SMALL HYPHEN-MINUS
|
|
case 0xFF0D: // FULLWIDTH HYPHEN-MINUS
|
|
|
|
// Pe
|
|
case 0x0029: // RIGHT PARENTHESIS
|
|
case 0x005D: // RIGHT SQUARE BRACKET
|
|
case 0x007D: // RIGHT CURLY BRACKET
|
|
case 0x0F3B: // TIBETAN MARK GUG RTAGS GYAS
|
|
case 0x0F3D: // TIBETAN MARK ANG KHANG GYAS
|
|
case 0x169C: // OGHAM REVERSED FEATHER MARK
|
|
case 0x2046: // RIGHT SQUARE BRACKET WITH QUILL
|
|
case 0x207E: // SUPERSCRIPT RIGHT PARENTHESIS
|
|
case 0x208E: // SUBSCRIPT RIGHT PARENTHESIS
|
|
case 0x232A: // RIGHT-POINTING ANGLE BRACKET
|
|
case 0x23B5: // BOTTOM SQUARE BRACKET
|
|
case 0x2769: // MEDIUM RIGHT PARENTHESIS ORNAMENT
|
|
case 0x276B: // MEDIUM FLATTENED RIGHT PARENTHESIS ORNAMENT
|
|
case 0x276D: // MEDIUM RIGHT-POINTING ANGLE BRACKET ORNAMENT
|
|
case 0x276F: // HEAVY RIGHT-POINTING ANGLE QUOTATION MARK ORNAMENT
|
|
case 0x2771: // HEAVY RIGHT-POINTING ANGLE BRACKET ORNAMENT
|
|
case 0x2773: // LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT
|
|
case 0x2775: // MEDIUM RIGHT CURLY BRACKET ORNAMENT
|
|
case 0x27E7: // MATHEMATICAL RIGHT WHITE SQUARE BRACKET
|
|
case 0x27E9: // MATHEMATICAL RIGHT ANGLE BRACKET
|
|
case 0x27EB: // MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET
|
|
case 0x2984: // RIGHT WHITE CURLY BRACKET
|
|
case 0x2986: // RIGHT WHITE PARENTHESIS
|
|
case 0x2988: // Z NOTATION RIGHT IMAGE BRACKET
|
|
case 0x298A: // Z NOTATION RIGHT BINDING BRACKET
|
|
case 0x298C: // RIGHT SQUARE BRACKET WITH UNDERBAR
|
|
case 0x298E: // RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER
|
|
case 0x2990: // RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER
|
|
case 0x2992: // RIGHT ANGLE BRACKET WITH DOT
|
|
case 0x2994: // RIGHT ARC GREATER-THAN BRACKET
|
|
case 0x2996: // DOUBLE RIGHT ARC LESS-THAN BRACKET
|
|
case 0x2998: // RIGHT BLACK TORTOISE SHELL BRACKET
|
|
case 0x29D9: // RIGHT WIGGLY FENCE
|
|
case 0x29DB: // RIGHT DOUBLE WIGGLY FENCE
|
|
case 0x29FD: // RIGHT-POINTING CURVED ANGLE BRACKET
|
|
case 0x3009: // RIGHT ANGLE BRACKET
|
|
case 0x300B: // RIGHT DOUBLE ANGLE BRACKET
|
|
case 0x300D: // RIGHT CORNER BRACKET
|
|
case 0x300F: // RIGHT WHITE CORNER BRACKET
|
|
case 0x3011: // RIGHT BLACK LENTICULAR BRACKET
|
|
case 0x3015: // RIGHT TORTOISE SHELL BRACKET
|
|
case 0x3017: // RIGHT WHITE LENTICULAR BRACKET
|
|
case 0x3019: // RIGHT WHITE TORTOISE SHELL BRACKET
|
|
case 0x301B: // RIGHT WHITE SQUARE BRACKET
|
|
case 0x301E: // DOUBLE PRIME QUOTATION MARK
|
|
case 0x301F: // LOW DOUBLE PRIME QUOTATION MARK
|
|
case 0xFD3F: // ORNATE RIGHT PARENTHESIS
|
|
case 0xFE36: // PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS
|
|
case 0xFE38: // PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET
|
|
case 0xFE3A: // PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET
|
|
case 0xFE3C: // PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET
|
|
case 0xFE3E: // PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET
|
|
case 0xFE40: // PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET
|
|
case 0xFE42: // PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET
|
|
case 0xFE44: // PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET
|
|
case 0xFE48: // PRESENTATION FORM FOR VERTICAL RIGHT SQUARE BRACKET
|
|
case 0xFE5A: // SMALL RIGHT PARENTHESIS
|
|
case 0xFE5C: // SMALL RIGHT CURLY BRACKET
|
|
case 0xFE5E: // SMALL RIGHT TORTOISE SHELL BRACKET
|
|
case 0xFF09: // FULLWIDTH RIGHT PARENTHESIS
|
|
case 0xFF3D: // FULLWIDTH RIGHT SQUARE BRACKET
|
|
case 0xFF5D: // FULLWIDTH RIGHT CURLY BRACKET
|
|
case 0xFF60: // FULLWIDTH RIGHT WHITE PARENTHESIS
|
|
case 0xFF63: // HALFWIDTH RIGHT CORNER BRACKET
|
|
|
|
// Pf
|
|
case 0x00BB: // RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
|
|
case 0x2019: // RIGHT SINGLE QUOTATION MARK
|
|
case 0x201D: // RIGHT DOUBLE QUOTATION MARK
|
|
case 0x203A: // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
|
|
|
|
|
|
// Pi
|
|
case 0x00AB: // LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
|
|
case 0x2018: // LEFT SINGLE QUOTATION MARK
|
|
case 0x201C: // LEFT DOUBLE QUOTATION MARK
|
|
case 0x2039: // SINGLE LEFT-POINTING ANGLE QUOTATION MARK
|
|
|
|
// Po
|
|
case 0x0021: // EXCLAMATION MARK
|
|
case 0x0022: // QUOTATION MARK
|
|
case 0x0023: // NUMBER SIGN
|
|
case 0x0025: // PERCENT SIGN
|
|
case 0x0026: // AMPERSAND
|
|
case 0x0027: // APOSTROPHE
|
|
case 0x002A: // ASTERISK
|
|
case 0x002C: // COMMA
|
|
case 0x002E: // FULL STOP
|
|
case 0x002F: // SOLIDUS
|
|
case 0x003A: // COLON
|
|
case 0x003B: // SEMICOLON
|
|
case 0x003F: // QUESTION MARK
|
|
case 0x0040: // COMMERCIAL AT
|
|
case 0x005C: // REVERSE SOLIDUS
|
|
case 0x00A1: // INVERTED EXCLAMATION MARK
|
|
case 0x00B7: // MIDDLE DOT
|
|
case 0x00BF: // INVERTED QUESTION MARK
|
|
case 0x037E: // GREEK QUESTION MARK
|
|
case 0x0387: // GREEK ANO TELEIA
|
|
case 0x055A: // ARMENIAN APOSTROPHE
|
|
case 0x055B: // ARMENIAN EMPHASIS MARK
|
|
case 0x055C: // ARMENIAN EXCLAMATION MARK
|
|
case 0x055D: // ARMENIAN COMMA
|
|
case 0x055E: // ARMENIAN QUESTION MARK
|
|
case 0x055F: // ARMENIAN ABBREVIATION MARK
|
|
case 0x0589: // ARMENIAN FULL STOP
|
|
case 0x05BE: // HEBREW PUNCTUATION MAQAF
|
|
case 0x05C0: // HEBREW PUNCTUATION PASEQ
|
|
case 0x05C3: // HEBREW PUNCTUATION SOF PASUQ
|
|
case 0x05F3: // HEBREW PUNCTUATION GERESH
|
|
case 0x05F4: // HEBREW PUNCTUATION GERSHAYIM
|
|
case 0x060C: // ARABIC COMMA
|
|
case 0x060D: // ARABIC DATE SEPARATOR
|
|
case 0x061B: // ARABIC SEMICOLON
|
|
case 0x061F: // ARABIC QUESTION MARK
|
|
case 0x066A: // ARABIC PERCENT SIGN
|
|
case 0x066B: // ARABIC DECIMAL SEPARATOR
|
|
case 0x066C: // ARABIC THOUSANDS SEPARATOR
|
|
case 0x066D: // ARABIC FIVE POINTED STAR
|
|
case 0x06D4: // ARABIC FULL STOP
|
|
case 0x0700: // SYRIAC END OF PARAGRAPH
|
|
case 0x0701: // SYRIAC SUPRALINEAR FULL STOP
|
|
case 0x0702: // SYRIAC SUBLINEAR FULL STOP
|
|
case 0x0703: // SYRIAC SUPRALINEAR COLON
|
|
case 0x0704: // SYRIAC SUBLINEAR COLON
|
|
case 0x0705: // SYRIAC HORIZONTAL COLON
|
|
case 0x0706: // SYRIAC COLON SKEWED LEFT
|
|
case 0x0707: // SYRIAC COLON SKEWED RIGHT
|
|
case 0x0708: // SYRIAC SUPRALINEAR COLON SKEWED LEFT
|
|
case 0x0709: // SYRIAC SUBLINEAR COLON SKEWED RIGHT
|
|
case 0x070A: // SYRIAC CONTRACTION
|
|
case 0x070B: // SYRIAC HARKLEAN OBELUS
|
|
case 0x070C: // SYRIAC HARKLEAN METOBELUS
|
|
case 0x070D: // SYRIAC HARKLEAN ASTERISCUS
|
|
case 0x0964: // DEVANAGARI DANDA
|
|
case 0x0965: // DEVANAGARI DOUBLE DANDA
|
|
case 0x0970: // DEVANAGARI ABBREVIATION SIGN
|
|
case 0x0DF4: // SINHALA PUNCTUATION KUNDDALIYA
|
|
case 0x0E4F: // THAI CHARACTER FONGMAN
|
|
case 0x0E5A: // THAI CHARACTER ANGKHANKHU
|
|
case 0x0E5B: // THAI CHARACTER KHOMUT
|
|
case 0x0F04: // TIBETAN MARK INITIAL YIG MGO MDUN MA
|
|
case 0x0F05: // TIBETAN MARK CLOSING YIG MGO SGAB MA
|
|
case 0x0F06: // TIBETAN MARK CARET YIG MGO PHUR SHAD MA
|
|
case 0x0F07: // TIBETAN MARK YIG MGO TSHEG SHAD MA
|
|
case 0x0F08: // TIBETAN MARK SBRUL SHAD
|
|
case 0x0F09: // TIBETAN MARK BSKUR YIG MGO
|
|
case 0x0F0A: // TIBETAN MARK BKA- SHOG YIG MGO
|
|
case 0x0F0B: // TIBETAN MARK INTERSYLLABIC TSHEG
|
|
case 0x0F0C: // TIBETAN MARK DELIMITER TSHEG BSTAR
|
|
case 0x0F0D: // TIBETAN MARK SHAD
|
|
case 0x0F0E: // TIBETAN MARK NYIS SHAD
|
|
case 0x0F0F: // TIBETAN MARK TSHEG SHAD
|
|
case 0x0F10: // TIBETAN MARK NYIS TSHEG SHAD
|
|
case 0x0F11: // TIBETAN MARK RIN CHEN SPUNGS SHAD
|
|
case 0x0F12: // TIBETAN MARK RGYA GRAM SHAD
|
|
case 0x0F85: // TIBETAN MARK PALUTA
|
|
case 0x104A: // MYANMAR SIGN LITTLE SECTION
|
|
case 0x104B: // MYANMAR SIGN SECTION
|
|
case 0x104C: // MYANMAR SYMBOL LOCATIVE
|
|
case 0x104D: // MYANMAR SYMBOL COMPLETED
|
|
case 0x104E: // MYANMAR SYMBOL AFOREMENTIONED
|
|
case 0x104F: // MYANMAR SYMBOL GENITIVE
|
|
case 0x10FB: // GEORGIAN PARAGRAPH SEPARATOR
|
|
case 0x1361: // ETHIOPIC WORDSPACE
|
|
case 0x1362: // ETHIOPIC FULL STOP
|
|
case 0x1363: // ETHIOPIC COMMA
|
|
case 0x1364: // ETHIOPIC SEMICOLON
|
|
case 0x1365: // ETHIOPIC COLON
|
|
case 0x1366: // ETHIOPIC PREFACE COLON
|
|
case 0x1367: // ETHIOPIC QUESTION MARK
|
|
case 0x1368: // ETHIOPIC PARAGRAPH SEPARATOR
|
|
case 0x166D: // CANADIAN SYLLABICS CHI SIGN
|
|
case 0x166E: // CANADIAN SYLLABICS FULL STOP
|
|
case 0x16EB: // RUNIC SINGLE PUNCTUATION
|
|
case 0x16EC: // RUNIC MULTIPLE PUNCTUATION
|
|
case 0x16ED: // RUNIC CROSS PUNCTUATION
|
|
case 0x1735: // PHILIPPINE SINGLE PUNCTUATION
|
|
case 0x1736: // PHILIPPINE DOUBLE PUNCTUATION
|
|
case 0x17D4: // KHMER SIGN KHAN
|
|
case 0x17D5: // KHMER SIGN BARIYOOSAN
|
|
case 0x17D6: // KHMER SIGN CAMNUC PII KUUH
|
|
case 0x17D8: // KHMER SIGN BEYYAL
|
|
case 0x17D9: // KHMER SIGN PHNAEK MUAN
|
|
case 0x17DA: // KHMER SIGN KOOMUUT
|
|
case 0x1800: // MONGOLIAN BIRGA
|
|
case 0x1801: // MONGOLIAN ELLIPSIS
|
|
case 0x1802: // MONGOLIAN COMMA
|
|
case 0x1803: // MONGOLIAN FULL STOP
|
|
case 0x1804: // MONGOLIAN COLON
|
|
case 0x1805: // MONGOLIAN FOUR DOTS
|
|
case 0x1807: // MONGOLIAN SIBE SYLLABLE BOUNDARY MARKER
|
|
case 0x1808: // MONGOLIAN MANCHU COMMA
|
|
case 0x1809: // MONGOLIAN MANCHU FULL STOP
|
|
case 0x180A: // MONGOLIAN NIRUGU
|
|
case 0x1944: // LIMBU EXCLAMATION MARK
|
|
case 0x1945: // LIMBU QUESTION MARK
|
|
case 0x2016: // DOUBLE VERTICAL LINE
|
|
case 0x2017: // DOUBLE LOW LINE
|
|
case 0x2020: // DAGGER
|
|
case 0x2021: // DOUBLE DAGGER
|
|
case 0x2022: // BULLET
|
|
case 0x2023: // TRIANGULAR BULLET
|
|
case 0x2024: // ONE DOT LEADER
|
|
case 0x2025: // TWO DOT LEADER
|
|
case 0x2026: // HORIZONTAL ELLIPSIS
|
|
case 0x2027: // HYPHENATION POINT
|
|
case 0x2030: // PER MILLE SIGN
|
|
case 0x2031: // PER TEN THOUSAND SIGN
|
|
case 0x2032: // PRIME
|
|
case 0x2033: // DOUBLE PRIME
|
|
case 0x2034: // TRIPLE PRIME
|
|
case 0x2035: // REVERSED PRIME
|
|
case 0x2036: // REVERSED DOUBLE PRIME
|
|
case 0x2037: // REVERSED TRIPLE PRIME
|
|
case 0x2038: // CARET
|
|
case 0x203B: // REFERENCE MARK
|
|
case 0x203C: // DOUBLE EXCLAMATION MARK
|
|
case 0x203D: // INTERROBANG
|
|
case 0x203E: // OVERLINE
|
|
case 0x2041: // CARET INSERTION POINT
|
|
case 0x2042: // ASTERISM
|
|
case 0x2043: // HYPHEN BULLET
|
|
case 0x2047: // DOUBLE QUESTION MARK
|
|
case 0x2048: // QUESTION EXCLAMATION MARK
|
|
case 0x2049: // EXCLAMATION QUESTION MARK
|
|
case 0x204A: // TIRONIAN SIGN ET
|
|
case 0x204B: // REVERSED PILCROW SIGN
|
|
case 0x204C: // BLACK LEFTWARDS BULLET
|
|
case 0x204D: // BLACK RIGHTWARDS BULLET
|
|
case 0x204E: // LOW ASTERISK
|
|
case 0x204F: // REVERSED SEMICOLON
|
|
case 0x2050: // CLOSE UP
|
|
case 0x2051: // TWO ASTERISKS ALIGNED VERTICALLY
|
|
case 0x2053: // SWUNG DASH
|
|
case 0x2057: // QUADRUPLE PRIME
|
|
case 0x23B6: // BOTTOM SQUARE BRACKET OVER TOP SQUARE BRACKET
|
|
case 0x3001: // IDEOGRAPHIC COMMA
|
|
case 0x3002: // IDEOGRAPHIC FULL STOP
|
|
case 0x3003: // DITTO MARK
|
|
case 0x303D: // PART ALTERNATION MARK
|
|
case 0xFE30: // PRESENTATION FORM FOR VERTICAL TWO DOT LEADER
|
|
case 0xFE45: // SESAME DOT
|
|
case 0xFE46: // WHITE SESAME DOT
|
|
case 0xFE49: // DASHED OVERLINE
|
|
case 0xFE4A: // CENTRELINE OVERLINE
|
|
case 0xFE4B: // WAVY OVERLINE
|
|
case 0xFE4C: // DOUBLE WAVY OVERLINE
|
|
case 0xFE50: // SMALL COMMA
|
|
case 0xFE51: // SMALL IDEOGRAPHIC COMMA
|
|
case 0xFE52: // SMALL FULL STOP
|
|
case 0xFE54: // SMALL SEMICOLON
|
|
case 0xFE55: // SMALL COLON
|
|
case 0xFE56: // SMALL QUESTION MARK
|
|
case 0xFE57: // SMALL EXCLAMATION MARK
|
|
case 0xFE5F: // SMALL NUMBER SIGN
|
|
case 0xFE60: // SMALL AMPERSAND
|
|
case 0xFE61: // SMALL ASTERISK
|
|
case 0xFE68: // SMALL REVERSE SOLIDUS
|
|
case 0xFE6A: // SMALL PERCENT SIGN
|
|
case 0xFE6B: // SMALL COMMERCIAL AT
|
|
case 0xFF01: // FULLWIDTH EXCLAMATION MARK
|
|
case 0xFF02: // FULLWIDTH QUOTATION MARK
|
|
case 0xFF03: // FULLWIDTH NUMBER SIGN
|
|
case 0xFF05: // FULLWIDTH PERCENT SIGN
|
|
case 0xFF06: // FULLWIDTH AMPERSAND
|
|
case 0xFF07: // FULLWIDTH APOSTROPHE
|
|
case 0xFF0A: // FULLWIDTH ASTERISK
|
|
case 0xFF0C: // FULLWIDTH COMMA
|
|
case 0xFF0E: // FULLWIDTH FULL STOP
|
|
case 0xFF0F: // FULLWIDTH SOLIDUS
|
|
case 0xFF1A: // FULLWIDTH COLON
|
|
case 0xFF1B: // FULLWIDTH SEMICOLON
|
|
case 0xFF1F: // FULLWIDTH QUESTION MARK
|
|
case 0xFF20: // FULLWIDTH COMMERCIAL AT
|
|
case 0xFF3C: // FULLWIDTH REVERSE SOLIDUS
|
|
case 0xFF61: // HALFWIDTH IDEOGRAPHIC FULL STOP
|
|
case 0xFF64: // HALFWIDTH IDEOGRAPHIC COMMA
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
wstring trimWhitespaceOrPunct( wstring const & in )
|
|
{
|
|
wchar const * wordBegin = in.c_str();
|
|
wstring::size_type wordSize = in.size();
|
|
|
|
// Skip any leading whitespace
|
|
while( *wordBegin && ( Folding::isWhitespace( *wordBegin ) || Folding::isPunct( *wordBegin ) ) )
|
|
{
|
|
++wordBegin;
|
|
--wordSize;
|
|
}
|
|
|
|
// Skip any trailing whitespace
|
|
while( wordSize && ( Folding::isWhitespace( wordBegin[ wordSize - 1 ] ) ||
|
|
Folding::isPunct( wordBegin[ wordSize - 1 ] ) ) )
|
|
--wordSize;
|
|
|
|
return wstring( wordBegin, wordSize );
|
|
}
|
|
|
|
}
|