Websites: Add alternative loading method

config.cc

@@ -203,11 +203,11 @@ WebSites makeDefaultWebSites()
 {
   WebSites ws;
 
-  ws.push_back( WebSite( "b88cb2898e634c6638df618528284c2d", "Google En-En (Oxford)", "https://www.google.com/dictionary?aq=f&langpair=en|en&q=%GDWORD%&hl=en", false, "" ) );
+  ws.push_back( WebSite( "b88cb2898e634c6638df618528284c2d", "Google En-En (Oxford)", "https://www.google.com/dictionary?aq=f&langpair=en|en&q=%GDWORD%&hl=en", false, "", true ) );
-  ws.push_back( WebSite( "f376365a0de651fd7505e7e5e683aa45", "Urban Dictionary", "https://www.urbandictionary.com/define.php?term=%GDWORD%", false, "" ) );
+  ws.push_back( WebSite( "f376365a0de651fd7505e7e5e683aa45", "Urban Dictionary", "https://www.urbandictionary.com/define.php?term=%GDWORD%", false, "", true ) );
-  ws.push_back( WebSite( "324ca0306187df7511b26d3847f4b07c", "Multitran (En)", "https://multitran.ru/c/m.exe?CL=1&l1=1&s=%GD1251%", false, "" ) );
+  ws.push_back( WebSite( "324ca0306187df7511b26d3847f4b07c", "Multitran (En)", "https://multitran.ru/c/m.exe?CL=1&l1=1&s=%GD1251%", false, "", true ) );
-  ws.push_back( WebSite( "924db471b105299c82892067c0f10787", "Lingvo (En-Ru)", "http://lingvopro.abbyyonline.com/en/Search/en-ru/%GDWORD%", false, "" ) );
+  ws.push_back( WebSite( "924db471b105299c82892067c0f10787", "Lingvo (En-Ru)", "http://lingvopro.abbyyonline.com/en/Search/en-ru/%GDWORD%", false, "", true ) );
-  ws.push_back( WebSite( "087a6d65615fb047f4c80eef0a9465db", "Michaelis (Pt-En)", "http://michaelis.uol.com.br/moderno/ingles/index.php?lingua=portugues-ingles&palavra=%GDISO1%", false, "" ) );
+  ws.push_back( WebSite( "087a6d65615fb047f4c80eef0a9465db", "Michaelis (Pt-En)", "http://michaelis.uol.com.br/moderno/ingles/index.php?lingua=portugues-ingles&palavra=%GDISO1%", false, "", true ) );
 
   return ws;
 }
@@ -634,6 +634,7 @@ Class load() throw( exError )
       w.url = ws.attribute( "url" );
       w.enabled = ( ws.attribute( "enabled" ) == "1" );
       w.iconFilename = ws.attribute( "icon" );
+      w.inside_iframe = ( ws.attribute( "inside_iframe", "1" ) == "1" );
 
       c.webSites.push_back( w );
     }
@@ -1353,6 +1354,10 @@ void save( Class const & c ) throw( exError )
       QDomAttr icon = dd.createAttribute( "icon" );
       icon.setValue( i->iconFilename );
       ws.setAttributeNode( icon );
+
+      QDomAttr inside_iframe = dd.createAttribute( "inside_iframe" );
+      inside_iframe.setValue( i->inside_iframe ? "1" : "0" );
+      ws.setAttributeNode( inside_iframe );
     }
   }
 

config.hh

@@ -276,17 +276,20 @@ struct WebSite
   QString id, name, url;
   bool enabled;
   QString iconFilename;
+  bool inside_iframe;
 
   WebSite(): enabled( false )
   {}
 
   WebSite( QString const & id_, QString const & name_, QString const & url_,
-           bool enabled_, QString const & iconFilename_ ):
+           bool enabled_, QString const & iconFilename_, bool inside_iframe_ ):
-    id( id_ ), name( name_ ), url( url_ ), enabled( enabled_ ), iconFilename( iconFilename_ ) {}
+    id( id_ ), name( name_ ), url( url_ ), enabled( enabled_ ), iconFilename( iconFilename_ ),
+    inside_iframe( inside_iframe_ ) {}
 
   bool operator == ( WebSite const & other ) const
   { return id == other.id && name == other.name && url == other.url &&
-           enabled == other.enabled && iconFilename == other.iconFilename; }
+           enabled == other.enabled && iconFilename == other.iconFilename &&
+           inside_iframe == other.inside_iframe; }
 };
 
 /// All the WebSites

loaddictionaries.cc

@@ -328,7 +328,7 @@ void loadDictionaries( QWidget * parent, bool showInitially,
   ///// WebSites are very simple, no need to create them asyncronously
   {
     vector< sptr< Dictionary::Class > > dicts =
-      WebSite::makeDictionaries( cfg.webSites );
+      WebSite::makeDictionaries( cfg.webSites, dictNetMgr );
 
     dictionaries.insert( dictionaries.end(), dicts.begin(), dicts.end() );
   }

sources.cc

@@ -58,6 +58,7 @@ Sources::Sources( QWidget * parent, Config::Class const & cfg):
   ui.webSites->resizeColumnToContents( 1 );
   ui.webSites->resizeColumnToContents( 2 );
   ui.webSites->resizeColumnToContents( 3 );
+  ui.webSites->resizeColumnToContents( 4 );
 
   ui.dictServers->setTabKeyNavigation( true );
   ui.dictServers->setModel( &dictServersModel );
@@ -588,7 +589,7 @@ Qt::ItemFlags WebSitesModel::flags( QModelIndex const & index ) const
 
   if ( index.isValid() )
   {
-    if ( !index.column() )
+    if ( index.column() <= 1 )
       result |= Qt::ItemIsUserCheckable;
     else
       result |= Qt::ItemIsEditable;
@@ -615,16 +616,26 @@ int WebSitesModel::columnCount( QModelIndex const & parent ) const
 
 QVariant WebSitesModel::headerData( int section, Qt::Orientation /*orientation*/, int role ) const
 {
+  if( role == Qt::ToolTipRole )
+  {
+    if( section == 1 )
+      return tr( "Insert article as link inside <iframe> tag" );
+
+    return QVariant();
+  }
+
   if ( role == Qt::DisplayRole )
     switch( section )
     {
       case 0:
         return tr( "Enabled" );
       case 1:
-        return tr( "Name" );
+        return tr( "As link" );
       case 2:
-        return tr( "Address" );
+        return tr( "Name" );
       case 3:
+        return tr( "Address" );
+      case 4:
         return tr( "Icon" );
       default:
         return QVariant();
@@ -638,15 +649,23 @@ QVariant WebSitesModel::data( QModelIndex const & index, int role ) const
   if ( index.row() >= webSites.size() )
     return QVariant();
 
+  if( role == Qt::ToolTipRole )
+  {
+    if( index.column() == 1 )
+      return tr( "Insert article as link inside <iframe> tag" );
+
+    return QVariant();
+  }
+
   if ( role == Qt::DisplayRole || role == Qt::EditRole )
   {
     switch( index.column() )
     {
-      case 1:
-        return webSites[ index.row() ].name;
       case 2:
-        return webSites[ index.row() ].url;
+        return webSites[ index.row() ].name;
       case 3:
+        return webSites[ index.row() ].url;
+      case 4:
         return webSites[ index.row() ].iconFilename;
       default:
         return QVariant();
@@ -656,6 +675,9 @@ QVariant WebSitesModel::data( QModelIndex const & index, int role ) const
   if ( role == Qt::CheckStateRole && !index.column() )
     return webSites[ index.row() ].enabled ? Qt::Checked : Qt::Unchecked;
 
+  if ( role == Qt::CheckStateRole && index.column() == 1 )
+    return webSites[ index.row() ].inside_iframe ? Qt::Checked : Qt::Unchecked;
+
   return QVariant();
 }
 
@@ -677,18 +699,26 @@ bool WebSitesModel::setData( QModelIndex const & index, const QVariant & value,
     return true;
   }
 
+  if ( role == Qt::CheckStateRole && index.column() == 1 )
+  {
+    webSites[ index.row() ].inside_iframe = !webSites[ index.row() ].inside_iframe;
+
+    dataChanged( index, index );
+    return true;
+  }
+
   if ( role == Qt::DisplayRole || role == Qt::EditRole )
     switch( index.column() )
     {
-      case 1:
+      case 2:
         webSites[ index.row() ].name =  value.toString();
         dataChanged( index, index );
         return true;
-      case 2:
+      case 3:
         webSites[ index.row() ].url =  value.toString();
         dataChanged( index, index );
         return true;
-      case 3:
+      case 4:
         webSites[ index.row() ].iconFilename =  value.toString();
         dataChanged( index, index );
         return true;

website.cc

@@ -8,6 +8,7 @@
 #include <QTextCodec>
 #include <QDir>
 #include <QFileInfo>
+#include <QRegExp>
 
 namespace WebSite {
 
@@ -20,16 +21,22 @@ class WebSiteDictionary: public Dictionary::Class
   string name;
   QByteArray urlTemplate;
   QString iconFilename;
+  bool inside_iframe;
+  QNetworkAccessManager & netMgr;
 
 public:
 
   WebSiteDictionary( string const & id, string const & name_,
                      QString const & urlTemplate_,
-                     QString const & iconFilename_ ):
+                     QString const & iconFilename_,
+                     bool inside_iframe_,
+                     QNetworkAccessManager & netMgr_ ):
     Dictionary::Class( id, vector< string >() ),
     name( name_ ),
     urlTemplate( QUrl( urlTemplate_ ).toEncoded() ),
-    iconFilename( iconFilename_ )
+    iconFilename( iconFilename_ ),
+    inside_iframe( inside_iframe_ ),
+    netMgr( netMgr_ )
   {
     dictionaryDescription = urlTemplate_;
   }
@@ -69,6 +76,198 @@ sptr< WordSearchRequest > WebSiteDictionary::prefixMatch( wstring const & /*word
   return sr;
 }
 
+class WebSiteArticleRequest: public WebSiteDataRequestSlots
+{
+  typedef std::list< std::pair< QNetworkReply *, bool > > NetReplies;
+  QNetworkReply * netReply;
+  QString url;
+  Class * dictPtr;
+  QNetworkAccessManager & mgr;
+
+public:
+
+  WebSiteArticleRequest( QString const & url, QNetworkAccessManager & _mgr,
+                         Class * dictPtr_ );
+  ~WebSiteArticleRequest()
+  {}
+
+  virtual void cancel();
+
+private:
+
+  virtual void requestFinished( QNetworkReply * );
+};
+
+void WebSiteArticleRequest::cancel()
+{
+  finish();
+}
+
+WebSiteArticleRequest::WebSiteArticleRequest( QString const & url_,
+                                              QNetworkAccessManager & _mgr,
+                                              Class * dictPtr_ ):
+  url( url_ ), dictPtr( dictPtr_ ), mgr( _mgr )
+{
+  connect( &mgr, SIGNAL( finished( QNetworkReply * ) ),
+           this, SLOT( requestFinished( QNetworkReply * ) ),
+           Qt::QueuedConnection );
+
+  QUrl reqUrl( url );
+
+  netReply = mgr.get( QNetworkRequest( reqUrl ) );
+
+#ifndef QT_NO_OPENSSL
+  connect( netReply, SIGNAL( sslErrors( QList< QSslError > ) ),
+           netReply, SLOT( ignoreSslErrors() ) );
+#endif
+}
+
+void WebSiteArticleRequest::requestFinished( QNetworkReply * r )
+{
+  if ( isFinished() ) // Was cancelled
+    return;
+
+  if ( r != netReply )
+  {
+    // Well, that's not our reply, don't do anything
+    return;
+  }
+
+  if ( netReply->error() == QNetworkReply::NoError )
+  {
+    // Check for redirect reply
+
+    QVariant possibleRedirectUrl = netReply->attribute( QNetworkRequest::RedirectionTargetAttribute );
+    QUrl redirectUrl = possibleRedirectUrl.toUrl();
+    if( !redirectUrl.isEmpty() )
+    {
+      netReply->deleteLater();
+      netReply = mgr.get( QNetworkRequest( redirectUrl ) );
+#ifndef QT_NO_OPENSSL
+      connect( netReply, SIGNAL( sslErrors( QList< QSslError > ) ),
+               netReply, SLOT( ignoreSslErrors() ) );
+#endif
+      return;
+    }
+
+    // Handle reply data
+
+    QByteArray replyData = netReply->readAll();
+    QString articleString;
+
+    QTextCodec * codec = QTextCodec::codecForHtml( replyData, 0 );
+    if( codec )
+      articleString = codec->toUnicode( replyData );
+    else
+      articleString = QString::fromUtf8( replyData );
+
+    QString divName = QString( "website_" ) + dictPtr->getId().c_str();
+
+    // Change links from relative to absolute
+
+    QString root = netReply->url().scheme() + "://" + netReply->url().host();
+    QString base = root + netReply->url().path();
+    while( !base.isEmpty() && !base.endsWith( "/" ) )
+      base.chop( 1 );
+
+    QRegExp tags( "<\\s*(a|link|img|script)\\s+[^>]*(src|href)\\s*=\\s*['\"][^>]+>",
+                  Qt::CaseInsensitive, QRegExp::RegExp2 );
+    QRegExp links( "\\b(src|href)\\s*=\\s*(['\"])([^'\"]+['\"])",
+                   Qt::CaseInsensitive, QRegExp::RegExp2 );
+    int pos = 0;
+    while( pos >= 0 )
+    {
+      pos = articleString.indexOf( tags, pos );
+      if( pos < 0 )
+        break;
+
+      QString tag = tags.cap();
+
+      int linkPos = tag.indexOf( links );
+      if( linkPos < 0 )
+      {
+        pos += tag.size();
+        continue;
+      }
+
+      QString url = links.cap( 3 );
+
+      if( url.indexOf( ":/" ) >= 0 || url.indexOf( "data:" ) >= 0
+          || url.indexOf( "mailto:" ) >= 0 || url.startsWith( "#" ) )
+      {
+        // External link, anchor or base64-encoded data
+        pos += tag.size();
+        continue;
+      }
+
+      QString newUrl = links.cap( 1 ) + "=" + links.cap( 2 );
+      if( url.startsWith( "//" ) )
+        newUrl += netReply->url().scheme() + ":";
+      else
+      if( url.startsWith( "/" ) )
+        newUrl += root;
+      else
+        newUrl += base;
+      newUrl += links.cap( 3 );
+
+      tag.replace( linkPos, links.cap().size(), newUrl );
+      articleString.replace( pos, tags.cap().size(), tag );
+
+      pos += tag.size();
+    }
+
+    // Check for unclosed <span> and <div>
+
+    int openTags = articleString.count( QRegExp( "<\\s*span\\b", Qt::CaseInsensitive ) );
+    int closedTags = articleString.count( QRegExp( "<\\s*/span\\s*>", Qt::CaseInsensitive ) );
+    while( openTags > closedTags )
+    {
+      articleString += "</span>";
+      closedTags += 1;
+    }
+
+    openTags = articleString.count( QRegExp( "<\\s*div\\b", Qt::CaseInsensitive ) );
+    closedTags = articleString.count( QRegExp( "<\\s*/div\\s*>", Qt::CaseInsensitive ) );
+    while( openTags > closedTags )
+    {
+      articleString += "</div>";
+      closedTags += 1;
+    }
+
+    // See Issue #271: A mechanism to clean-up invalid HTML cards.
+    articleString += "</font>""</font>""</font>""</font>""</font>""</font>"
+                     "</font>""</font>""</font>""</font>""</font>""</font>"
+                     "</b></b></b></b></b></b></b></b>"
+                     "</i></i></i></i></i></i></i></i>"
+                     "</a></a></a></a></a></a></a></a>";
+
+    QByteArray articleBody = articleString.toUtf8();
+
+    QString divStr = QString( "<div class=\"" ) + divName + "\"";
+    divStr += dictPtr->isToLanguageRTL() ? " dir=\"rtl\">" : ">";
+
+    articleBody.prepend( divStr.toUtf8() );
+    articleBody.append( "</div>" );
+
+    Mutex::Lock _( dataMutex );
+
+    size_t prevSize = data.size();
+
+    data.resize( prevSize + articleBody.size() );
+
+    memcpy( &data.front() + prevSize, articleBody.data(), articleBody.size() );
+
+    hasAnyData = true;
+
+  }
+  else
+    setErrorString( netReply->errorString() );
+
+  netReply->deleteLater();
+
+  finish();
+}
+
 sptr< DataRequest > WebSiteDictionary::getArticle( wstring const & str,
                                                    vector< wstring > const &,
                                                    wstring const & context )
@@ -122,6 +321,10 @@ sptr< DataRequest > WebSiteDictionary::getArticle( wstring const & str,
     }
   }
 
+  if( inside_iframe )
+  {
+    // Just insert link in <iframe> tag
+
     string result = "<div class=\"website_padding\"></div>";
 
     result += string( "<iframe id=\"gdexpandframe-" ) + getId() +
@@ -140,6 +343,11 @@ sptr< DataRequest > WebSiteDictionary::getArticle( wstring const & str,
     return dr;
   }
 
+  // To load data from site
+
+  return new WebSiteArticleRequest( urlString, netMgr, this );
+}
+
 void WebSiteDictionary::loadIcon() throw()
 {
   if ( dictionaryIconLoaded )
@@ -158,7 +366,8 @@ void WebSiteDictionary::loadIcon() throw()
 
 }
 
-vector< sptr< Dictionary::Class > > makeDictionaries( Config::WebSites const & ws )
+vector< sptr< Dictionary::Class > > makeDictionaries( Config::WebSites const & ws,
+                                                      QNetworkAccessManager & mgr )
   throw( std::exception )
 {
   vector< sptr< Dictionary::Class > > result;
@@ -169,7 +378,10 @@ vector< sptr< Dictionary::Class > > makeDictionaries( Config::WebSites const & w
       result.push_back( new WebSiteDictionary( ws[ x ].id.toUtf8().data(),
                                                ws[ x ].name.toUtf8().data(),
                                                ws[ x ].url,
-                                               ws[ x ].iconFilename ) );
+                                               ws[ x ].iconFilename,
+                                               ws[ x ].inside_iframe,
+                                               mgr )
+                      );
   }
 
   return result;

website.hh

@@ -6,6 +6,8 @@
 
 #include "dictionary.hh"
 #include "config.hh"
+#include <QNetworkAccessManager>
+#include <QNetworkReply>
 
 /// Support for any web sites via a templated url.
 namespace WebSite {
@@ -13,9 +15,21 @@ namespace WebSite {
 using std::vector;
 using std::string;
 
-vector< sptr< Dictionary::Class > > makeDictionaries( Config::WebSites const & )
+vector< sptr< Dictionary::Class > > makeDictionaries( Config::WebSites const &,
+                                                      QNetworkAccessManager & )
     throw( std::exception );
 
+/// Exposed here for moc
+class WebSiteDataRequestSlots: public Dictionary::DataRequest
+{
+  Q_OBJECT
+
+protected slots:
+
+  virtual void requestFinished( QNetworkReply * )
+  {}
+};
+
 }
 
 #endif