fix #420 and add some more fixes for XML -> json utility function, also make filtering dialog work with "rawContents" to enable basic testing

Author: Martin Rotter

resources/docs/Message-filters.md

@@ -50,7 +50,7 @@ Here is the reference of methods and properties of some types available in your
 | `String url` | URL of the message. |
 | `String author` | Author of the message. |
 | `String contents` | Contents of the message. |
-| `String rawContents` | This is RAW contents of the message as it was obtained from remote service/feed. You can expect raw `XML` or `JSON` element data here. Note that this attribute has some value only if `alreadyStoredInDb` returns `false`. In other words, this attribute is not persistently stored inside RSS Guard's DB. Also, this attribute might not be filled when testing the filter, it is only filled during live filter execution on real incoming messages. |
+| `String rawContents` | This is RAW contents of the message as it was obtained from remote service/feed. You can expect raw `XML` or `JSON` element data here. Note that this attribute has some value only if `alreadyStoredInDb` returns `false`. In other words, this attribute is not persistently stored inside RSS Guard's DB. Also, this attribute is artificially filled with ATOM-like data when testing the filter. |
 | `Number score` | Arbitrary number in range <0.0, 100.0>. You can use this number to sort messages in a custom fashion as this attribute also has its own column in messages list. |
 | `Date created` | Date/time of the message. |
 | `Boolean isRead` | Is message read? |

src/librssguard/core/filterutils.cpp

@@ -33,8 +33,13 @@ QString jsonProcessXmlElement(const QDomElement& elem) {
   }
 
   QStringList elems;
+  QString elem_text;
 
   for (int i = 0; i < elem.childNodes().size(); i++) {
+    if (elem.childNodes().at(i).isText()) {
+      elem_text = jsonEscapeString(elem.childNodes().at(i).nodeValue());
+    }
+
     if (!elem.childNodes().at(i).isElement()) {
       continue;
     }
@@ -43,22 +48,26 @@ QString jsonProcessXmlElement(const QDomElement& elem) {
                                    jsonProcessXmlElement(elem.childNodes().at(i).toElement()));
   }
 
-  if (attrs.isEmpty()) {
-    if (elems.isEmpty()) {
-      return QSL("\"%1\"").arg(jsonEscapeString(elem.text()));
-    }
-    else {
-      return QSL("{%1}").arg(elems.join(QSL(",\n")));
-    }
+  QString str;
+
+  if (!elems.isEmpty() && !attrs.isEmpty()) {
+    str = QSL("{%1, %2, %3}").arg(attrs.join(QSL(",\n")),
+                                  elems.join(QSL(",\n")),
+                                  QSL("\"__text\": \"%1\"").arg(elem_text));
+  }
+  else if (!elems.isEmpty()) {
+    str = QSL("{%1, %2}").arg(elems.join(QSL(",\n")),
+                              QSL("\"__text\": \"%1\"").arg(elem_text));
   }
-  else if (elems.isEmpty()) {
-    return QSL("{%1, \"__text\": \"%2\"}").arg(attrs.join(QSL(",\n")),
-                                               jsonEscapeString(elem.text()));
+  else if (!attrs.isEmpty()) {
+    str = QSL("{%1, %2}").arg(attrs.join(QSL(",\n")),
+                              QSL("\"__text\": \"%1\"").arg(elem_text));
   }
   else {
-    return QSL("{%1, %2}").arg(attrs.join(QSL(",\n")),
-                               elems.join(QSL(",\n")));
+    str = QSL("{%1}").arg(QSL("\"__text\": \"%1\"").arg(elem_text));
   }
+
+  return str;
 }
 
 QString FilterUtils::fromXmlToJson(const QString& xml) const {

src/librssguard/core/message.cpp

@@ -120,6 +120,23 @@ Message Message::fromSqlRecord(const QSqlRecord& record, bool* result) {
   return message;
 }
 
+QString Message::generateRawAtomContents(const Message& msg) {
+  return QSL("<entry>"
+             "<title>%1</title>"
+             "<link href=\"%2\" rel=\"alternate\" type=\"text/html\" title=\"%1\"/>"
+             "<published>%3</published>"
+             "<author><name>%6</name></author>"
+             "<updated>%3</updated>"
+             "<id>%4</id>"
+             "<summary type=\"html\">%5</summary>"
+             "</entry>").arg(msg.m_title,
+                             msg.m_url,
+                             msg.m_created.toUTC().toString(QSL("yyyy-MM-ddThh:mm:ss")),
+                             msg.m_url,
+                             msg.m_contents.toHtmlEscaped(),
+                             msg.m_author);
+}
+
 QDataStream& operator<<(QDataStream& out, const Message& my_obj) {
   out << my_obj.m_accountId
       << my_obj.m_customHash

src/librssguard/core/message.h

@@ -39,6 +39,7 @@ class RSSGUARD_DLLSPEC Message {
     // Creates Message from given record, which contains
     // row from query SELECT * FROM Messages WHERE ....;
     static Message fromSqlRecord(const QSqlRecord& record, bool* result = nullptr);
+    static QString generateRawAtomContents(const Message& msg);
 
   public:
     QString m_title;

src/librssguard/core/messagesforfiltersmodel.cpp

@@ -117,6 +117,7 @@ void MessagesForFiltersModel::testFilter(MessageFilter* filter, QJSEngine* engin
   for (int i = 0; i < m_messages.size(); i++) {
     Message* msg = messageForRow(i);
 
+    msg->m_rawContents = Message::generateRawAtomContents(*msg);
     msg_proxy->setMessage(msg);
 
     try {

src/librssguard/gui/dialogs/formmessagefiltersmanager.cpp

@@ -263,11 +263,13 @@ void FormMessageFiltersManager::testFilter() {
                  "  Author = '%3'\n"
                  "  Is read/important = '%4/%5'\n"
                  "  Created on = '%6'\n"
-                 "  Contents = '%7'").arg(msg.m_title, msg.m_url, msg.m_author,
-                                          msg.m_isRead ? tr("yes") : tr("no"),
-                                          msg.m_isImportant ? tr("yes") : tr("no"),
-                                          QString::number(msg.m_created.toMSecsSinceEpoch()),
-                                          msg.m_contents);
+                 "  Contents = '%7'\n"
+                 "  RAW contents = '%8'").arg(msg.m_title, msg.m_url, msg.m_author,
+                                              msg.m_isRead ? tr("yes") : tr("no"),
+                                              msg.m_isImportant ? tr("yes") : tr("no"),
+                                              QString::number(msg.m_created.toMSecsSinceEpoch()),
+                                              msg.m_contents,
+                                              msg.m_rawContents);
 
     m_ui.m_txtErrors->insertPlainText(answer);
   }
@@ -315,22 +317,35 @@ void FormMessageFiltersManager::processCheckedFeeds() {
 
         // Create backup of message.
         Message* msg = &msgs[i]; msg->m_assignedLabels = labels_in_message;
+
+        msg->m_rawContents = Message::generateRawAtomContents(*msg);
+
         Message msg_backup(*msg);
 
         msg_obj.setMessage(msg);
 
-        MessageObject::FilteringAction result = fltr->filterMessage(&filter_engine);
         bool remove_from_list = false;
 
-        if (result == MessageObject::FilteringAction::Purge) {
-          remove_from_list = true;
+        try {
+          MessageObject::FilteringAction result = fltr->filterMessage(&filter_engine);
+
+          if (result == MessageObject::FilteringAction::Purge) {
+            remove_from_list = true;
 
-          // Purge the message completely and remove leftovers.
-          DatabaseQueries::purgeMessage(database, msg->m_id);
-          DatabaseQueries::purgeLeftoverLabelAssignments(database, msg->m_accountId);
+            // Purge the message completely and remove leftovers.
+            DatabaseQueries::purgeMessage(database, msg->m_id);
+            DatabaseQueries::purgeLeftoverLabelAssignments(database, msg->m_accountId);
+          }
+          else if (result == MessageObject::FilteringAction::Ignore) {
+            remove_from_list = true;
+          }
         }
-        else if (result == MessageObject::FilteringAction::Ignore) {
-          remove_from_list = true;
+        catch (const FilteringException& ex) {
+          qCriticalNN << LOGSEC_CORE
+                      << "Error when running script when processing existing messages:"
+                      << QUOTE_W_SPACE_DOT(ex.message());
+
+          continue;
         }
 
         if (!msg_backup.m_isRead && msg->m_isRead) {
@@ -585,6 +600,7 @@ Message FormMessageFiltersManager::testingMessage() const {
   msg.m_isImportant = m_ui.m_cbSampleImportant->isChecked();
   msg.m_created = QDateTime::fromMSecsSinceEpoch(m_ui.m_txtSampleCreatedOn->text().toLongLong());
   msg.m_contents = m_ui.m_txtSampleContents->toPlainText();
+  msg.m_rawContents = Message::generateRawAtomContents(msg);
 
   return msg;
 }