Blocker fixes

10 files changed, 202 insertions, 101 deletions

diff --git a/data/poi.conf b/data/poi.conf
index c9461f5..409ade5 100644
--- a/data/poi.conf
+++ b/data/poi.conf
@@ -48,9 +48,10 @@ tabtoolbarMovable=false
 shortcut="Ctrl+Shift+A"
 path="$settings/"
 subscriptions=[
+    "custom.txt"
 #    "https://easylist.to/easylist/easylist.txt",
-    "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt",
-    "https://easylist.to/easylist/easyprivacy.txt"
+#    "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt",
+#    "https://easylist.to/easylist/easyprivacy.txt"
 ]
 
 # Bookmark manager
diff --git a/smolbote.qbs b/smolbote.qbs
index 7e7c61b..236c6da 100644
--- a/smolbote.qbs
+++ b/smolbote.qbs
@@ -99,25 +99,37 @@ Project {
             ]
         }
 
+        Group {
+            name: "Request Filter"
+            files: [
+                "src/blocker/blockermanager.cpp",
+                "src/blocker/blockermanager.h",
+                "src/blocker/blockerrule.cpp",
+                "src/blocker/blockerrule.h",
+                "src/blocker/blockersubscription.cpp",
+                "src/blocker/blockersubscription.h",
+                "src/blocker/regexp.cpp",
+                "src/blocker/regexp.h",
+                "src/blocker/subscriptiondialog.ui",
+                "src/blocker/subscriptionform.ui",
+                "src/webengine/urlinterceptor.cpp",
+                "src/webengine/urlinterceptor.h",
+            ]
+            cpp.defines: {
+                if(project.deprecatedWarnings)
+                    defines.push("QT_DEPRECATED_WARNINGS", "QT_DISABLE_DEPRECATED_BEFORE="+project.deprecatedBefore);
+                defines.push("DEBUG_VERBOSE")
+                return defines;
+            }
+        }
+
         files: [
             "data/resources.qrc",
-            "src/blocker/blockermanager.cpp",
-            "src/blocker/blockermanager.h",
-            "src/blocker/blockerrule.cpp",
-            "src/blocker/blockerrule.h",
-            "src/blocker/blockersubscription.cpp",
-            "src/blocker/blockersubscription.h",
-            "src/blocker/regexp.cpp",
-            "src/blocker/regexp.h",
-            "src/blocker/subscriptiondialog.ui",
-            "src/blocker/subscriptionform.ui",
             "src/forms/profiledialog.cpp",
             "src/forms/profiledialog.h",
             "src/forms/profiledialog.ui",
             "src/settings.cpp",
             "src/settings.h",
-            "src/webengine/urlinterceptor.cpp",
-            "src/webengine/urlinterceptor.h",
             "src/webengine/webengineprofile.cpp",
             "src/webengine/webengineprofile.h",
             "src/webengine/webview.cpp",
diff --git a/src/blocker/blockermanager.cpp b/src/blocker/blockermanager.cpp
index 3dd516c..e8533fe 100644
--- a/src/blocker/blockermanager.cpp
+++ b/src/blocker/blockermanager.cpp
@@ -34,7 +34,8 @@ BlockerManager::BlockerManager(QWidget *parent) :
     ui->setupUi(this);
 
     for(QString listUrl : sSettings->value("blocker.subscriptions").toStringList()) {
-        BlockerSubscription *sub = new BlockerSubscription(listUrl, this);
+        BlockerSubscription *sub = new BlockerSubscription(QUrl(listUrl), this);
+        m_subscriptions.append(sub);
         ui->tabWidget->addTab(sub, sub->name());
     }
 }
@@ -43,3 +44,8 @@ BlockerManager::~BlockerManager()
 {
     delete ui;
 }
+
+QVector<BlockerSubscription *> BlockerManager::subscriptions() const
+{
+    return m_subscriptions;
+}
diff --git a/src/blocker/blockermanager.h b/src/blocker/blockermanager.h
index eb5e04e..726053c 100644
--- a/src/blocker/blockermanager.h
+++ b/src/blocker/blockermanager.h
@@ -27,6 +27,7 @@ namespace Ui {
 class UrlInterceptorDialog;
 }
 
+class BlockerSubscription;
 class BlockerManager : public QDialog
 {
     Q_OBJECT
@@ -35,8 +36,11 @@ public:
     explicit BlockerManager(QWidget *parent = 0);
     ~BlockerManager();
 
+    QVector<BlockerSubscription* > subscriptions() const;
+
 private:
     Ui::UrlInterceptorDialog *ui;
+    QVector<BlockerSubscription *> m_subscriptions;
 };
 
 #endif // URLINTERCEPTORDIALOG_H
diff --git a/src/blocker/blockerrule.cpp b/src/blocker/blockerrule.cpp
index 1c118e2..fb7f2bf 100644
--- a/src/blocker/blockerrule.cpp
+++ b/src/blocker/blockerrule.cpp
@@ -28,6 +28,7 @@
 BlockerRule::BlockerRule(QString rule, QObject *parent) :
     QObject(parent)
 {
+    m_filter = rule;
     QString pattern = rule;
 
     // Empty rule or comment
@@ -36,62 +37,26 @@ BlockerRule::BlockerRule(QString rule, QObject *parent) :
         return;
     }
 
-    // Ignore element hiding rules for now
-    if(pattern.contains("##") || pattern.contains("#@#")) {
-        m_valid = false;
-        return;
-    }
-
     // Exception
     if(pattern.startsWith("@@")) {
         m_exception = true;
         pattern.remove(0, 2);
     }
 
+    // Ignore element hiding rules for now
+    if(pattern.contains("##") || pattern.contains("#@#")) {
+        m_valid = false;
+        return;
+    }
+
     // Options
     if(pattern.contains("$")) {
         QString opts = pattern.mid(pattern.indexOf("$")+1);
+        pattern.remove(pattern.indexOf("$"), pattern.length());
+
         for(QString opt : opts.split(',')) {
-            if(opt.endsWith("script")) {
-                if(opt.startsWith("~")) {
-                    m_whitelistOptions.setFlag(RuleOption::script, true);
-                } else {
-                    m_blacklistOptions.setFlag(RuleOption::script, true);
-                }
-            } else if(opt.endsWith("image")) {
-                if(opt.startsWith("~")) {
-                    m_whitelistOptions.setFlag(RuleOption::image, true);
-                } else {
-                    m_blacklistOptions.setFlag(RuleOption::image, true);
-                }
-            } else if(opt.endsWith("stylesheet")) {
-                if(opt.startsWith("~")) {
-                    m_whitelistOptions.setFlag(RuleOption::stylesheet, true);
-                } else {
-                    m_blacklistOptions.setFlag(RuleOption::stylesheet, true);
-                }
-            } else if(opt.endsWith("object")) {
-                if(opt.startsWith("~")) {
-                    m_whitelistOptions.setFlag(RuleOption::object, true);
-                } else {
-                    m_blacklistOptions.setFlag(RuleOption::object, true);
-                }
-            } else if(opt.endsWith("object-subrequest")) {
-                if(opt.startsWith("~")) {
-                    m_whitelistOptions.setFlag(RuleOption::objectsubrequest, true);
-                } else {
-                    m_blacklistOptions.setFlag(RuleOption::objectsubrequest, true);
-                }
-            } else if(opt.endsWith("subdocument")) {
-                if(opt.startsWith("~")) {
-                    m_whitelistOptions.setFlag(RuleOption::subdocument, true);
-                } else {
-                    m_blacklistOptions.setFlag(RuleOption::subdocument, true);
-                }
-            }
+            parseOption(opt);
         }
-
-        pattern.remove(pattern.indexOf("$"), pattern.length());
     }
 
     // Regular expression
@@ -157,9 +122,43 @@ bool BlockerRule::isException()
     return m_exception;
 }
 
-QString BlockerRule::toString() const
+QString BlockerRule::filter() const
 {
-    return QString("On [%1]: %2 %3").arg(domainExpression.pattern()).arg(ruleExpression.pattern()).arg(QString::number(m_blacklistOptions, 2));
+    return m_filter;
+}
+
+void BlockerRule::parseOption(const QString &opt)
+{
+    if(opt.startsWith("script")) {
+        m_blacklistOptions.setFlag(RuleOption::script, true);
+    } else if(opt.startsWith("~script")) {
+        m_whitelistOptions.setFlag(RuleOption::script, true);
+
+    } else if(opt.startsWith("image")) {
+        m_blacklistOptions.setFlag(RuleOption::image, true);
+    } else if(opt.startsWith("~image")) {
+        m_whitelistOptions.setFlag(RuleOption::image, true);
+
+    } else if(opt.startsWith("stylesheet")) {
+        m_blacklistOptions.setFlag(RuleOption::stylesheet, true);
+    } else if(opt.startsWith("~stylesheet")) {
+        m_whitelistOptions.setFlag(RuleOption::stylesheet, true);
+
+    } else if(opt.startsWith("object")) {
+        m_blacklistOptions.setFlag(RuleOption::object, true);
+    } else if(opt.startsWith("~object")) {
+        m_whitelistOptions.setFlag(RuleOption::object, true);
+
+    } else if(opt.startsWith("object-subrequest")) {
+        m_blacklistOptions.setFlag(RuleOption::objectsubrequest, true);
+    } else if(opt.startsWith("~object-subrequest")) {
+        m_whitelistOptions.setFlag(RuleOption::objectsubrequest, true);
+
+    } else if(opt.startsWith("subdocument")) {
+        m_blacklistOptions.setFlag(RuleOption::subdocument, true);
+    } else if(opt.startsWith("~subdocument")) {
+        m_whitelistOptions.setFlag(RuleOption::subdocument, true);
+    }
 }
 
 bool BlockerRule::matchOptions(const QWebEngineUrlRequestInfo &info, const RuleOptions &options)
diff --git a/src/blocker/blockerrule.h b/src/blocker/blockerrule.h
index 0981514..85a5dc1 100644
--- a/src/blocker/blockerrule.h
+++ b/src/blocker/blockerrule.h
@@ -30,6 +30,8 @@ class BlockerRule : public QObject
 {
     Q_OBJECT
 public:
+
+    // https://adblockplus.org/en/filters#options
     enum RuleOption {
         script = 1,
         image = 2,
@@ -41,14 +43,11 @@ public:
         ping = 128,
         websocket = 256,
         document = 512,
-
         elemhide = 1024,
         generichide = 2048,
         genericblock = 4096,
-
         other = 8192
     };
-
     Q_DECLARE_FLAGS(RuleOptions, RuleOption)
 
     explicit BlockerRule(QString rule, QObject *parent = 0);
@@ -56,17 +55,22 @@ public:
     bool match(const QWebEngineUrlRequestInfo &info);
     bool isValid();
     bool isException();
-    QString toString() const;
+    QString filter() const;
 
 signals:
 
 public slots:
 
 private:
+    void parseOption(const QString &opt);
     bool matchOptions(const QWebEngineUrlRequestInfo &info, const RuleOptions &options);
 
+    QString m_filter;
+
     bool m_valid;
     bool m_exception = false;
+    bool m_elementRule;
+
     QStringList hostsBlacklist;
     QStringList hostsWhitelist;
     RegExp domainExpression;
diff --git a/src/blocker/blockersubscription.cpp b/src/blocker/blockersubscription.cpp
index 02def48..86cb0b5 100644
--- a/src/blocker/blockersubscription.cpp
+++ b/src/blocker/blockersubscription.cpp
@@ -22,35 +22,32 @@
 #include "ui_subscriptionform.h"
 
 #include "browser.h"
-#include <QFile>
 #include <QNetworkRequest>
 #include <QNetworkReply>
 
-BlockerSubscription::BlockerSubscription(const QString url, QWidget *parent) :
+BlockerSubscription::BlockerSubscription(const QUrl url, QWidget *parent) :
     QWidget(parent),
     ui(new Ui::SubscriptionForm)
 {
     ui->setupUi(this);
-    QUrl _url = QUrl::fromUserInput(url);
-    m_name = _url.fileName();
-
-    if(!sSettings->value("blocker.path").toString().isEmpty()) {
-        QString cacheName = sSettings->value("blocker.path").toString() + m_name;
-        QFile *cache = new QFile(cacheName);
-        if(cache->exists()) {
-            load(cache);
-        }
-
+    m_name = url.fileName();
+    m_url = url;
+
+    QString subPath = sSettings->value("blocker.path").toString() + m_name;
+    qDebug("Adding subscription [%s]", qUtf8Printable(subPath));
+    QFile *sub = new QFile(subPath);
+    if(sub->exists()) {
+        sub->open(QIODevice::ReadOnly);
+        load(sub);
     } else {
-
-        // no cache path specified - pull the subscription
-        QNetworkRequest request;
-        request.setUrl(QUrl::fromUserInput(url));
-
-        QNetworkReply *reply = sNetwork->get(request);
-        connect(reply, &QNetworkReply::finished, [this, reply]() {
-            this->load(reply);
-        });
+        if(!url.scheme().startsWith("http")) {
+            qWarning("Invalid url, subscription cannot be updated");
+            sub->deleteLater();
+            return;
+        }
+        qDebug("Subscription doesn't exist, updating...");
+        sub->open(QIODevice::ReadWrite);
+        update(sub);
     }
 }
 
@@ -64,14 +61,69 @@ QString BlockerSubscription::name() const
     return m_name;
 }
 
+/**
+ * Check if a URL request should be blocked or not
+ * @param info
+ * @return true if it should be blocked; false otherwise
+ */
+BlockerSubscription::MatchResult BlockerSubscription::match(QWebEngineUrlRequestInfo &info)
+{
+    MatchResult result;
+
+    for(BlockerRule* rule : m_urlWhitelist) {
+        if(rule->match(info)) {
+            // this request is whitelisted
+            result.match = true;
+            result.block = false;
+            result.pattern = rule->filter();
+            return result;
+        }
+    }
+
+    // request is not in the whitelist
+    for(BlockerRule* rule : m_urlBlacklist) {
+        if(rule->match(info)) {
+            // this request is blacklisted
+            result.match = true;
+            result.block = true;
+            result.pattern = rule->filter();
+            return result;
+        }
+    }
+
+    // request matches neither whitelist nor blacklist
+    result.match = false;
+    result.block = false;
+    return result;
+}
+
+void BlockerSubscription::update(QFile *cache)
+{
+    // no cache path specified - pull the subscription
+    QNetworkRequest request;
+    request.setUrl(m_url);
+
+    QNetworkReply *reply = sNetwork->get(request);
+    connect(reply, &QNetworkReply::readyRead, [this, reply, cache]() {
+        cache->write(reply->readAll());
+    });
+    connect(reply, &QNetworkReply::finished, [this, reply, cache]() {
+        cache->write(reply->readAll());
+        cache->flush();
+        cache->seek(0);
+        reply->deleteLater();
+        qDebug("Subscription updated: [%s]", qUtf8Printable(this->m_name));
+        this->load(cache);
+    });
+}
+
 void BlockerSubscription::load(QIODevice *dev)
 {
     QTextStream subscription(dev);
 
     QString header = subscription.readLine();
     if(!header.startsWith("[Adblock Plus")) {
-        qDebug("Invalid format of subscription: %s", qUtf8Printable(m_name));
-        return;
+        qWarning("Invalid format of subscription: %s", qUtf8Printable(m_name));
     }
 
     // clear all lists
@@ -85,24 +137,23 @@ void BlockerSubscription::load(QIODevice *dev)
             if(line.startsWith('!')) {
                 parseComment(line);
             } else {
-                // The line is not a comment
-
+                // The line is not empty or a comment
                 rules++;
                 BlockerRule *rule = new BlockerRule(line, this);
 
                 if(rule->isValid()) {
                     if(rule->isException()) {
                         m_urlWhitelist.append(rule);
-                        ui->whitelist_listWidget->addItem(rule->toString());
+                        ui->whitelist_listWidget->addItem(rule->filter());
                     } else {
-                        ui->blacklist_listWidget->addItem(rule->toString());
+                        ui->blacklist_listWidget->addItem(rule->filter());
                         m_urlBlacklist.append(rule);
                     }
                 }
 
             }
-        }
-    }
+        } // line.isEmpty
+    } // subscription.atEnd()
 
     qDebug("Loaded %i/%i rules from subscription %s", m_urlBlacklist.count() + m_urlWhitelist.count(), rules, qUtf8Printable(m_name));
     dev->deleteLater();
diff --git a/src/blocker/blockersubscription.h b/src/blocker/blockersubscription.h
index fddb93d..918a4dc 100644
--- a/src/blocker/blockersubscription.h
+++ b/src/blocker/blockersubscription.h
@@ -22,6 +22,7 @@
 #define SUBSCRIPTIONFORM_H
 
 #include <QWidget>
+#include <QFile>
 #include "blocker/blockerrule.h"
 
 namespace Ui {
@@ -33,12 +34,20 @@ class BlockerSubscription : public QWidget
     Q_OBJECT
 
 public:
-    explicit BlockerSubscription(const QString url, QWidget *parent = 0);
+    struct MatchResult {
+        bool match;
+        bool block;
+        QString pattern;
+    };
+
+    explicit BlockerSubscription(const QUrl url, QWidget *parent = 0);
     ~BlockerSubscription();
 
     QString name() const;
+    MatchResult match(QWebEngineUrlRequestInfo &info);
 
 private slots:
+    void update(QFile *cache);
     void load(QIODevice *dev);
 
 private:
@@ -46,6 +55,7 @@ private:
 
     Ui::SubscriptionForm *ui;
     QString m_name;
+    QUrl m_url;
 
     QList<BlockerRule*> m_urlWhitelist; // exception rules
     QList<BlockerRule*> m_urlBlacklist; // block rules
diff --git a/src/webengine/urlinterceptor.cpp b/src/webengine/urlinterceptor.cpp
index 3c6a56e..94bfffa 100644
--- a/src/webengine/urlinterceptor.cpp
+++ b/src/webengine/urlinterceptor.cpp
@@ -19,6 +19,7 @@
  ******************************************************************************/
 
 #include "urlinterceptor.h"
+#include "blocker/blockersubscription.h"
 
 UrlRequestInterceptor::UrlRequestInterceptor(QObject *parent) :
     QWebEngineUrlRequestInterceptor(parent)
@@ -27,10 +28,23 @@ UrlRequestInterceptor::UrlRequestInterceptor(QObject *parent) :
 
 void UrlRequestInterceptor::interceptRequest(QWebEngineUrlRequestInfo &info)
 {
-    //
+#ifdef DEBUG_VERBOSE
+    qDebug("%s", qUtf8Printable(info.requestUrl().toString()));
+#endif
+
+    for(BlockerSubscription *s : m_manager->subscriptions()) {
+        BlockerSubscription::MatchResult r = s->match(info);
+        if(r.match) {
+            info.block(r.block);
+#ifdef DEBUG_VERBOSE
+            qDebug("+--> matched [%s] [%s]", r.block ? "blocked" : "allowed", qUtf8Printable(r.pattern));
+#endif
+            return;
+        }
+    }
 }
 
-void UrlRequestInterceptor::setSubscription(BlockerManager *subscription)
+void UrlRequestInterceptor::setSubscription(BlockerManager *manager)
 {
-    m_blocker = subscription;
+    m_manager = manager;
 }
diff --git a/src/webengine/urlinterceptor.h b/src/webengine/urlinterceptor.h
index 00fe666..2e5d6bf 100644
--- a/src/webengine/urlinterceptor.h
+++ b/src/webengine/urlinterceptor.h
@@ -31,14 +31,14 @@ public:
     explicit UrlRequestInterceptor(QObject *parent = 0);
 
     void interceptRequest(QWebEngineUrlRequestInfo &info);
-    void setSubscription(BlockerManager *subscription);
+    void setSubscription(BlockerManager *manager);
 
 signals:
 
 public slots:
 
 private:
-    BlockerManager *m_blocker;
+    BlockerManager *m_manager;
 };
 
 #endif // ADBLOCKINTERCEPTOR_H