Rewrite lib/urlfilter

- Make HostList and AdBlockList implementations independent from each
other
- Move urlfilter tests to lib/urlfilter

7 files changed, 79 insertions, 140 deletions

diff --git a/src/browser.cpp b/src/browser.cpp
index 42bbc5d..3a23eeb 100644
--- a/src/browser.cpp
+++ b/src/browser.cpp
@@ -18,7 +18,6 @@
 #include "profilemanager.h"
 #include "subwindow/subwindow.h"
 #include "util.h"
-#include "webengine/filter.h"
 #include "webengine/urlinterceptor.h"
 #include "webprofile.h"
 #include <QAction>
@@ -35,6 +34,9 @@
 #include <version.h>
 #include "mainwindow/menubar.h"
 #include "webengine/webview.h"
+#include "urlfilter.h"
+#include "adblock/adblocklist.h"
+#include "hostlist/hostlist.h"
 
 Browser::Browser(int &argc, char *argv[], bool allowSecondary)
     : SingleApplication(argc, argv, allowSecondary, SingleApplication::User | SingleApplication::SecondaryNotification | SingleApplication::ExcludeAppVersion)
@@ -99,7 +101,16 @@ QPair<QString, Profile *> Browser::loadProfile(const QString &id, bool isOffTheR
         profile = m_profileManager->createProfile(id, isOffTheRecord);
     }
     connect(profile, &WebProfile::downloadRequested, m_downloads.get(), &DownloadsWidget::addDownload);
-    auto *interceptor = new UrlRequestInterceptor(m_urlFilter.get(), profile, profile);
+    auto *interceptor = new UrlRequestInterceptor(profile, profile);
+    for(UrlFilter *filter : m_filters) {
+        interceptor->addFilter(filter);
+    }
+    const auto headers = m_config->value<QStringList>("filter.header").value_or(QStringList());
+    for(const QString &header : headers) {
+        const auto h = header.split(QLatin1Literal(":"));
+        if(h.length() == 2)
+            interceptor->addHttpHeader(h.at(0).toLatin1(), h.at(1).toLatin1());
+    }
     profile->setRequestInterceptor(interceptor);
 
     return QPair<QString, WebProfile *>(m_profileManager->id(profile), profile);
@@ -164,7 +175,20 @@ void Browser::setup(QVector<QPluginLoader *> plugins)
     // downloads
     m_downloads = std::make_unique<DownloadsWidget>(m_config->value<QString>("downloads.path").value());
     // url request filter
-    m_urlFilter = std::make_unique<Filter>(m_config);
+    for(const QString &hostlist : Util::files(m_config->value<QString>("filter.hosts").value_or(QString()))) {
+        QFile f(hostlist);
+        if(f.open(QIODevice::ReadOnly | QIODevice::Text)) {
+            m_filters.append(new HostList(&f));
+            f.close();
+        }
+    }
+    for(const QString &adblock : Util::files(m_config->value<QString>("filter.adblock").value_or(QString()))) {
+        QFile f(adblock);
+        if(f.open(QIODevice::ReadOnly | QIODevice::Text)) {
+            m_filters.append(new AdBlockList(&f));
+            f.close();
+        }
+    }
     // cookie request filter
 
     // load profiles
diff --git a/src/browser.h b/src/browser.h
index 53ee521..8a40152 100644
--- a/src/browser.h
+++ b/src/browser.h
@@ -19,10 +19,10 @@
 #include <QMenu>
 #include <QPluginLoader>
 
+class UrlFilter;
 class Configuration;
 class BookmarksWidget;
 class DownloadsWidget;
-class Filter;
 class MainWindow;
 class ProfileManager;
 class Browser : public SingleApplication, public BrowserInterface
@@ -91,7 +91,7 @@ private:
     std::shared_ptr<BookmarksWidget> m_bookmarks;
     std::unique_ptr<DownloadsWidget> m_downloads;
     ProfileManager *m_profileManager;
-    std::unique_ptr<Filter> m_urlFilter;
+    QVector<UrlFilter *> m_filters;
 
     QVector<MainWindow *> m_windows;
     QVector<PluginInfo*> m_plugins;
diff --git a/src/meson.build b/src/meson.build
index f07a2ec..fb338d8 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -4,7 +4,7 @@ poi_moc = qt5.preprocess(
         'mainwindow/mainwindow.h', 'mainwindow/menubar.h', 'mainwindow/widgets/dockwidget.h', 'mainwindow/widgets/menusearch.h', 'mainwindow/widgets/navigationbar.h', 'mainwindow/widgets/searchform.h',
         'session/savesessiondialog.h', 'session/sessiondialog.h', 'session/sessionform.h',
         'subwindow/subwindow.h', 'subwindow/tabwidget.h',
-        'webengine/filter.h', 'webengine/urlinterceptor.h', 'webengine/webpage.h', 'webengine/webview.h'],
+        'webengine/urlinterceptor.h', 'webengine/webpage.h', 'webengine/webview.h'],
     ui_files: ['mainwindow/widgets/searchform.ui', 'session/savesessiondialog.ui', 'session/sessiondialog.ui', 'session/sessionform.ui'],
     qresources: '../data/resources.qrc',
     rcc_extra_arguments: ['--format-version=1'],
@@ -35,7 +35,6 @@ poi = executable(get_option('poiName'), install: true,
     'subwindow/subwindow.cpp',
     'subwindow/tabwidget.cpp',
 
-    'webengine/filter.cpp',
     'webengine/urlinterceptor.cpp',
     'webengine/webpage.cpp',
     'webengine/webview.cpp',
diff --git a/src/webengine/filter.cpp b/src/webengine/filter.cpp
deleted file mode 100644
index f1a38af..0000000
--- a/src/webengine/filter.cpp
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * This file is part of smolbote. It's copyrighted by the contributors recorded
- * in the version control history of the file, available from its original
- * location: https://neueland.iserlohn-fortress.net/gitea/aqua/smolbote
- *
- * SPDX-License-Identifier: GPL-3.0
- */
-
-#include "filter.h"
-#include "configuration.h"
-#include "urlinterceptor.h"
-#include "util.h"
-#include <QDir>
-#include <QJsonArray>
-#include <QJsonDocument>
-#include <QTextStream>
-
-Filter::Filter::Filter(const std::unique_ptr<Configuration> &config, QObject *parent)
-    : QObject(parent)
-{
-    // parse headers
-    if(config->exists("filter.header")) {
-        const auto headers = config->value<QStringList>("filter.header").value();
-        for(const QString header : headers) {
-            const auto list = header.split(QLatin1Literal(":"));
-            if(list.length() == 2)
-                m_headers.insert(list.at(0).toLatin1(), list.at(1).toLatin1());
-        }
-#ifdef QT_DEBUG
-        qDebug("Added %i custom http headers", m_headers.size());
-#endif
-    }
-
-    const QStringList hostfiles = Util::files(config->value<QString>("filter.hosts").value());
-    //qDebug("filter.path=[%s]", qUtf8Printable(config->value<QString>("filter.hosts").value()));
-    for(const QString &hostfile : hostfiles) {
-        QFile f(hostfile);
-        if(f.open(QIODevice::ReadOnly | QIODevice::Text)) {
-#ifdef QT_DEBUG
-            qDebug("Loading hostlist filters [%s]", qUtf8Printable(hostfile));
-#endif
-            loadHostlist(f, &filters);
-            f.close();
-        }
-    }
-}
-
-void Filter::filterRequest(QWebEngineUrlRequestInfo &info) const
-{
-    auto matches = filters.match(info.firstPartyUrl().toString(), info.requestUrl().toString());
-    for(const auto &rule : matches) {
-        switch(rule->action().first) {
-        case FilterLeaf::NotMatched:
-#ifdef QT_DEBUG
-            qDebug("Paradoxical match: request matched, but not matched.");
-            qDebug(" - %s", qUtf8Printable(info.requestUrl().toString()));
-#endif
-            break;
-        case FilterLeaf::Block:
-            //qDebug("block %s", qUtf8Printable(info.requestUrl().toString()));
-            info.block(true);
-            break;
-        case FilterLeaf::Allow:
-            info.block(false);
-            break;
-        //case FilterLeaf::Redirect:
-        //    break;
-        }
-    }
-}
diff --git a/src/webengine/filter.h b/src/webengine/filter.h
deleted file mode 100644
index c49bed9..0000000
--- a/src/webengine/filter.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * This file is part of smolbote. It's copyrighted by the contributors recorded
- * in the version control history of the file, available from its original
- * location: https://neueland.iserlohn-fortress.net/gitea/aqua/smolbote
- *
- * SPDX-License-Identifier: GPL-3.0
- */
-
-#ifndef SMOLBOTE_FILTER_H
-#define SMOLBOTE_FILTER_H
-
-#include <QByteArray>
-#include <QMap>
-#include <QVector>
-#include <optional>
-#include <memory>
-#include "filtertree.h"
-
-class Configuration;
-class Filter : public QObject
-{
-    Q_OBJECT
-public:
-    struct HostRule {
-        bool isBlocking;
-    };
-
-    explicit Filter(const std::unique_ptr<Configuration> &config, QObject *parent = nullptr);
-    ~Filter() override = default;
-
-    void filterRequest(QWebEngineUrlRequestInfo &info) const;
-
-    const QMap<QByteArray, QByteArray> headers() const
-    {
-        return qAsConst(m_headers);
-    }
-
-private:
-    FilterTree filters;
-    QMap<QByteArray, QByteArray> m_headers;
-};
-
-#endif // SMOLBOTE_FILTER_H
diff --git a/src/webengine/urlinterceptor.cpp b/src/webengine/urlinterceptor.cpp
index 7e5630f..490dea6 100644
--- a/src/webengine/urlinterceptor.cpp
+++ b/src/webengine/urlinterceptor.cpp
@@ -7,36 +7,61 @@
  */
 
 #include "urlinterceptor.h"
-#include "formats/adblockrule.h"
-#include <QDir>
-#include <QJsonArray>
-#include <QJsonDocument>
-#include <QTextStream>
-#include <boost/algorithm/string.hpp>
-#include "configuration.h"
-#include "filter.h"
 #include "webprofile.h"
+#include "urlfilter.h"
 
 // test DNT on https://browserleaks.com/donottrack
 
-UrlRequestInterceptor::UrlRequestInterceptor(Filter* filter, WebProfile* profile, QObject* parent)
+UrlRequestInterceptor::UrlRequestInterceptor(WebProfile* profile, QObject* parent)
     : QWebEngineUrlRequestInterceptor(parent)
 {
-    Q_CHECK_PTR(filter);
-    m_filter = filter;
     Q_CHECK_PTR(profile);
     m_profile = profile;
 }
 
+void UrlRequestInterceptor::addHttpHeader(const QByteArray &key, const QByteArray &value)
+{
+    headers.append(qMakePair(key, value));
+}
+
+void UrlRequestInterceptor::addFilter(UrlFilter *filter)
+{
+    if(filter != nullptr)
+        filters.append(filter);
+}
+void UrlRequestInterceptor::removeFilter(UrlFilter *filter)
+{
+    if(filter != nullptr)
+        filters.removeOne(filter);
+}
+
 void UrlRequestInterceptor::interceptRequest(QWebEngineUrlRequestInfo &info)
 {
-    m_filter->filterRequest(info);
+    for(const auto *filter : filters) {
+        const auto match = filter->match(info.firstPartyUrl(), info.requestUrl(), info.resourceType());
+
+        // skip if no match
+        if(match.first == UrlFilter::NotMatched)
+            continue;
+
+        else {
+            if(match.first == UrlFilter::Allow)
+                info.block(false);
+            else if(match.first == UrlFilter::Block)
+                info.block(true);
+            else if(match.first == UrlFilter::Redirect)
+                info.redirect(QUrl::fromUserInput(match.second));
+            // we found a match, skip the rest
+            break;
+        }
+    }
 
     // set headers
-    for(auto i = m_filter->headers().constBegin(); i != m_filter->headers().constEnd(); ++i) {
-        info.setHttpHeader(i.key(), i.value());
+    for(const auto &header : headers) {
+        info.setHttpHeader(header.first, header.second);
     }
     for(auto i = m_profile->headers().constBegin(); i != m_profile->headers().constEnd(); ++i) {
         info.setHttpHeader(i.key(), i.value());
     }
 }
+
diff --git a/src/webengine/urlinterceptor.h b/src/webengine/urlinterceptor.h
index 62fd683..4909586 100644
--- a/src/webengine/urlinterceptor.h
+++ b/src/webengine/urlinterceptor.h
@@ -9,26 +9,30 @@
 #ifndef SMOLBOTE_URLREQUESTINTERCEPTOR_H
 #define SMOLBOTE_URLREQUESTINTERCEPTOR_H
 
-#include <QByteArray>
 #include <QVector>
 #include <QWebEngineUrlRequestInterceptor>
-#include <memory>
+#include <QByteArray>
 
-class Filter;
+class UrlFilter;
 class WebProfile;
-class Configuration;
 class UrlRequestInterceptor : public QWebEngineUrlRequestInterceptor
 {
     Q_OBJECT
 public:
-    explicit UrlRequestInterceptor(Filter *filter, WebProfile *profile, QObject *parent = nullptr);
+    explicit UrlRequestInterceptor(WebProfile *profile, QObject *parent = nullptr);
     ~UrlRequestInterceptor() override = default;
 
+    void addHttpHeader(const QByteArray &key, const QByteArray &value);
+
+    void addFilter(UrlFilter *filter);
+    void removeFilter(UrlFilter *filter);
+
     void interceptRequest(QWebEngineUrlRequestInfo &info) override;
 
 private:
-    Filter *m_filter;
     WebProfile *m_profile;
+    QVector<QPair<QByteArray, QByteArray>> headers;
+    QVector<UrlFilter*> filters;
 };
 
 #endif // SMOLBOTE_URLREQUESTINTERCEPTOR_H