staging: add filterlist parallel downloader

7 files changed, 147 insertions, 1 deletions

diff --git a/meson.build b/meson.build
index 8619d5b..5a62bc1 100644
--- a/meson.build
+++ b/meson.build
@@ -94,6 +94,7 @@ subdir('tools')
 subdir('test/firefox-bookmarks-json-parser')
 subdir('test/matcherbenchmark')
 
+subdir('staging/filterlist')
 subdir('staging/adblock')
 
 ssconfig = poi_sourceset.apply(cdata)
diff --git a/staging/adblock/filterlist.h b/staging/adblock/filterlist.h
index 801700f..3ee18b3 100644
--- a/staging/adblock/filterlist.h
+++ b/staging/adblock/filterlist.h
@@ -18,7 +18,7 @@ namespace AdblockPlus
 class Rule;
 class FilterList : public QObject
 {
-    Q_DISABLE_COPY(FilterList);
+    Q_DISABLE_COPY(FilterList)
 
 public:
     explicit FilterList(QObject *parent = nullptr);
diff --git a/staging/filterlist/downloadmanager.cpp b/staging/filterlist/downloadmanager.cpp
new file mode 100644
index 0000000..bbc9287
--- /dev/null
+++ b/staging/filterlist/downloadmanager.cpp
@@ -0,0 +1,53 @@
+/*
+ * This file is part of smolbote. It's copyrighted by the contributors recorded
+ * in the version control history of the file, available from its original
+ * location: https://library.iserlohn-fortress.net/aqua/smolbote.git
+ *
+ * SPDX-License-Identifier: GPL-3.0
+ */
+
+#include "downloadmanager.h"
+
+QString saveFileName(const QUrl &url, const QString &filename)
+{
+    if(!filename.isEmpty())
+        return filename;
+
+    QString path = url.path();
+    QString basename = QFileInfo(path).fileName();
+
+    if(basename.isEmpty())
+        basename = "download";
+
+    return basename;
+}
+
+DownloadManager::DownloadManager(QObject *parent)
+    : QObject(parent)
+{
+}
+
+QNetworkReply *DownloadManager::download(const QUrl &url, const QString &filename)
+{
+    auto *file = new QFile(saveFileName(url, filename), this);
+    if(!file->open(QIODevice::WriteOnly)) {
+        delete file;
+        return nullptr;
+    }
+
+    QNetworkRequest request(url);
+    QNetworkReply *reply = manager.get(request);
+
+    connect(reply, &QNetworkReply::readyRead, this, [reply, file]() {
+        file->write(reply->readAll());
+    });
+
+    connect(reply, &QNetworkReply::finished, this, [reply, file]() {
+        file->write(reply->readAll());
+        file->close();
+        file->deleteLater();
+        reply->deleteLater();
+    });
+
+    return reply;
+}
diff --git a/staging/filterlist/downloadmanager.h b/staging/filterlist/downloadmanager.h
new file mode 100644
index 0000000..edc011d
--- /dev/null
+++ b/staging/filterlist/downloadmanager.h
@@ -0,0 +1,21 @@
+/*
+ * This file is part of smolbote. It's copyrighted by the contributors recorded
+ * in the version control history of the file, available from its original
+ * location: https://library.iserlohn-fortress.net/aqua/smolbote.git
+ *
+ * SPDX-License-Identifier: GPL-3.0
+ */
+
+#include <QtNetwork>
+
+class DownloadManager : public QObject
+{
+    Q_OBJECT
+
+public:
+    DownloadManager(QObject *parent = nullptr);
+    QNetworkReply *download(const QUrl &url, const QString &filename = QString());
+
+private:
+    QNetworkAccessManager manager;
+};
diff --git a/staging/filterlist/meson.build b/staging/filterlist/meson.build
new file mode 100644
index 0000000..b8f2263
--- /dev/null
+++ b/staging/filterlist/meson.build
@@ -0,0 +1,15 @@
+dep_staging_utils = declare_dependency(
+  include_directories: include_directories('.'),
+  link_with: static_library('staging-utils',
+    [ 'downloadmanager.cpp',
+      mod_qt5.preprocess(moc_headers: 'downloadmanager.h')
+    ],
+    dependencies: dep_qt5
+  )
+)
+
+executable('filterlist',
+  dependencies: [ dep_qt5, dep_staging_utils ],
+  sources: [ 'test/main.cpp' ]
+)
+
diff --git a/staging/filterlist/test/main.cpp b/staging/filterlist/test/main.cpp
new file mode 100644
index 0000000..ccaae47
--- /dev/null
+++ b/staging/filterlist/test/main.cpp
@@ -0,0 +1,44 @@
+#include "downloadmanager.h"
+#include <QtCore>
+
+int main(int argc, char **argv)
+{
+    if(argc != 2) {
+        qDebug("Usage: %s filters.txt", argv[0]);
+        return 77;
+    }
+
+    QCoreApplication app(argc, argv);
+
+    DownloadManager manager;
+    QSettings listconf(argv[1], QSettings::IniFormat);
+
+    QVector<QNetworkReply *> downloads;
+
+    qDebug("Filters:");
+    for(auto &g : listconf.childGroups()) {
+        listconf.beginGroup(g);
+        const auto url = listconf.value("Href").toUrl();
+        qDebug("|%s |%s|", qUtf8Printable(g.leftJustified(16, ' ', true)), qUtf8Printable(listconf.value("Href").toString().leftJustified(100, ' ', true)));
+        auto *reply = manager.download(url);
+        downloads.append(reply);
+
+        QObject::connect(reply, &QNetworkReply::finished, [&downloads, reply]() {
+            if(reply->error() == QNetworkReply::NoError) {
+                qDebug("downloaded %s", qUtf8Printable(reply->url().toString()));
+            } else {
+                qDebug("failed %s", qUtf8Printable(reply->url().toString()));
+                qDebug("error [%i]: %s", reply->error(), qUtf8Printable(reply->errorString()));
+            }
+
+            downloads.removeAll(reply);
+            if(downloads.isEmpty()) {
+                QCoreApplication::instance()->quit();
+            }
+        });
+        listconf.endGroup();
+    }
+    qDebug("---");
+
+    return app.exec();
+}
diff --git a/staging/filterlist/test/sample-filters.txt b/staging/filterlist/test/sample-filters.txt
new file mode 100644
index 0000000..a7c04fa
--- /dev/null
+++ b/staging/filterlist/test/sample-filters.txt
@@ -0,0 +1,12 @@
+[easylist]
+Title =
+Href = https://easylist.to/easylist/easylist.txt
+Updated =
+Valid =
+
+[easylist-noelemhide]
+Title =
+Href = https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt
+Updated =
+Valid =
+