From f0d67a082a0f28d7b4380d3b6a88f1041bfae8bb Mon Sep 17 00:00:00 2001 From: Aqua-sama Date: Fri, 14 Feb 2020 11:44:38 +0200 Subject: staging: add filterlist parallel downloader --- meson.build | 1 + staging/adblock/filterlist.h | 2 +- staging/filterlist/downloadmanager.cpp | 53 ++++++++++++++++++++++++++++++ staging/filterlist/downloadmanager.h | 21 ++++++++++++ staging/filterlist/meson.build | 15 +++++++++ staging/filterlist/test/main.cpp | 44 +++++++++++++++++++++++++ staging/filterlist/test/sample-filters.txt | 12 +++++++ 7 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 staging/filterlist/downloadmanager.cpp create mode 100644 staging/filterlist/downloadmanager.h create mode 100644 staging/filterlist/meson.build create mode 100644 staging/filterlist/test/main.cpp create mode 100644 staging/filterlist/test/sample-filters.txt diff --git a/meson.build b/meson.build index 8619d5b..5a62bc1 100644 --- a/meson.build +++ b/meson.build @@ -94,6 +94,7 @@ subdir('tools') subdir('test/firefox-bookmarks-json-parser') subdir('test/matcherbenchmark') +subdir('staging/filterlist') subdir('staging/adblock') ssconfig = poi_sourceset.apply(cdata) diff --git a/staging/adblock/filterlist.h b/staging/adblock/filterlist.h index 801700f..3ee18b3 100644 --- a/staging/adblock/filterlist.h +++ b/staging/adblock/filterlist.h @@ -18,7 +18,7 @@ namespace AdblockPlus class Rule; class FilterList : public QObject { - Q_DISABLE_COPY(FilterList); + Q_DISABLE_COPY(FilterList) public: explicit FilterList(QObject *parent = nullptr); diff --git a/staging/filterlist/downloadmanager.cpp b/staging/filterlist/downloadmanager.cpp new file mode 100644 index 0000000..bbc9287 --- /dev/null +++ b/staging/filterlist/downloadmanager.cpp @@ -0,0 +1,53 @@ +/* + * This file is part of smolbote. It's copyrighted by the contributors recorded + * in the version control history of the file, available from its original + * location: https://library.iserlohn-fortress.net/aqua/smolbote.git + * + * SPDX-License-Identifier: GPL-3.0 + */ + +#include "downloadmanager.h" + +QString saveFileName(const QUrl &url, const QString &filename) +{ + if(!filename.isEmpty()) + return filename; + + QString path = url.path(); + QString basename = QFileInfo(path).fileName(); + + if(basename.isEmpty()) + basename = "download"; + + return basename; +} + +DownloadManager::DownloadManager(QObject *parent) + : QObject(parent) +{ +} + +QNetworkReply *DownloadManager::download(const QUrl &url, const QString &filename) +{ + auto *file = new QFile(saveFileName(url, filename), this); + if(!file->open(QIODevice::WriteOnly)) { + delete file; + return nullptr; + } + + QNetworkRequest request(url); + QNetworkReply *reply = manager.get(request); + + connect(reply, &QNetworkReply::readyRead, this, [reply, file]() { + file->write(reply->readAll()); + }); + + connect(reply, &QNetworkReply::finished, this, [reply, file]() { + file->write(reply->readAll()); + file->close(); + file->deleteLater(); + reply->deleteLater(); + }); + + return reply; +} diff --git a/staging/filterlist/downloadmanager.h b/staging/filterlist/downloadmanager.h new file mode 100644 index 0000000..edc011d --- /dev/null +++ b/staging/filterlist/downloadmanager.h @@ -0,0 +1,21 @@ +/* + * This file is part of smolbote. It's copyrighted by the contributors recorded + * in the version control history of the file, available from its original + * location: https://library.iserlohn-fortress.net/aqua/smolbote.git + * + * SPDX-License-Identifier: GPL-3.0 + */ + +#include + +class DownloadManager : public QObject +{ + Q_OBJECT + +public: + DownloadManager(QObject *parent = nullptr); + QNetworkReply *download(const QUrl &url, const QString &filename = QString()); + +private: + QNetworkAccessManager manager; +}; diff --git a/staging/filterlist/meson.build b/staging/filterlist/meson.build new file mode 100644 index 0000000..b8f2263 --- /dev/null +++ b/staging/filterlist/meson.build @@ -0,0 +1,15 @@ +dep_staging_utils = declare_dependency( + include_directories: include_directories('.'), + link_with: static_library('staging-utils', + [ 'downloadmanager.cpp', + mod_qt5.preprocess(moc_headers: 'downloadmanager.h') + ], + dependencies: dep_qt5 + ) +) + +executable('filterlist', + dependencies: [ dep_qt5, dep_staging_utils ], + sources: [ 'test/main.cpp' ] +) + diff --git a/staging/filterlist/test/main.cpp b/staging/filterlist/test/main.cpp new file mode 100644 index 0000000..ccaae47 --- /dev/null +++ b/staging/filterlist/test/main.cpp @@ -0,0 +1,44 @@ +#include "downloadmanager.h" +#include + +int main(int argc, char **argv) +{ + if(argc != 2) { + qDebug("Usage: %s filters.txt", argv[0]); + return 77; + } + + QCoreApplication app(argc, argv); + + DownloadManager manager; + QSettings listconf(argv[1], QSettings::IniFormat); + + QVector downloads; + + qDebug("Filters:"); + for(auto &g : listconf.childGroups()) { + listconf.beginGroup(g); + const auto url = listconf.value("Href").toUrl(); + qDebug("|%s |%s|", qUtf8Printable(g.leftJustified(16, ' ', true)), qUtf8Printable(listconf.value("Href").toString().leftJustified(100, ' ', true))); + auto *reply = manager.download(url); + downloads.append(reply); + + QObject::connect(reply, &QNetworkReply::finished, [&downloads, reply]() { + if(reply->error() == QNetworkReply::NoError) { + qDebug("downloaded %s", qUtf8Printable(reply->url().toString())); + } else { + qDebug("failed %s", qUtf8Printable(reply->url().toString())); + qDebug("error [%i]: %s", reply->error(), qUtf8Printable(reply->errorString())); + } + + downloads.removeAll(reply); + if(downloads.isEmpty()) { + QCoreApplication::instance()->quit(); + } + }); + listconf.endGroup(); + } + qDebug("---"); + + return app.exec(); +} diff --git a/staging/filterlist/test/sample-filters.txt b/staging/filterlist/test/sample-filters.txt new file mode 100644 index 0000000..a7c04fa --- /dev/null +++ b/staging/filterlist/test/sample-filters.txt @@ -0,0 +1,12 @@ +[easylist] +Title = +Href = https://easylist.to/easylist/easylist.txt +Updated = +Valid = + +[easylist-noelemhide] +Title = +Href = https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt +Updated = +Valid = + -- cgit v1.2.1