From 6faa12680a0d7d2f13c2862628325ab65521004b Mon Sep 17 00:00:00 2001 From: Andrea Diamantini Date: Sat, 13 Mar 2010 23:24:45 +0100 Subject: Implemented automatic adblock update. This (squashed) commit adds this new feature in rekonq, letting people to simply "forgot" adblock and let rekonq do everything for it I added: - a new (rekonq) adblock widget - an asyncronous method to update rules from the net every TOT days - a better AdBlockManager management. What it is actually missing is the adp protocol support to add new subscriptions to adblock. This will come the next week. For now this part seems stable and needs just testing :) --- src/CMakeLists.txt | 2 + src/adblock/adblockmanager.cpp | 198 ++++++++++++++++++++++++++++++--------- src/adblock/adblockmanager.h | 18 +++- src/adblock/adblockrule.cpp | 6 ++ src/adblock/adblockrule.h | 4 +- src/rekonq.kcfg | 24 +++++ src/settings/adblockwidget.cpp | 158 +++++++++++++++++++++++++++++++ src/settings/adblockwidget.h | 56 +++++++++++ src/settings/settings_adblock.ui | 167 +++++++++++++++++++++++++++++++++ src/settings/settingsdialog.cpp | 22 +++-- src/webpage.cpp | 8 +- 11 files changed, 605 insertions(+), 58 deletions(-) create mode 100644 src/settings/adblockwidget.cpp create mode 100644 src/settings/adblockwidget.h create mode 100644 src/settings/settings_adblock.ui diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 00cfbf77..f05bbbdb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,7 @@ SET( rekonq_KDEINIT_SRCS rekonqpage/previewselectorbar.cpp #---------------------------------------- settings/settingsdialog.cpp + settings/adblockwidget.cpp #---------------------------------------- bookmarks/bookmarksmanager.cpp bookmarks/bookmarkspanel.cpp @@ -54,6 +55,7 @@ KDE4_ADD_UI_FILES( rekonq_KDEINIT_SRCS settings/settings_tabs.ui settings/settings_fonts.ui settings/settings_webkit.ui + settings/settings_adblock.ui cleardata.ui ) diff --git a/src/adblock/adblockmanager.cpp b/src/adblock/adblockmanager.cpp index 06f0d164..d2c2b013 100644 --- a/src/adblock/adblockmanager.cpp +++ b/src/adblock/adblockmanager.cpp @@ -28,6 +28,9 @@ #include "adblockmanager.h" #include "adblockmanager.moc" +// Auto Includes +#include "rekonq.h" + // Local Includes #include "adblocknetworkreply.h" #include "webpage.h" @@ -36,6 +39,7 @@ #include #include #include +#include // Qt Includes #include @@ -46,8 +50,8 @@ AdBlockManager::AdBlockManager(QObject *parent) : QObject(parent) , _isAdblockEnabled(false) , _isHideAdsEnabled(false) + , _index(0) { - loadSettings(); } @@ -58,52 +62,78 @@ AdBlockManager::~AdBlockManager() void AdBlockManager::loadSettings() { - KSharedConfig::Ptr config = KSharedConfig::openConfig("khtmlrc", KConfig::NoGlobals); - KConfigGroup cg( config, "Filter Settings" ); + _index = 0; + _buffer.clear(); + + _whiteList.clear(); + _blackList.clear(); + _hideList.clear(); + + _isAdblockEnabled = ReKonfig::adBlockEnabled(); + kDebug() << "ADBLOCK ENABLED = " << _isAdblockEnabled; + + // no need to load filters if adblock is not enabled :) + if(!_isAdblockEnabled) + return; + + // just to be sure.. + _isHideAdsEnabled = ReKonfig::hideAdsEnabled(); + + // local settings + KSharedConfig::Ptr config = KGlobal::config(); + KConfigGroup rulesGroup( config, "rules" ); + QStringList rules; + rules = rulesGroup.readEntry( "local-rules" , QStringList() ); + loadRules( rules ); - if ( cg.exists() ) + // ---------------------------------------------------------- + + QDateTime today = QDateTime::currentDateTime(); + QDateTime lastUpdate = ReKonfig::lastUpdate(); // the day of the implementation.. :) + int days = ReKonfig::updateInterval(); + + if( today > lastUpdate.addDays( days ) ) { - _isAdblockEnabled = cg.readEntry("Enabled", false); - _isHideAdsEnabled = cg.readEntry("Shrink", false); + ReKonfig::setLastUpdate( today ); + + updateNextSubscription(); + return; + } - // no need to load filters if adblock is not enabled :) - if(!_isAdblockEnabled) - return; + // else + QStringList names = ReKonfig::subscriptionNames(); + foreach(const QString &name, names) + { + rules = rulesGroup.readEntry( name + "-rules" , QStringList() ); + loadRules(rules); + } +} - _whiteList.clear(); - _blackList.clear(); - _hideList.clear(); - - QMap entryMap = cg.entryMap(); - QMap::ConstIterator it; - for( it = entryMap.constBegin(); it != entryMap.constEnd(); ++it ) - { - QString name = it.key(); - QString url = it.value(); - if (name.startsWith(QLatin1String("Filter"))) +void AdBlockManager::loadRules(const QStringList &rules) +{ + foreach(const QString &stringRule, rules) + { + // ! rules are comments + if( !stringRule.startsWith('!') && !stringRule.startsWith('[') && !stringRule.isEmpty() ) + { + // white rules + if( stringRule.startsWith( QLatin1String("@@") ) ) { - if(!url.startsWith('!')) - { - // white rules - if( url.startsWith( QLatin1String("@@") ) ) - { - AdBlockRule rule( url.mid(2) ); - _whiteList << rule; - continue; - } - - // hide (CSS) rules - if( url.startsWith( QLatin1String("##") ) ) - { - _hideList << url.mid(2); - continue; - } - - AdBlockRule rule( url ); - _blackList << rule; - } + AdBlockRule rule( stringRule.mid(2) ); + _whiteList << rule; + continue; } + + // hide (CSS) rules + if( stringRule.startsWith( QLatin1String("##") ) ) + { + _hideList << stringRule.mid(2); + continue; + } + + AdBlockRule rule( stringRule ); + _blackList << rule; } } } @@ -125,7 +155,9 @@ QNetworkReply *AdBlockManager::block(const QNetworkRequest &request) { if(filter.match(urlString)) { - kDebug() << "****ADBLOCK: WHITE RULE (@@) Matched: ***********" << urlString; + kDebug() << "****ADBLOCK: WHITE RULE (@@) Matched: ***********"; + kDebug() << "Filter exp: " << filter.pattern(); + kDebug() << "UrlString: " << urlString; return 0; } } @@ -135,7 +167,9 @@ QNetworkReply *AdBlockManager::block(const QNetworkRequest &request) { if(filter.match(urlString)) { - kDebug() << "****ADBLOCK: BLACK RULE Matched: ***********" << urlString; + kDebug() << "****ADBLOCK: BLACK RULE Matched: ***********"; + kDebug() << "Filter exp: " << filter.pattern(); + kDebug() << "UrlString: " << urlString; AdBlockNetworkReply *reply = new AdBlockNetworkReply(request, urlString, this); return reply; } @@ -164,8 +198,88 @@ void AdBlockManager::applyHidingRules(WebPage *page) foreach (QWebElement element, elements) { + kDebug() << "Hide element: " << element.localName(); element.setStyleProperty(QLatin1String("visibility"), QLatin1String("hidden")); element.removeFromDocument(); } } } + + +void AdBlockManager::updateNextSubscription() +{ + kDebug() << "OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO"; + + QStringList subUrlStrings = ReKonfig::subscriptionUrls(); + + if( _index < subUrlStrings.size() ) + { + QString urlString = subUrlStrings.at(_index); + kDebug() << "DOWNLOADING FROM " << urlString; + KUrl subUrl = KUrl( urlString ); + + KIO::TransferJob* job = KIO::get( subUrl , KIO::Reload , KIO::HideProgressInfo ); + connect(job, SIGNAL(data(KIO::Job*, const QByteArray&)), this, SLOT(subscriptionData(KIO::Job*, const QByteArray&))); + connect(job, SIGNAL(result(KJob*)), this, SLOT(slotResult(KJob*))); + + return; + } + + _index = 0; + _buffer.clear(); +} + + +void AdBlockManager::slotResult(KJob *job) +{ + kDebug() << "SLOTRESULT"; + if(job->error()) + return; + + QList list = _buffer.split('\n'); + QStringList ruleList; + foreach(const QByteArray &ba, list) + { + kDebug() << ba; + ruleList << QString(ba); + } + loadRules(ruleList); + saveRules(ruleList); + + _index++; + + // last.. + updateNextSubscription(); +} + + +void AdBlockManager::subscriptionData(KIO::Job* job, const QByteArray& data) +{ + kDebug() << "subscriptionData"; + Q_UNUSED(job) + + if (data.isEmpty()) + return; + + int oldSize = _buffer.size(); + _buffer.resize( _buffer.size() + data.size() ); + memcpy( _buffer.data() + oldSize, data.data(), data.size() ); +} + + +void AdBlockManager::saveRules(const QStringList &rules) +{ + QStringList cleanedRules; + foreach(const QString &r, rules) + { + if( !r.startsWith('!') && !r.startsWith('[') && !r.isEmpty() ) + cleanedRules << r; + } + + QStringList names = ReKonfig::subscriptionNames(); + QString name = names.at(_index) + "-rules"; + + KSharedConfig::Ptr config = KGlobal::config(); + KConfigGroup cg( config , "rules" ); + cg.writeEntry( name, cleanedRules ); +} diff --git a/src/adblock/adblockmanager.h b/src/adblock/adblockmanager.h index d1bbe7ba..aac78e3b 100644 --- a/src/adblock/adblockmanager.h +++ b/src/adblock/adblockmanager.h @@ -112,6 +112,8 @@ #include #include #include +#include +#include // Forward Includes class QNetworkRequest; @@ -129,9 +131,20 @@ public: AdBlockManager(QObject *parent = 0); ~AdBlockManager(); - void loadSettings(); QNetworkReply *block(const QNetworkRequest &request); void applyHidingRules(WebPage *page); + +public slots: + void loadSettings(); + +private: + void updateNextSubscription(); + void saveRules(const QStringList &); + void loadRules(const QStringList &); + +private slots: + void slotResult(KJob *); + void subscriptionData(KIO::Job*, const QByteArray&); private: bool _isAdblockEnabled; @@ -140,6 +153,9 @@ private: AdBlockRuleList _blackList; AdBlockRuleList _whiteList; QStringList _hideList; + + int _index; + QByteArray _buffer; }; #endif diff --git a/src/adblock/adblockrule.cpp b/src/adblock/adblockrule.cpp index 25ca3678..c0c3fd5b 100644 --- a/src/adblock/adblockrule.cpp +++ b/src/adblock/adblockrule.cpp @@ -181,3 +181,9 @@ QString AdBlockRule::convertPatternToRegExp(const QString &wildcardPattern) // Finally, return... return pattern; } + + +QString AdBlockRule::pattern() const +{ + return m_regExp.pattern(); +} diff --git a/src/adblock/adblockrule.h b/src/adblock/adblockrule.h index 8f79d16e..ee4825d1 100644 --- a/src/adblock/adblockrule.h +++ b/src/adblock/adblockrule.h @@ -69,7 +69,9 @@ public: AdBlockRule(const QString &filter); bool match(const QString &encodedUrl) const; - + + QString pattern() const; + private: QString convertPatternToRegExp(const QString &wildcardPattern); diff --git a/src/rekonq.kcfg b/src/rekonq.kcfg index bbf20ac1..42ace3d3 100644 --- a/src/rekonq.kcfg +++ b/src/rekonq.kcfg @@ -7,6 +7,7 @@ QtWebKit +QDateTime KUrl @@ -151,4 +152,27 @@ + + + + + true + + + true + + + easylist + + + https://easylist-downloads.adblockplus.org/easylist.txt + + + QDateTime(QDate(2009,03,13)) + + + 7 + + + diff --git a/src/settings/adblockwidget.cpp b/src/settings/adblockwidget.cpp new file mode 100644 index 00000000..45e5fd5c --- /dev/null +++ b/src/settings/adblockwidget.cpp @@ -0,0 +1,158 @@ +/* ============================================================ +* +* This file is a part of the rekonq project +* +* Copyright (C) 2010 by Andrea Diamantini +* +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License as +* published by the Free Software Foundation; either version 2 of +* the License or (at your option) version 3 or any later version +* accepted by the membership of KDE e.V. (or its successor approved +* by the membership of KDE e.V.), which shall act as a proxy +* defined in Section 14 of version 3 of the license. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +* ============================================================ */ + + +#include "adblockwidget.h" +#include "adblockwidget.moc" + +#include "rekonq.h" + +#include +#include +#include + +#include +#include +#include + + +AdBlockWidget::AdBlockWidget(QWidget *parent) + : QWidget(parent) +{ + setupUi(this); + + hintLabel->setText( i18n("Filter expression (e.g. http://www.example.com/ad/*, more information):") ); + connect( hintLabel, SIGNAL(linkActivated(const QString &)), this, SLOT(slotInfoLinkActivated(const QString &)) ); + + listWidget->setSortingEnabled(true); + listWidget->setSelectionMode(QAbstractItemView::SingleSelection); + + searchLine->setListWidget(listWidget); + + insertButton->setIcon( KIcon("list-add") ); + connect( insertButton, SIGNAL( clicked() ), this, SLOT( insertRule() ) ); + + removeButton->setIcon( KIcon("list-remove") ); + connect( removeButton, SIGNAL( clicked() ), this, SLOT( removeRule() ) ); + + load(); +} + + +void AdBlockWidget::slotInfoLinkActivated(const QString &url) +{ + Q_UNUSED(url) + + QString hintHelpString = i18n("

Enter an expression to filter. Filters can be defined as either:" + "

  • a shell-style wildcard, e.g. http://www.example.com/ads*, the wildcards *?[] may be used
    • " + "
  • a full regular expression by surrounding the string with '/', e.g. /\\/(ad|banner)\\./
" + "

Any filter string can be preceded by '@@' to whitelist (allow) any matching URL, " + "which takes priority over any blacklist (blocking) filter."); + + QWhatsThis::showText( QCursor::pos(), hintHelpString ); +} + + +void AdBlockWidget::insertRule() +{ + QString rule = addFilterLineEdit->text(); + if(rule.isEmpty()) + return; + + listWidget->addItem( rule ); + addFilterLineEdit->clear(); +} + + +void AdBlockWidget::removeRule() +{ + listWidget->takeItem( listWidget->currentRow() ); +} + + +void AdBlockWidget::load() +{ + bool isAdBlockEnabled = ReKonfig::adBlockEnabled(); + checkEnableAdblock->setChecked(isAdBlockEnabled); + + bool areImageFiltered = ReKonfig::hideAdsEnabled(); + checkHideImages->setChecked(areImageFiltered); + + QStringList subscriptions = ReKonfig::subscriptionNames(); + + // load automatic rules + foreach(QString sub, subscriptions) + { + QTreeWidgetItem *subItem = new QTreeWidgetItem(treeWidget); + subItem->setText(0, sub); + loadRules(subItem); + } + + // load local rules + KSharedConfig::Ptr config = KGlobal::config(); + KConfigGroup localGroup( config, "rules" ); + QStringList rules = localGroup.readEntry( "local-rules" , QStringList() ); + foreach(const QString &rule, rules) + { + listWidget->addItem( rule ); + } +} + + +void AdBlockWidget::loadRules(QTreeWidgetItem *item) +{ + KSharedConfig::Ptr config = KGlobal::config(); + KConfigGroup localGroup( config, "rules" ); + + QString str = item->text(0) + "-rules"; + kDebug() << str; + QStringList rules = localGroup.readEntry( str , QStringList() ); + + foreach(const QString &rule, rules) + { + QTreeWidgetItem *subItem = new QTreeWidgetItem(item); + subItem->setText(0, rule); + } +} + + +void AdBlockWidget::save() +{ + int n; + + // local rules + KSharedConfig::Ptr config = KGlobal::config(); + KConfigGroup localGroup( config , "rules" ); + + QStringList localRules; + + n = listWidget->count(); + for(int i = 0; i < n; ++i) + { + QListWidgetItem *item = listWidget->takeItem(i); + localRules << item->text(); + } + localGroup.writeEntry( "local-rules" , localRules ); +} diff --git a/src/settings/adblockwidget.h b/src/settings/adblockwidget.h new file mode 100644 index 00000000..67061800 --- /dev/null +++ b/src/settings/adblockwidget.h @@ -0,0 +1,56 @@ +/* ============================================================ +* +* This file is a part of the rekonq project +* +* Copyright (C) 2010 by Andrea Diamantini +* +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License as +* published by the Free Software Foundation; either version 2 of +* the License or (at your option) version 3 or any later version +* accepted by the membership of KDE e.V. (or its successor approved +* by the membership of KDE e.V.), which shall act as a proxy +* defined in Section 14 of version 3 of the license. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +* ============================================================ */ + + +#ifndef ADBLOCK_WIDGET_H +#define ADBLOCK_WIDGET_H + + +#include "ui_settings_adblock.h" + +#include +#include + + +class AdBlockWidget : public QWidget, private Ui::adblock +{ +Q_OBJECT + +public: + AdBlockWidget(QWidget *parent = 0); + + void save(); + +private slots: + void slotInfoLinkActivated(const QString &); + void insertRule(); + void removeRule(); + +private: + void load(); + void loadRules(QTreeWidgetItem *item); +}; + +#endif // ADBLOCK_WIDGET_H diff --git a/src/settings/settings_adblock.ui b/src/settings/settings_adblock.ui new file mode 100644 index 00000000..28e1f973 --- /dev/null +++ b/src/settings/settings_adblock.ui @@ -0,0 +1,167 @@ + + + adblock + + + + 0 + 0 + 601 + 507 + + + + Form + + + + + + &Enable AdBlock + + + + + + + &Hide filtered Elements + + + + + + + + + + 0 + + + + Automatic Filters + + + + + + + 1 + + + + + + + + + + Automatic update interval: + + + + + + + 7 + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + Manual Filters + + + + + + + + Search: + + + + + + + + + + + + + + + TextLabel + + + + + + + + + + + + ... + + + + + + + ... + + + + + + + + + + + + + + KListWidget + QListWidget +

klistwidget.h
+ + + KLineEdit + QLineEdit +
klineedit.h
+ + + KTabWidget + QTabWidget +
ktabwidget.h
+ 1 + + + KListWidgetSearchLine + KLineEdit +
klistwidgetsearchline.h
+ + + + + diff --git a/src/settings/settingsdialog.cpp b/src/settings/settingsdialog.cpp index e7a9d1e8..fa1e8891 100644 --- a/src/settings/settingsdialog.cpp +++ b/src/settings/settingsdialog.cpp @@ -36,12 +36,14 @@ #include "application.h" #include "mainwindow.h" #include "webtab.h" +#include "adblockwidget.h" //Ui Includes #include "ui_settings_general.h" #include "ui_settings_tabs.h" #include "ui_settings_fonts.h" #include "ui_settings_webkit.h" +#include "ui_settings_adblock.h" // KDE Includes #include @@ -63,12 +65,14 @@ private: Ui::tabs tabsUi; Ui::fonts fontsUi; Ui::webkit webkitUi; + + AdBlockWidget *adBlockWidg; KCModuleProxy *proxyModule; KCModuleProxy *ebrowsingModule; KCModuleProxy *cookiesModule; KCModuleProxy *cacheModule; - KCModuleProxy *adblockModule; + KShortcutsEditor *shortcutsEditor; Private(SettingsDialog *parent); @@ -123,10 +127,10 @@ Private::Private(SettingsDialog *parent) KIcon webkitIcon = KIcon(QIcon(webkitIconPath)); pageItem->setIcon(webkitIcon); - KCModuleInfo adblockInfo("khtml_filter.desktop"); - adblockModule = new KCModuleProxy(adblockInfo,parent); - pageItem = parent->addPage(adblockModule, i18n(adblockInfo.moduleName().toLocal8Bit())); - pageItem->setIcon(KIcon(adblockInfo.icon())); + adBlockWidg = new AdBlockWidget(parent); + adBlockWidg->layout()->setMargin(0); + pageItem = parent->addPage(adBlockWidg , i18n("Ad Block")); + pageItem->setIcon( KIcon("preferences-web-browser-adblock") ); shortcutsEditor = new KShortcutsEditor(Application::instance()->mainWindow()->actionCollection(), parent); pageItem = parent->addPage(shortcutsEditor , i18n("Shortcuts")); @@ -137,7 +141,7 @@ Private::Private(SettingsDialog *parent) pageItem = parent->addPage(ebrowsingModule, i18n(ebrowsingInfo.moduleName().toLocal8Bit())); pageItem->setIcon(KIcon(ebrowsingInfo.icon())); - // WARNING remember wheh changing here that the smaller netbooks + // WARNING remember wheh changing here that the smallest netbooks // have a 1024x576 resolution. So DON'T bother that limits!! parent->setMinimumSize(700,525); } @@ -162,8 +166,7 @@ SettingsDialog::SettingsDialog(QWidget *parent) connect(d->cookiesModule, SIGNAL(changed(bool)), this, SLOT(updateButtons())); connect(d->proxyModule, SIGNAL(changed(bool)), this, SLOT(updateButtons())); connect(d->cacheModule, SIGNAL(changed(bool)), this, SLOT(updateButtons())); - connect(d->adblockModule, SIGNAL(changed(bool)), this, SLOT(updateButtons())); - + connect(d->shortcutsEditor, SIGNAL(keyChange()), this, SLOT(updateButtons())); connect(this, SIGNAL(applyClicked()), this, SLOT(saveSettings())); @@ -213,7 +216,7 @@ void SettingsDialog::saveSettings() d->proxyModule->save(); d->cacheModule->save(); d->shortcutsEditor->save(); - d->adblockModule->save(); + d->adBlockWidg->save(); } @@ -224,7 +227,6 @@ bool SettingsDialog::hasChanged() || d->cookiesModule->changed() || d->proxyModule->changed() || d->cacheModule->changed() - || d->adblockModule->changed() || d->shortcutsEditor->isModified(); ; } diff --git a/src/webpage.cpp b/src/webpage.cpp index f1591cee..4caf5a83 100644 --- a/src/webpage.cpp +++ b/src/webpage.cpp @@ -352,15 +352,15 @@ void WebPage::downloadAllContentsWithKGet() QWebElementCollection images = mainFrame()->documentElement().findAll("img"); foreach(QWebElement img, images) { - relativeUrl.setEncodedUrl(img.attribute("src").toUtf8(),KUrl::TolerantMode); - contents << baseUrl.resolved(relativeUrl).toString(); + relativeUrl.setEncodedUrl(img.attribute("src").toUtf8(),KUrl::TolerantMode); + contents << baseUrl.resolved(relativeUrl).toString(); } QWebElementCollection links = mainFrame()->documentElement().findAll("a"); foreach(QWebElement link, links) { - relativeUrl.setEncodedUrl(link.attribute("href").toUtf8(),KUrl::TolerantMode); - contents << baseUrl.resolved(relativeUrl).toString(); + relativeUrl.setEncodedUrl(link.attribute("href").toUtf8(),KUrl::TolerantMode); + contents << baseUrl.resolved(relativeUrl).toString(); } if(!QDBusConnection::sessionBus().interface()->isServiceRegistered("org.kde.kget")) -- cgit v1.2.1