git.sur5r.net Git - minitube/blob - src/channelaggregator.cpp

   1 /* $BEGIN_LICENSE
   2
   3 This file is part of Minitube.
   4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
   5
   6 Minitube is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, either version 3 of the License, or
   9 (at your option) any later version.
  10
  11 Minitube is distributed in the hope that it will be useful,
  12 but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Minitube.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 $END_LICENSE */
  20
  21 #include "channelaggregator.h"
  22 #include "ytchannel.h"
  23 #include "ytsearch.h"
  24 #include "searchparams.h"
  25 #include "database.h"
  26 #include "video.h"
  27 #ifdef APP_MAC
  28 #include "macutils.h"
  29 #endif
  30 #include "networkaccess.h"
  31
  32 namespace The {
  33     NetworkAccess* http();
  34 }
  35
  36 ChannelAggregator::ChannelAggregator(QObject *parent) : QObject(parent),
  37     unwatchedCount(-1),
  38     running(false),
  39     stopped(false),
  40     currentChannel(0) {
  41     checkInterval = 1800;
  42
  43     timer = new QTimer(this);
  44     timer->setInterval(60000 * 5);
  45     connect(timer, SIGNAL(timeout()), SLOT(run()));
  46 }
  47
  48 ChannelAggregator* ChannelAggregator::instance() {
  49     static ChannelAggregator* i = new ChannelAggregator();
  50     return i;
  51 }
  52
  53 void ChannelAggregator::start() {
  54     stopped = false;
  55     updateUnwatchedCount();
  56     QTimer::singleShot(0, this, SLOT(run()));
  57     if (!timer->isActive()) timer->start();
  58 }
  59
  60 void ChannelAggregator::stop() {
  61     timer->stop();
  62     stopped = true;
  63 }
  64
  65 YTChannel* ChannelAggregator::getChannelToCheck() {
  66     if (stopped) return 0;
  67     QSqlDatabase db = Database::instance().getConnection();
  68     QSqlQuery query(db);
  69     query.prepare("select user_id from subscriptions where checked<? "
  70                   "order by checked limit 1");
  71     query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
  72     bool success = query.exec();
  73     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
  74     if (query.next())
  75         return YTChannel::forId(query.value(0).toString());
  76     return 0;
  77 }
  78
  79 void ChannelAggregator::run() {
  80     if (running) return;
  81     if (!Database::exists()) return;
  82     running = true;
  83     newVideoCount = 0;
  84     updatedChannels.clear();
  85
  86     if (!Database::instance().getConnection().transaction())
  87         qWarning() << "Transaction failed" << __PRETTY_FUNCTION__;
  88
  89     processNextChannel();
  90 }
  91
  92 void ChannelAggregator::processNextChannel() {
  93     if (stopped) {
  94         running = false;
  95         return;
  96     }
  97     YTChannel* channel = getChannelToCheck();
  98     if (channel) {
  99         checkWebPage(channel);
 100     } else finish();
 101 }
 102
 103 void ChannelAggregator::checkWebPage(YTChannel *channel) {
 104     currentChannel = channel;
 105     QString url = "https://www.youtube.com/channel/" + channel->getChannelId() + "/videos";
 106     QObject *reply = The::http()->get(url);
 107
 108     connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
 109     connect(reply, SIGNAL(error(QNetworkReply*)), SLOT(errorWebPage(QNetworkReply*)));
 110 }
 111
 112 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
 113     bool hasNewVideos = true;
 114     QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
 115     if (re.indexIn(bytes) != -1) {
 116         QString videoId = re.cap(1);
 117         QString latestVideoId = currentChannel->latestVideoId();
 118         // qDebug() << "Comparing" << videoId << latestVideoId;
 119         hasNewVideos = videoId != latestVideoId;
 120     }
 121     if (hasNewVideos) {
 122         if (currentChannel) {
 123             reallyProcessChannel(currentChannel);
 124             currentChannel = 0;
 125         }
 126     } else {
 127         currentChannel->updateChecked();
 128         currentChannel = 0;
 129         processNextChannel();
 130     }
 131 }
 132
 133 void ChannelAggregator::errorWebPage(QNetworkReply *reply) {
 134     Q_UNUSED(reply);
 135     reallyProcessChannel(currentChannel);
 136     currentChannel = 0;
 137 }
 138
 139 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
 140     SearchParams *params = new SearchParams();
 141     params->setChannelId(channel->getChannelId());
 142     params->setSortBy(SearchParams::SortByNewest);
 143     params->setTransient(true);
 144     params->setPublishedAfter(channel->getChecked());
 145     YTSearch *videoSource = new YTSearch(params, this);
 146     connect(videoSource, SIGNAL(gotVideos(QList<Video*>)), SLOT(videosLoaded(QList<Video*>)));
 147     videoSource->loadVideos(50, 1);
 148
 149     channel->updateChecked();
 150 }
 151
 152 void ChannelAggregator::finish() {
 153     currentChannel = 0;
 154
 155     /*
 156     foreach (YTChannel *channel, updatedChannels)
 157         if (channel->updateNotifyCount())
 158             emit channelChanged(channel);
 159     updateUnwatchedCount();
 160     */
 161
 162     QSqlDatabase db = Database::instance().getConnection();
 163     if (!db.commit())
 164         qWarning() << "Commit failed" << __PRETTY_FUNCTION__;
 165
 166     /*
 167     QByteArray b = db.databaseName().right(20).toLocal8Bit();
 168     const char* s = b.constData();
 169     const int l = strlen(s);
 170     int t = 1;
 171     for (int i = 0; i < l; i++)
 172         t += t % 2 ? s[i] / l : s[i] / t;
 173     if (t != s[0]) return;
 174     */
 175
 176 #ifdef Q_OS_MAC
 177     if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
 178         QString channelNames;
 179         const int total = updatedChannels.size();
 180         for (int i = 0; i < total; ++i) {
 181             YTChannel *channel = updatedChannels.at(i);
 182             channelNames += channel->getDisplayName();
 183             if (i < total-1) channelNames.append(", ");
 184         }
 185         channelNames = tr("By %1").arg(channelNames);
 186         int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
 187         mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount),
 188                     channelNames, QString());
 189     }
 190 #endif
 191
 192     running = false;
 193 }
 194
 195 void ChannelAggregator::videosLoaded(const QList<Video*> &videos) {
 196     sender()->deleteLater();
 197
 198     foreach (Video* video, videos) {
 199         addVideo(video);
 200         qApp->processEvents();
 201     }
 202
 203     if (!videos.isEmpty()) {
 204         YTChannel *channel = YTChannel::forId(videos.first()->channelId());
 205         channel->updateNotifyCount();
 206         emit channelChanged(channel);
 207         updateUnwatchedCount();
 208         foreach (Video* video, videos) video->deleteLater();
 209     }
 210
 211     QTimer::singleShot(0, this, SLOT(processNextChannel()));
 212 }
 213
 214 void ChannelAggregator::updateUnwatchedCount() {
 215     if (!Database::exists()) return;
 216     QSqlDatabase db = Database::instance().getConnection();
 217     QSqlQuery query(db);
 218     query.prepare("select sum(notify_count) from subscriptions");
 219     bool success = query.exec();
 220     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 221     if (!query.next()) return;
 222     int newUnwatchedCount = query.value(0).toInt();
 223     if (newUnwatchedCount != unwatchedCount) {
 224         unwatchedCount = newUnwatchedCount;
 225         emit unwatchedCountChanged(unwatchedCount);
 226     }
 227 }
 228
 229 void ChannelAggregator::addVideo(Video *video) {
 230     QSqlDatabase db = Database::instance().getConnection();
 231
 232     QSqlQuery query(db);
 233     query.prepare("select count(*) from subscriptions_videos where video_id=?");
 234     query.bindValue(0, video->id());
 235     bool success = query.exec();
 236     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 237     if (!query.next()) return;
 238     int count = query.value(0).toInt();
 239     if (count > 0) return;
 240
 241     // qDebug() << "Inserting" << video->author() << video->title();
 242
 243     YTChannel *channel = YTChannel::forId(video->channelId());
 244     if (!channel) {
 245         qWarning() << "channelId not present in db" << video->channelId() << video->channelTitle();
 246         return;
 247     }
 248
 249     if (!updatedChannels.contains(channel))
 250         updatedChannels << channel;
 251
 252     uint now = QDateTime::currentDateTimeUtc().toTime_t();
 253     uint published = video->published().toTime_t();
 254     if (published > now) {
 255         qDebug() << "fixing publish time";
 256         published = now;
 257     }
 258
 259     query = QSqlQuery(db);
 260     query.prepare("insert into subscriptions_videos "
 261                   "(video_id,channel_id,published,added,watched,"
 262                   "title,author,user_id,description,url,thumb_url,views,duration) "
 263                   "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
 264     query.bindValue(0, video->id());
 265     query.bindValue(1, channel->getId());
 266     query.bindValue(2, published);
 267     query.bindValue(3, now);
 268     query.bindValue(4, 0);
 269     query.bindValue(5, video->title());
 270     query.bindValue(6, video->channelTitle());
 271     query.bindValue(7, video->channelId());
 272     query.bindValue(8, video->description());
 273     query.bindValue(9, video->webpage());
 274     query.bindValue(10, video->thumbnailUrl());
 275     query.bindValue(11, video->viewCount());
 276     query.bindValue(12, video->duration());
 277     success = query.exec();
 278     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 279
 280     newVideoCount++;
 281
 282     query = QSqlQuery(db);
 283     query.prepare("update subscriptions set updated=? where user_id=?");
 284     query.bindValue(0, published);
 285     query.bindValue(1, channel->getChannelId());
 286     success = query.exec();
 287     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 288 }
 289
 290 void ChannelAggregator::markAllAsWatched() {
 291     uint now = QDateTime::currentDateTimeUtc().toTime_t();
 292
 293     QSqlDatabase db = Database::instance().getConnection();
 294     QSqlQuery query(db);
 295     query.prepare("update subscriptions set watched=?, notify_count=0");
 296     query.bindValue(0, now);
 297     bool success = query.exec();
 298     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 299     unwatchedCount = 0;
 300
 301     foreach (YTChannel *channel, YTChannel::getCachedChannels()) {
 302         channel->setWatched(now);
 303         channel->setNotifyCount(0);
 304     }
 305
 306     emit unwatchedCountChanged(0);
 307 }
 308
 309 void ChannelAggregator::videoWatched(Video *video) {
 310     if (!Database::exists()) return;
 311     QSqlDatabase db = Database::instance().getConnection();
 312     QSqlQuery query(db);
 313     query.prepare("update subscriptions_videos set watched=? where video_id=?");
 314     query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
 315     query.bindValue(1, video->id());
 316     bool success = query.exec();
 317     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 318     if (query.numRowsAffected() > 0) {
 319         YTChannel *channel = YTChannel::forId(video->channelId());
 320         channel->updateNotifyCount();
 321     }
 322 }
 323
 324 void ChannelAggregator::cleanup() {
 325     const int maxVideos = 1000;
 326     const int maxDeletions = 1000;
 327     if (!Database::exists()) return;
 328     QSqlDatabase db = Database::instance().getConnection();
 329
 330     QSqlQuery query(db);
 331     bool success = query.exec("select count(*) from subscriptions_videos");
 332     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 333     if (!query.next()) return;
 334     int count = query.value(0).toInt();
 335     if (count <= maxVideos) return;
 336
 337     query = QSqlQuery(db);
 338     query.prepare("delete from subscriptions_videos where id in "
 339                   "(select id from subscriptions_videos "
 340                   "order by published desc limit ?,?)");
 341     query.bindValue(0, maxVideos);
 342     query.bindValue(1, maxDeletions);
 343     success = query.exec();
 344     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 345 }