git.sur5r.net Git - minitube/blob - src/channelaggregator.cpp

   1 /* $BEGIN_LICENSE
   2
   3 This file is part of Minitube.
   4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
   5
   6 Minitube is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, either version 3 of the License, or
   9 (at your option) any later version.
  10
  11 Minitube is distributed in the hope that it will be useful,
  12 but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Minitube.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 $END_LICENSE */
  20
  21 #include "channelaggregator.h"
  22 #include "database.h"
  23 #include "searchparams.h"
  24 #include "video.h"
  25 #include "ytchannel.h"
  26 #include "ytsearch.h"
  27 #ifdef APP_MAC
  28 #include "macutils.h"
  29 #endif
  30 #include "http.h"
  31 #include "httputils.h"
  32
  33 ChannelAggregator::ChannelAggregator(QObject *parent)
  34     : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
  35     checkInterval = 1800;
  36
  37     timer = new QTimer(this);
  38     timer->setInterval(60000 * 5);
  39     connect(timer, SIGNAL(timeout()), SLOT(run()));
  40 }
  41
  42 ChannelAggregator *ChannelAggregator::instance() {
  43     static ChannelAggregator *i = new ChannelAggregator();
  44     return i;
  45 }
  46
  47 void ChannelAggregator::start() {
  48     stopped = false;
  49     updateUnwatchedCount();
  50     QTimer::singleShot(0, this, SLOT(run()));
  51     if (!timer->isActive()) timer->start();
  52 }
  53
  54 void ChannelAggregator::stop() {
  55     timer->stop();
  56     stopped = true;
  57 }
  58
  59 YTChannel *ChannelAggregator::getChannelToCheck() {
  60     if (stopped) return 0;
  61     QSqlDatabase db = Database::instance().getConnection();
  62     QSqlQuery query(db);
  63     query.prepare("select user_id from subscriptions where checked<? "
  64                   "order by checked limit 1");
  65     query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
  66     bool success = query.exec();
  67     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
  68     if (query.next()) return YTChannel::forId(query.value(0).toString());
  69     return 0;
  70 }
  71
  72 void ChannelAggregator::run() {
  73     if (running) return;
  74     if (!Database::exists()) return;
  75     running = true;
  76     newVideoCount = 0;
  77     updatedChannels.clear();
  78     updatedChannels.squeeze();
  79
  80     if (!Database::instance().getConnection().transaction())
  81         qWarning() << "Transaction failed" << __PRETTY_FUNCTION__;
  82
  83     processNextChannel();
  84 }
  85
  86 void ChannelAggregator::processNextChannel() {
  87     if (stopped) {
  88         running = false;
  89         return;
  90     }
  91     YTChannel *channel = getChannelToCheck();
  92     if (channel) {
  93         checkWebPage(channel);
  94     } else
  95         finish();
  96 }
  97
  98 void ChannelAggregator::checkWebPage(YTChannel *channel) {
  99     currentChannel = channel;
 100     QString url = "https://www.youtube.com/channel/" + channel->getChannelId() + "/videos";
 101     QObject *reply = HttpUtils::yt().get(url);
 102
 103     connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
 104     connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
 105 }
 106
 107 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
 108     bool hasNewVideos = true;
 109     QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
 110     if (re.indexIn(bytes) != -1) {
 111         QString videoId = re.cap(1);
 112         QString latestVideoId = currentChannel->latestVideoId();
 113         qDebug() << "Comparing" << videoId << latestVideoId;
 114         hasNewVideos = videoId != latestVideoId;
 115     }
 116     if (hasNewVideos) {
 117         if (currentChannel) {
 118             reallyProcessChannel(currentChannel);
 119             currentChannel = 0;
 120         }
 121     } else {
 122         currentChannel->updateChecked();
 123         currentChannel = 0;
 124         processNextChannel();
 125     }
 126 }
 127
 128 void ChannelAggregator::errorWebPage(const QString &message) {
 129     Q_UNUSED(message);
 130     reallyProcessChannel(currentChannel);
 131     currentChannel = 0;
 132 }
 133
 134 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
 135     SearchParams *params = new SearchParams();
 136     params->setChannelId(channel->getChannelId());
 137     params->setSortBy(SearchParams::SortByNewest);
 138     params->setTransient(true);
 139     params->setPublishedAfter(channel->getChecked());
 140     YTSearch *videoSource = new YTSearch(params);
 141     connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)), SLOT(videosLoaded(QVector<Video *>)));
 142     videoSource->loadVideos(50, 1);
 143
 144     channel->updateChecked();
 145 }
 146
 147 void ChannelAggregator::finish() {
 148     currentChannel = 0;
 149
 150     QSqlDatabase db = Database::instance().getConnection();
 151     if (!db.commit()) qWarning() << "Commit failed" << __PRETTY_FUNCTION__;
 152
 153 #ifdef Q_OS_MAC
 154     if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
 155         QString channelNames;
 156         const int total = updatedChannels.size();
 157         for (int i = 0; i < total; ++i) {
 158             YTChannel *channel = updatedChannels.at(i);
 159             channelNames += channel->getDisplayName();
 160             if (i < total - 1) channelNames.append(", ");
 161         }
 162         channelNames = tr("By %1").arg(channelNames);
 163         int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
 164         mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
 165                     QString());
 166     }
 167 #endif
 168
 169     running = false;
 170 }
 171
 172 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
 173     sender()->deleteLater();
 174
 175     for (Video *video : videos) {
 176         addVideo(video);
 177         qApp->processEvents();
 178     }
 179
 180     if (!videos.isEmpty()) {
 181         YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
 182         channel->updateNotifyCount();
 183         emit channelChanged(channel);
 184         updateUnwatchedCount();
 185         for (Video *video : videos)
 186             video->deleteLater();
 187     }
 188
 189     QTimer::singleShot(0, this, SLOT(processNextChannel()));
 190 }
 191
 192 void ChannelAggregator::updateUnwatchedCount() {
 193     if (!Database::exists()) return;
 194     QSqlDatabase db = Database::instance().getConnection();
 195     QSqlQuery query(db);
 196     query.prepare("select sum(notify_count) from subscriptions");
 197     bool success = query.exec();
 198     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 199     if (!query.next()) return;
 200     int newUnwatchedCount = query.value(0).toInt();
 201     if (newUnwatchedCount != unwatchedCount) {
 202         unwatchedCount = newUnwatchedCount;
 203         emit unwatchedCountChanged(unwatchedCount);
 204     }
 205 }
 206
 207 void ChannelAggregator::addVideo(Video *video) {
 208     QSqlDatabase db = Database::instance().getConnection();
 209
 210     QSqlQuery query(db);
 211     query.prepare("select count(*) from subscriptions_videos where video_id=?");
 212     query.bindValue(0, video->getId());
 213     bool success = query.exec();
 214     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 215     if (!query.next()) return;
 216     int count = query.value(0).toInt();
 217     if (count > 0) return;
 218
 219     // qDebug() << "Inserting" << video->author() << video->title();
 220
 221     YTChannel *channel = YTChannel::forId(video->getChannelId());
 222     if (!channel) {
 223         qWarning() << "channelId not present in db" << video->getChannelId()
 224                    << video->getChannelTitle();
 225         return;
 226     }
 227
 228     if (!updatedChannels.contains(channel)) updatedChannels << channel;
 229
 230     uint now = QDateTime::currentDateTimeUtc().toTime_t();
 231     uint published = video->getPublished().toTime_t();
 232     if (published > now) {
 233         qDebug() << "fixing publish time";
 234         published = now;
 235     }
 236
 237     query = QSqlQuery(db);
 238     query.prepare("insert into subscriptions_videos "
 239                   "(video_id,channel_id,published,added,watched,"
 240                   "title,author,user_id,description,url,thumb_url,views,duration) "
 241                   "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
 242     query.bindValue(0, video->getId());
 243     query.bindValue(1, channel->getId());
 244     query.bindValue(2, published);
 245     query.bindValue(3, now);
 246     query.bindValue(4, 0);
 247     query.bindValue(5, video->getTitle());
 248     query.bindValue(6, video->getChannelTitle());
 249     query.bindValue(7, video->getChannelId());
 250     query.bindValue(8, video->getDescription());
 251     query.bindValue(9, video->getWebpage());
 252     query.bindValue(10, video->getThumbnailUrl());
 253     query.bindValue(11, video->getViewCount());
 254     query.bindValue(12, video->getDuration());
 255     success = query.exec();
 256     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 257
 258     newVideoCount++;
 259
 260     query = QSqlQuery(db);
 261     query.prepare("update subscriptions set updated=? where user_id=?");
 262     query.bindValue(0, published);
 263     query.bindValue(1, channel->getChannelId());
 264     success = query.exec();
 265     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 266 }
 267
 268 void ChannelAggregator::markAllAsWatched() {
 269     uint now = QDateTime::currentDateTimeUtc().toTime_t();
 270
 271     QSqlDatabase db = Database::instance().getConnection();
 272     QSqlQuery query(db);
 273     query.prepare("update subscriptions set watched=?, notify_count=0");
 274     query.bindValue(0, now);
 275     bool success = query.exec();
 276     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 277     unwatchedCount = 0;
 278
 279     const auto &channels = YTChannel::getCachedChannels();
 280     for (YTChannel *channel : channels) {
 281         channel->setWatched(now);
 282         channel->setNotifyCount(0);
 283     }
 284
 285     emit unwatchedCountChanged(0);
 286 }
 287
 288 void ChannelAggregator::videoWatched(Video *video) {
 289     if (!Database::exists()) return;
 290     QSqlDatabase db = Database::instance().getConnection();
 291     QSqlQuery query(db);
 292     query.prepare("update subscriptions_videos set watched=? where video_id=?");
 293     query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
 294     query.bindValue(1, video->getId());
 295     bool success = query.exec();
 296     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 297     if (query.numRowsAffected() > 0) {
 298         YTChannel *channel = YTChannel::forId(video->getChannelId());
 299         channel->updateNotifyCount();
 300     }
 301 }
 302
 303 void ChannelAggregator::cleanup() {
 304     const int maxVideos = 1000;
 305     const int maxDeletions = 1000;
 306     if (!Database::exists()) return;
 307     QSqlDatabase db = Database::instance().getConnection();
 308
 309     QSqlQuery query(db);
 310     bool success = query.exec("select count(*) from subscriptions_videos");
 311     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 312     if (!query.next()) return;
 313     int count = query.value(0).toInt();
 314     if (count <= maxVideos) return;
 315
 316     query = QSqlQuery(db);
 317     query.prepare("delete from subscriptions_videos where id in "
 318                   "(select id from subscriptions_videos "
 319                   "order by published desc limit ?,?)");
 320     query.bindValue(0, maxVideos);
 321     query.bindValue(1, maxDeletions);
 322     success = query.exec();
 323     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 324 }