git.sur5r.net Git - minitube/blob - src/channelaggregator.cpp

   1 /* $BEGIN_LICENSE
   2
   3 This file is part of Minitube.
   4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
   5
   6 Minitube is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, either version 3 of the License, or
   9 (at your option) any later version.
  10
  11 Minitube is distributed in the hope that it will be useful,
  12 but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Minitube.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 $END_LICENSE */
  20
  21 #include "channelaggregator.h"
  22 #include "database.h"
  23 #include "searchparams.h"
  24 #include "video.h"
  25 #include "ytchannel.h"
  26 #include "ytsearch.h"
  27 #ifdef APP_MAC
  28 #include "macutils.h"
  29 #endif
  30 #include "http.h"
  31 #include "httputils.h"
  32
  33 #include "ivchannelsource.h"
  34 #include "videoapi.h"
  35 #include "ytjschannelsource.h"
  36
  37 ChannelAggregator::ChannelAggregator(QObject *parent)
  38     : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
  39     checkInterval = 3600;
  40
  41     timer = new QTimer(this);
  42     timer->setInterval(60000 * 5);
  43     connect(timer, SIGNAL(timeout()), SLOT(run()));
  44 }
  45
  46 ChannelAggregator *ChannelAggregator::instance() {
  47     static ChannelAggregator *i = new ChannelAggregator();
  48     return i;
  49 }
  50
  51 void ChannelAggregator::start() {
  52     stopped = false;
  53     updateUnwatchedCount();
  54     QTimer::singleShot(10000, this, SLOT(run()));
  55     if (!timer->isActive()) timer->start();
  56 }
  57
  58 void ChannelAggregator::stop() {
  59     timer->stop();
  60     stopped = true;
  61 }
  62
  63 YTChannel *ChannelAggregator::getChannelToCheck() {
  64     if (stopped) return 0;
  65     QSqlDatabase db = Database::instance().getConnection();
  66     QSqlQuery query(db);
  67     query.prepare("select user_id from subscriptions where checked<? "
  68                   "order by checked limit 1");
  69     query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
  70     bool success = query.exec();
  71     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
  72     if (query.next()) return YTChannel::forId(query.value(0).toString());
  73     return 0;
  74 }
  75
  76 void ChannelAggregator::run() {
  77     if (running) return;
  78     if (!Database::exists()) return;
  79     running = true;
  80     newVideoCount = 0;
  81     updatedChannels.clear();
  82     updatedChannels.squeeze();
  83
  84     if (!Database::instance().getConnection().transaction())
  85         qWarning() << "Transaction failed" << __PRETTY_FUNCTION__;
  86
  87     processNextChannel();
  88 }
  89
  90 void ChannelAggregator::processNextChannel() {
  91     if (stopped) {
  92         running = false;
  93         return;
  94     }
  95     YTChannel *channel = getChannelToCheck();
  96     if (channel) {
  97         checkWebPage(channel);
  98     } else
  99         finish();
 100 }
 101
 102 void ChannelAggregator::checkWebPage(YTChannel *channel) {
 103     currentChannel = channel;
 104
 105     QString channelId = channel->getChannelId();
 106     QString url;
 107     if (channelId.startsWith("UC") && !channelId.contains(' ')) {
 108         url = "https://www.youtube.com/channel/" + channelId + "/videos";
 109     } else {
 110         url = "https://www.youtube.com/user/" + channelId + "/videos";
 111     }
 112
 113     QObject *reply = HttpUtils::yt().get(url);
 114
 115     connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
 116     connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
 117 }
 118
 119 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
 120     bool hasNewVideos = true;
 121     QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
 122     if (re.indexIn(bytes) != -1) {
 123         QString videoId = re.cap(1);
 124         QString latestVideoId = currentChannel->latestVideoId();
 125         qDebug() << "Comparing" << videoId << latestVideoId;
 126         hasNewVideos = videoId != latestVideoId;
 127     } else {
 128         qDebug() << "Cannot capture latest video id";
 129     }
 130     if (hasNewVideos) {
 131         if (currentChannel) {
 132             reallyProcessChannel(currentChannel);
 133             currentChannel = 0;
 134         }
 135     } else {
 136         currentChannel->updateChecked();
 137         currentChannel = 0;
 138         processNextChannel();
 139     }
 140 }
 141
 142 void ChannelAggregator::errorWebPage(const QString &message) {
 143     Q_UNUSED(message);
 144     reallyProcessChannel(currentChannel);
 145     currentChannel = 0;
 146 }
 147
 148 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
 149     SearchParams *params = new SearchParams();
 150     params->setChannelId(channel->getChannelId());
 151     params->setSortBy(SearchParams::SortByNewest);
 152     params->setTransient(true);
 153     params->setPublishedAfter(channel->getChecked());
 154
 155     if (VideoAPI::impl() == VideoAPI::YT3) {
 156         YTSearch *videoSource = new YTSearch(params);
 157         connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
 158                 SLOT(videosLoaded(QVector<Video *>)));
 159         videoSource->loadVideos(50, 1);
 160     } else if (VideoAPI::impl() == VideoAPI::IV) {
 161         auto *videoSource = new IVChannelSource(params);
 162         connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
 163                 SLOT(videosLoaded(QVector<Video *>)));
 164         videoSource->loadVideos(50, 1);
 165     } else if (VideoAPI::impl() == VideoAPI::JS) {
 166         auto *videoSource = new YTJSChannelSource(params);
 167         connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
 168                 SLOT(videosLoaded(QVector<Video *>)));
 169         videoSource->loadVideos(50, 1);
 170     }
 171
 172     channel->updateChecked();
 173 }
 174
 175 void ChannelAggregator::finish() {
 176     currentChannel = 0;
 177
 178     QSqlDatabase db = Database::instance().getConnection();
 179     if (!db.commit()) qWarning() << "Commit failed" << __PRETTY_FUNCTION__;
 180
 181 #ifdef Q_OS_MAC
 182     if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
 183         QString channelNames;
 184         const int total = updatedChannels.size();
 185         for (int i = 0; i < total; ++i) {
 186             YTChannel *channel = updatedChannels.at(i);
 187             channelNames += channel->getDisplayName();
 188             if (i < total - 1) channelNames.append(", ");
 189         }
 190         channelNames = tr("By %1").arg(channelNames);
 191         int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
 192         mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
 193                     QString());
 194     }
 195 #endif
 196
 197     running = false;
 198 }
 199
 200 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
 201     sender()->deleteLater();
 202
 203     for (Video *video : videos) {
 204         addVideo(video);
 205         qApp->processEvents();
 206     }
 207
 208     if (!videos.isEmpty()) {
 209         YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
 210         channel->updateNotifyCount();
 211         emit channelChanged(channel);
 212         updateUnwatchedCount();
 213         for (Video *video : videos)
 214             video->deleteLater();
 215     }
 216
 217     QTimer::singleShot(0, this, SLOT(processNextChannel()));
 218 }
 219
 220 void ChannelAggregator::updateUnwatchedCount() {
 221     if (!Database::exists()) return;
 222     QSqlDatabase db = Database::instance().getConnection();
 223     QSqlQuery query(db);
 224     query.prepare("select sum(notify_count) from subscriptions");
 225     bool success = query.exec();
 226     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 227     if (!query.next()) return;
 228     int newUnwatchedCount = query.value(0).toInt();
 229     if (newUnwatchedCount != unwatchedCount) {
 230         unwatchedCount = newUnwatchedCount;
 231         emit unwatchedCountChanged(unwatchedCount);
 232     }
 233 }
 234
 235 void ChannelAggregator::addVideo(Video *video) {
 236     QSqlDatabase db = Database::instance().getConnection();
 237
 238     QSqlQuery query(db);
 239     query.prepare("select count(*) from subscriptions_videos where video_id=?");
 240     query.bindValue(0, video->getId());
 241     bool success = query.exec();
 242     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 243     if (!query.next()) return;
 244     int count = query.value(0).toInt();
 245     if (count > 0) return;
 246
 247     // qDebug() << "Inserting" << video->author() << video->title();
 248
 249     YTChannel *channel = YTChannel::forId(video->getChannelId());
 250     if (!channel) {
 251         qWarning() << "channelId not present in db" << video->getChannelId()
 252                    << video->getChannelTitle();
 253         return;
 254     }
 255
 256     if (!updatedChannels.contains(channel)) updatedChannels << channel;
 257
 258     uint now = QDateTime::currentDateTimeUtc().toTime_t();
 259     uint published = video->getPublished().toTime_t();
 260     if (published > now) {
 261         qDebug() << "fixing publish time";
 262         published = now;
 263     }
 264
 265     query = QSqlQuery(db);
 266     query.prepare("insert into subscriptions_videos "
 267                   "(video_id,channel_id,published,added,watched,"
 268                   "title,author,user_id,description,url,thumb_url,views,duration) "
 269                   "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
 270     query.bindValue(0, video->getId());
 271     query.bindValue(1, channel->getId());
 272     query.bindValue(2, published);
 273     query.bindValue(3, now);
 274     query.bindValue(4, 0);
 275     query.bindValue(5, video->getTitle());
 276     query.bindValue(6, video->getChannelTitle());
 277     query.bindValue(7, video->getChannelId());
 278     query.bindValue(8, video->getDescription());
 279     query.bindValue(9, video->getWebpage());
 280     query.bindValue(10, video->getThumbnailUrl());
 281     query.bindValue(11, video->getViewCount());
 282     query.bindValue(12, video->getDuration());
 283     success = query.exec();
 284     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 285
 286     newVideoCount++;
 287
 288     query = QSqlQuery(db);
 289     query.prepare("update subscriptions set updated=? where user_id=?");
 290     query.bindValue(0, published);
 291     query.bindValue(1, channel->getChannelId());
 292     success = query.exec();
 293     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 294 }
 295
 296 void ChannelAggregator::markAllAsWatched() {
 297     uint now = QDateTime::currentDateTimeUtc().toTime_t();
 298
 299     QSqlDatabase db = Database::instance().getConnection();
 300     QSqlQuery query(db);
 301     query.prepare("update subscriptions set watched=?, notify_count=0");
 302     query.bindValue(0, now);
 303     bool success = query.exec();
 304     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 305     unwatchedCount = 0;
 306
 307     const auto &channels = YTChannel::getCachedChannels();
 308     for (YTChannel *channel : channels) {
 309         channel->setWatched(now);
 310         channel->setNotifyCount(0);
 311     }
 312
 313     emit unwatchedCountChanged(0);
 314 }
 315
 316 void ChannelAggregator::videoWatched(Video *video) {
 317     if (!Database::exists()) return;
 318     QSqlDatabase db = Database::instance().getConnection();
 319     QSqlQuery query(db);
 320     query.prepare("update subscriptions_videos set watched=? where video_id=?");
 321     query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
 322     query.bindValue(1, video->getId());
 323     bool success = query.exec();
 324     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 325     if (query.numRowsAffected() > 0) {
 326         YTChannel *channel = YTChannel::forId(video->getChannelId());
 327         channel->updateNotifyCount();
 328     }
 329 }
 330
 331 void ChannelAggregator::cleanup() {
 332     const int maxVideos = 1000;
 333     const int maxDeletions = 1000;
 334     if (!Database::exists()) return;
 335     QSqlDatabase db = Database::instance().getConnection();
 336
 337     QSqlQuery query(db);
 338     bool success = query.exec("select count(*) from subscriptions_videos");
 339     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 340     if (!query.next()) return;
 341     int count = query.value(0).toInt();
 342     if (count <= maxVideos) return;
 343
 344     query = QSqlQuery(db);
 345     query.prepare("delete from subscriptions_videos where id in "
 346                   "(select id from subscriptions_videos "
 347                   "order by published desc limit ?,?)");
 348     query.bindValue(0, maxVideos);
 349     query.bindValue(1, maxDeletions);
 350     success = query.exec();
 351     if (!success) qWarning() << query.lastQuery() << query.lastError().text();
 352 }