3 This file is part of Minitube.
4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
6 Minitube is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
11 Minitube is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Minitube. If not, see <http://www.gnu.org/licenses/>.
21 #include "channelaggregator.h"
23 #include "searchparams.h"
25 #include "ytchannel.h"
31 #include "httputils.h"
33 #include "ivchannelsource.h"
35 #include "ytjschannelsource.h"
37 ChannelAggregator::ChannelAggregator(QObject *parent)
38 : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
41 timer = new QTimer(this);
42 timer->setInterval(60000 * 5);
43 connect(timer, SIGNAL(timeout()), SLOT(run()));
46 ChannelAggregator *ChannelAggregator::instance() {
47 static ChannelAggregator *i = new ChannelAggregator();
51 void ChannelAggregator::start() {
53 updateUnwatchedCount();
54 QTimer::singleShot(10000, this, SLOT(run()));
55 if (!timer->isActive()) timer->start();
58 void ChannelAggregator::stop() {
63 YTChannel *ChannelAggregator::getChannelToCheck() {
64 if (stopped) return 0;
65 QSqlDatabase db = Database::instance().getConnection();
67 query.prepare("select user_id from subscriptions where checked<? "
68 "order by checked limit 1");
69 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
70 bool success = query.exec();
71 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
72 if (query.next()) return YTChannel::forId(query.value(0).toString());
76 void ChannelAggregator::run() {
78 if (!Database::exists()) return;
81 updatedChannels.clear();
82 updatedChannels.squeeze();
87 void ChannelAggregator::processNextChannel() {
92 YTChannel *channel = getChannelToCheck();
94 checkWebPage(channel);
99 void ChannelAggregator::checkWebPage(YTChannel *channel) {
100 currentChannel = channel;
102 QString channelId = channel->getChannelId();
104 if (channelId.startsWith("UC") && !channelId.contains(' ')) {
105 url = "https://www.youtube.com/channel/" + channelId + "/videos";
107 url = "https://www.youtube.com/user/" + channelId + "/videos";
110 QObject *reply = HttpUtils::yt().get(url);
112 connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
113 connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
116 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
117 bool hasNewVideos = true;
118 QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
119 if (re.indexIn(bytes) != -1) {
120 QString videoId = re.cap(1);
121 QString latestVideoId = currentChannel->latestVideoId();
122 qDebug() << "Comparing" << videoId << latestVideoId;
123 hasNewVideos = videoId != latestVideoId;
125 qDebug() << "Cannot capture latest video id";
128 if (currentChannel) {
129 reallyProcessChannel(currentChannel);
133 currentChannel->updateChecked();
135 processNextChannel();
139 void ChannelAggregator::errorWebPage(const QString &message) {
141 reallyProcessChannel(currentChannel);
145 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
146 SearchParams *params = new SearchParams();
147 params->setChannelId(channel->getChannelId());
148 params->setSortBy(SearchParams::SortByNewest);
149 params->setTransient(true);
150 params->setPublishedAfter(channel->getChecked());
152 if (VideoAPI::impl() == VideoAPI::YT3) {
153 YTSearch *videoSource = new YTSearch(params);
154 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
155 SLOT(videosLoaded(QVector<Video *>)));
156 videoSource->loadVideos(50, 1);
157 } else if (VideoAPI::impl() == VideoAPI::IV) {
158 auto *videoSource = new IVChannelSource(params);
159 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
160 SLOT(videosLoaded(QVector<Video *>)));
161 videoSource->loadVideos(50, 1);
162 } else if (VideoAPI::impl() == VideoAPI::JS) {
163 auto *videoSource = new YTJSChannelSource(params);
164 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
165 SLOT(videosLoaded(QVector<Video *>)));
166 videoSource->loadVideos(50, 1);
169 channel->updateChecked();
172 void ChannelAggregator::finish() {
176 if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
177 QString channelNames;
178 const int total = updatedChannels.size();
179 for (int i = 0; i < total; ++i) {
180 YTChannel *channel = updatedChannels.at(i);
181 channelNames += channel->getDisplayName();
182 if (i < total - 1) channelNames.append(", ");
184 channelNames = tr("By %1").arg(channelNames);
185 int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
186 mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
194 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
195 sender()->deleteLater();
197 for (Video *video : videos) {
199 qApp->processEvents();
202 if (!videos.isEmpty()) {
203 YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
205 channel->updateNotifyCount();
206 emit channelChanged(channel);
208 updateUnwatchedCount();
209 for (Video *video : videos)
210 video->deleteLater();
213 QTimer::singleShot(0, this, SLOT(processNextChannel()));
216 void ChannelAggregator::updateUnwatchedCount() {
217 if (!Database::exists()) return;
218 QSqlDatabase db = Database::instance().getConnection();
220 query.prepare("select sum(notify_count) from subscriptions");
221 bool success = query.exec();
222 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
223 if (!query.next()) return;
224 int newUnwatchedCount = query.value(0).toInt();
225 if (newUnwatchedCount != unwatchedCount) {
226 unwatchedCount = newUnwatchedCount;
227 emit unwatchedCountChanged(unwatchedCount);
231 void ChannelAggregator::addVideo(Video *video) {
232 QSqlDatabase db = Database::instance().getConnection();
235 query.prepare("select count(*) from subscriptions_videos where video_id=?");
236 query.bindValue(0, video->getId());
237 bool success = query.exec();
238 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
239 if (!query.next()) return;
240 int count = query.value(0).toInt();
241 if (count > 0) return;
243 // qDebug() << "Inserting" << video->author() << video->title();
245 YTChannel *channel = YTChannel::forId(video->getChannelId());
247 qWarning() << "channelId not present in db" << video->getChannelId()
248 << video->getChannelTitle();
252 if (!updatedChannels.contains(channel)) updatedChannels << channel;
254 uint now = QDateTime::currentDateTimeUtc().toTime_t();
255 uint published = video->getPublished().toTime_t();
256 if (published > now) {
257 qDebug() << "fixing publish time";
261 query = QSqlQuery(db);
262 query.prepare("insert into subscriptions_videos "
263 "(video_id,channel_id,published,added,watched,"
264 "title,author,user_id,description,url,thumb_url,views,duration) "
265 "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
266 query.bindValue(0, video->getId());
267 query.bindValue(1, channel->getId());
268 query.bindValue(2, published);
269 query.bindValue(3, now);
270 query.bindValue(4, 0);
271 query.bindValue(5, video->getTitle());
272 query.bindValue(6, video->getChannelTitle());
273 query.bindValue(7, video->getChannelId());
274 query.bindValue(8, video->getDescription());
275 query.bindValue(9, video->getWebpage());
276 query.bindValue(10, video->getThumbnailUrl());
277 query.bindValue(11, video->getViewCount());
278 query.bindValue(12, video->getDuration());
279 success = query.exec();
280 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
284 query = QSqlQuery(db);
285 query.prepare("update subscriptions set updated=? where user_id=?");
286 query.bindValue(0, published);
287 query.bindValue(1, channel->getChannelId());
288 success = query.exec();
289 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
292 void ChannelAggregator::markAllAsWatched() {
293 uint now = QDateTime::currentDateTimeUtc().toTime_t();
295 QSqlDatabase db = Database::instance().getConnection();
297 query.prepare("update subscriptions set watched=?, notify_count=0");
298 query.bindValue(0, now);
299 bool success = query.exec();
300 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
303 const auto &channels = YTChannel::getCachedChannels();
304 for (YTChannel *channel : channels) {
305 channel->setWatched(now);
306 channel->setNotifyCount(0);
309 emit unwatchedCountChanged(0);
312 void ChannelAggregator::videoWatched(Video *video) {
313 if (!Database::exists()) return;
314 QSqlDatabase db = Database::instance().getConnection();
316 query.prepare("update subscriptions_videos set watched=? where video_id=?");
317 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
318 query.bindValue(1, video->getId());
319 bool success = query.exec();
320 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
321 if (query.numRowsAffected() > 0) {
322 YTChannel *channel = YTChannel::forId(video->getChannelId());
323 if (channel) channel->updateNotifyCount();
327 void ChannelAggregator::cleanup() {
328 const int maxVideos = 1000;
329 const int maxDeletions = 1000;
330 if (!Database::exists()) return;
331 QSqlDatabase db = Database::instance().getConnection();
334 bool success = query.exec("select count(*) from subscriptions_videos");
335 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
336 if (!query.next()) return;
337 int count = query.value(0).toInt();
338 if (count <= maxVideos) return;
340 query = QSqlQuery(db);
341 query.prepare("delete from subscriptions_videos where id in "
342 "(select id from subscriptions_videos "
343 "order by published desc limit ?,?)");
344 query.bindValue(0, maxVideos);
345 query.bindValue(1, maxDeletions);
346 success = query.exec();
347 if (!success) qWarning() << query.lastQuery() << query.lastError().text();