3 This file is part of Minitube.
4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
6 Minitube is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
11 Minitube is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Minitube. If not, see <http://www.gnu.org/licenses/>.
21 #include "channelaggregator.h"
23 #include "searchparams.h"
25 #include "ytchannel.h"
31 #include "httputils.h"
33 #include "ivchannelsource.h"
35 #include "ytjschannelsource.h"
37 ChannelAggregator::ChannelAggregator(QObject *parent)
38 : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
41 timer = new QTimer(this);
42 timer->setInterval(60000 * 5);
43 connect(timer, SIGNAL(timeout()), SLOT(run()));
46 ChannelAggregator *ChannelAggregator::instance() {
47 static ChannelAggregator *i = new ChannelAggregator();
51 void ChannelAggregator::start() {
53 updateUnwatchedCount();
54 QTimer::singleShot(10000, this, SLOT(run()));
55 if (!timer->isActive()) timer->start();
58 void ChannelAggregator::stop() {
63 YTChannel *ChannelAggregator::getChannelToCheck() {
64 if (stopped) return 0;
65 QSqlDatabase db = Database::instance().getConnection();
67 query.prepare("select user_id from subscriptions where checked<? "
68 "order by checked limit 1");
69 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
70 bool success = query.exec();
71 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
72 if (query.next()) return YTChannel::forId(query.value(0).toString());
76 void ChannelAggregator::run() {
78 if (!Database::exists()) return;
81 updatedChannels.clear();
82 updatedChannels.squeeze();
87 void ChannelAggregator::processNextChannel() {
92 YTChannel *channel = getChannelToCheck();
94 checkWebPage(channel);
99 void ChannelAggregator::checkWebPage(YTChannel *channel) {
100 currentChannel = channel;
102 QString channelId = channel->getChannelId();
104 if (channelId.startsWith("UC") && !channelId.contains(' ')) {
105 url = "https://www.youtube.com/channel/" + channelId + "/videos";
107 url = "https://www.youtube.com/user/" + channelId + "/videos";
110 QObject *reply = HttpUtils::yt().get(url);
112 connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
113 connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
116 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
117 bool hasNewVideos = true;
118 QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
119 if (re.indexIn(bytes) != -1) {
120 QString videoId = re.cap(1);
121 QString latestVideoId = currentChannel->latestVideoId();
122 qDebug() << "Comparing" << videoId << latestVideoId;
123 hasNewVideos = videoId != latestVideoId;
125 qDebug() << "Cannot capture latest video id";
128 if (currentChannel) {
129 reallyProcessChannel(currentChannel);
133 currentChannel->updateChecked();
135 QTimer::singleShot(5000, this, &ChannelAggregator::processNextChannel);
139 void ChannelAggregator::errorWebPage(const QString &message) {
141 reallyProcessChannel(currentChannel);
145 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
146 SearchParams *params = new SearchParams();
147 params->setChannelId(channel->getChannelId());
148 params->setSortBy(SearchParams::SortByNewest);
149 params->setTransient(true);
150 params->setPublishedAfter(channel->getChecked());
152 if (VideoAPI::impl() == VideoAPI::YT3) {
153 YTSearch *videoSource = new YTSearch(params);
154 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
155 SLOT(videosLoaded(QVector<Video *>)));
156 videoSource->loadVideos(50, 1);
157 } else if (VideoAPI::impl() == VideoAPI::IV) {
158 auto *videoSource = new IVChannelSource(params);
159 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
160 SLOT(videosLoaded(QVector<Video *>)));
161 videoSource->loadVideos(50, 1);
162 } else if (VideoAPI::impl() == VideoAPI::JS) {
163 auto *videoSource = new YTJSChannelSource(params);
164 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
165 SLOT(videosLoaded(QVector<Video *>)));
166 videoSource->loadVideos(50, 1);
169 channel->updateChecked();
172 void ChannelAggregator::finish() {
176 if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
177 QString channelNames;
178 const int total = updatedChannels.size();
179 for (int i = 0; i < total; ++i) {
180 YTChannel *channel = updatedChannels.at(i);
181 channelNames += channel->getDisplayName();
182 if (i < total - 1) channelNames.append(", ");
184 channelNames = tr("By %1").arg(channelNames);
185 int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
186 mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
194 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
195 sender()->deleteLater();
197 for (Video *video : videos) {
199 qApp->processEvents();
202 if (!videos.isEmpty()) {
203 YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
205 channel->updateNotifyCount();
206 emit channelChanged(channel);
208 updateUnwatchedCount();
209 for (Video *video : videos)
210 video->deleteLater();
213 QTimer::singleShot(0, this, SLOT(processNextChannel()));
216 void ChannelAggregator::updateUnwatchedCount() {
217 if (!Database::exists()) return;
218 QSqlDatabase db = Database::instance().getConnection();
220 query.prepare("select sum(notify_count) from subscriptions");
221 bool success = query.exec();
222 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
223 if (!query.next()) return;
224 int newUnwatchedCount = query.value(0).toInt();
225 if (newUnwatchedCount != unwatchedCount) {
226 unwatchedCount = newUnwatchedCount;
227 emit unwatchedCountChanged(unwatchedCount);
231 void ChannelAggregator::addVideo(Video *video) {
232 QSqlDatabase db = Database::instance().getConnection();
235 query.prepare("select count(*) from subscriptions_videos where video_id=?");
236 query.bindValue(0, video->getId());
237 bool success = query.exec();
238 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
239 if (!query.next()) return;
240 int count = query.value(0).toInt();
241 if (count > 0) return;
243 // qDebug() << "Inserting" << video->author() << video->title();
245 YTChannel *channel = YTChannel::forId(video->getChannelId());
247 qWarning() << "channelId not present in db" << video->getChannelId()
248 << video->getChannelTitle();
252 if (!updatedChannels.contains(channel)) updatedChannels << channel;
254 uint now = QDateTime::currentDateTimeUtc().toTime_t();
255 uint published = video->getPublished().toTime_t();
256 if (published > now) {
257 qDebug() << "fixing publish time";
261 query = QSqlQuery(db);
262 query.prepare("insert into subscriptions_videos "
263 "(video_id,channel_id,published,added,watched,"
264 "title,author,user_id,description,url,thumb_url,views,duration) "
265 "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
266 query.bindValue(0, video->getId());
267 query.bindValue(1, channel->getId());
268 query.bindValue(2, published);
269 query.bindValue(3, now);
270 query.bindValue(4, 0);
271 query.bindValue(5, video->getTitle());
272 query.bindValue(6, video->getChannelTitle());
273 query.bindValue(7, video->getChannelId());
274 query.bindValue(8, video->getDescription());
275 query.bindValue(9, video->getWebpage());
277 QJsonDocument thumbsDoc;
278 auto thumbsArray = thumbsDoc.array();
279 for (const auto &t : video->getThumbs()) {
280 thumbsArray.append(QJsonObject{
282 {"width", t.getWidth()},
283 {"height", t.getHeight()},
286 thumbsDoc.setArray(thumbsArray);
287 query.bindValue(10, thumbsDoc.toJson(QJsonDocument::Compact));
288 query.bindValue(11, video->getViewCount());
289 query.bindValue(12, video->getDuration());
290 success = query.exec();
291 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
295 query = QSqlQuery(db);
296 query.prepare("update subscriptions set updated=? where user_id=?");
297 query.bindValue(0, published);
298 query.bindValue(1, channel->getChannelId());
299 success = query.exec();
300 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
303 void ChannelAggregator::markAllAsWatched() {
304 uint now = QDateTime::currentDateTimeUtc().toTime_t();
306 QSqlDatabase db = Database::instance().getConnection();
308 query.prepare("update subscriptions set watched=?, notify_count=0");
309 query.bindValue(0, now);
310 bool success = query.exec();
311 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
314 const auto &channels = YTChannel::getCachedChannels();
315 for (YTChannel *channel : channels) {
316 channel->setWatched(now);
317 channel->setNotifyCount(0);
320 emit unwatchedCountChanged(0);
323 void ChannelAggregator::videoWatched(Video *video) {
324 if (!Database::exists()) return;
325 QSqlDatabase db = Database::instance().getConnection();
327 query.prepare("update subscriptions_videos set watched=? where video_id=?");
328 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
329 query.bindValue(1, video->getId());
330 bool success = query.exec();
331 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
332 if (query.numRowsAffected() > 0) {
333 YTChannel *channel = YTChannel::forId(video->getChannelId());
334 if (channel) channel->updateNotifyCount();
338 void ChannelAggregator::cleanup() {
339 const int maxVideos = 1000;
340 const int maxDeletions = 1000;
341 if (!Database::exists()) return;
342 QSqlDatabase db = Database::instance().getConnection();
345 bool success = query.exec("select count(*) from subscriptions_videos");
346 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
347 if (!query.next()) return;
348 int count = query.value(0).toInt();
349 if (count <= maxVideos) return;
351 query = QSqlQuery(db);
352 query.prepare("delete from subscriptions_videos where id in "
353 "(select id from subscriptions_videos "
354 "order by published desc limit ?,?)");
355 query.bindValue(0, maxVideos);
356 query.bindValue(1, maxDeletions);
357 success = query.exec();
358 if (!success) qWarning() << query.lastQuery() << query.lastError().text();