3 This file is part of Minitube.
4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
6 Minitube is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
11 Minitube is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Minitube. If not, see <http://www.gnu.org/licenses/>.
21 #include "channelaggregator.h"
23 #include "searchparams.h"
25 #include "ytchannel.h"
31 #include "httputils.h"
33 #include "ivchannelsource.h"
35 #include "ytjschannelsource.h"
37 ChannelAggregator::ChannelAggregator(QObject *parent)
38 : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
41 timer = new QTimer(this);
42 timer->setInterval(60000 * 5);
43 connect(timer, SIGNAL(timeout()), SLOT(run()));
46 ChannelAggregator *ChannelAggregator::instance() {
47 static ChannelAggregator *i = new ChannelAggregator();
51 void ChannelAggregator::start() {
53 updateUnwatchedCount();
54 QTimer::singleShot(10000, this, SLOT(run()));
55 if (!timer->isActive()) timer->start();
58 void ChannelAggregator::stop() {
63 YTChannel *ChannelAggregator::getChannelToCheck() {
64 if (stopped) return 0;
65 QSqlDatabase db = Database::instance().getConnection();
67 query.prepare("select user_id from subscriptions where checked<? "
68 "order by checked limit 1");
69 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
70 bool success = query.exec();
71 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
72 if (query.next()) return YTChannel::forId(query.value(0).toString());
76 void ChannelAggregator::run() {
78 if (!Database::exists()) return;
81 updatedChannels.clear();
82 updatedChannels.squeeze();
84 if (!Database::instance().getConnection().transaction())
85 qWarning() << "Transaction failed" << __PRETTY_FUNCTION__;
90 void ChannelAggregator::processNextChannel() {
95 YTChannel *channel = getChannelToCheck();
97 checkWebPage(channel);
102 void ChannelAggregator::checkWebPage(YTChannel *channel) {
103 currentChannel = channel;
105 QString channelId = channel->getChannelId();
107 if (channelId.startsWith("UC") && !channelId.contains(' ')) {
108 url = "https://www.youtube.com/channel/" + channelId + "/videos";
110 url = "https://www.youtube.com/user/" + channelId + "/videos";
113 QObject *reply = HttpUtils::yt().get(url);
115 connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
116 connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
119 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
120 bool hasNewVideos = true;
121 QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
122 if (re.indexIn(bytes) != -1) {
123 QString videoId = re.cap(1);
124 QString latestVideoId = currentChannel->latestVideoId();
125 qDebug() << "Comparing" << videoId << latestVideoId;
126 hasNewVideos = videoId != latestVideoId;
128 qDebug() << "Cannot capture latest video id";
131 if (currentChannel) {
132 reallyProcessChannel(currentChannel);
136 currentChannel->updateChecked();
138 processNextChannel();
142 void ChannelAggregator::errorWebPage(const QString &message) {
144 reallyProcessChannel(currentChannel);
148 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
149 SearchParams *params = new SearchParams();
150 params->setChannelId(channel->getChannelId());
151 params->setSortBy(SearchParams::SortByNewest);
152 params->setTransient(true);
153 params->setPublishedAfter(channel->getChecked());
155 if (VideoAPI::impl() == VideoAPI::YT3) {
156 YTSearch *videoSource = new YTSearch(params);
157 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
158 SLOT(videosLoaded(QVector<Video *>)));
159 videoSource->loadVideos(50, 1);
160 } else if (VideoAPI::impl() == VideoAPI::IV) {
161 auto *videoSource = new IVChannelSource(params);
162 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
163 SLOT(videosLoaded(QVector<Video *>)));
164 videoSource->loadVideos(50, 1);
165 } else if (VideoAPI::impl() == VideoAPI::JS) {
166 auto *videoSource = new YTJSChannelSource(params);
167 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
168 SLOT(videosLoaded(QVector<Video *>)));
169 videoSource->loadVideos(50, 1);
172 channel->updateChecked();
175 void ChannelAggregator::finish() {
178 QSqlDatabase db = Database::instance().getConnection();
179 if (!db.commit()) qWarning() << "Commit failed" << __PRETTY_FUNCTION__;
182 if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
183 QString channelNames;
184 const int total = updatedChannels.size();
185 for (int i = 0; i < total; ++i) {
186 YTChannel *channel = updatedChannels.at(i);
187 channelNames += channel->getDisplayName();
188 if (i < total - 1) channelNames.append(", ");
190 channelNames = tr("By %1").arg(channelNames);
191 int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
192 mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
200 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
201 sender()->deleteLater();
203 for (Video *video : videos) {
205 qApp->processEvents();
208 if (!videos.isEmpty()) {
209 YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
210 channel->updateNotifyCount();
211 emit channelChanged(channel);
212 updateUnwatchedCount();
213 for (Video *video : videos)
214 video->deleteLater();
217 QTimer::singleShot(0, this, SLOT(processNextChannel()));
220 void ChannelAggregator::updateUnwatchedCount() {
221 if (!Database::exists()) return;
222 QSqlDatabase db = Database::instance().getConnection();
224 query.prepare("select sum(notify_count) from subscriptions");
225 bool success = query.exec();
226 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
227 if (!query.next()) return;
228 int newUnwatchedCount = query.value(0).toInt();
229 if (newUnwatchedCount != unwatchedCount) {
230 unwatchedCount = newUnwatchedCount;
231 emit unwatchedCountChanged(unwatchedCount);
235 void ChannelAggregator::addVideo(Video *video) {
236 QSqlDatabase db = Database::instance().getConnection();
239 query.prepare("select count(*) from subscriptions_videos where video_id=?");
240 query.bindValue(0, video->getId());
241 bool success = query.exec();
242 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
243 if (!query.next()) return;
244 int count = query.value(0).toInt();
245 if (count > 0) return;
247 // qDebug() << "Inserting" << video->author() << video->title();
249 YTChannel *channel = YTChannel::forId(video->getChannelId());
251 qWarning() << "channelId not present in db" << video->getChannelId()
252 << video->getChannelTitle();
256 if (!updatedChannels.contains(channel)) updatedChannels << channel;
258 uint now = QDateTime::currentDateTimeUtc().toTime_t();
259 uint published = video->getPublished().toTime_t();
260 if (published > now) {
261 qDebug() << "fixing publish time";
265 query = QSqlQuery(db);
266 query.prepare("insert into subscriptions_videos "
267 "(video_id,channel_id,published,added,watched,"
268 "title,author,user_id,description,url,thumb_url,views,duration) "
269 "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
270 query.bindValue(0, video->getId());
271 query.bindValue(1, channel->getId());
272 query.bindValue(2, published);
273 query.bindValue(3, now);
274 query.bindValue(4, 0);
275 query.bindValue(5, video->getTitle());
276 query.bindValue(6, video->getChannelTitle());
277 query.bindValue(7, video->getChannelId());
278 query.bindValue(8, video->getDescription());
279 query.bindValue(9, video->getWebpage());
280 query.bindValue(10, video->getThumbnailUrl());
281 query.bindValue(11, video->getViewCount());
282 query.bindValue(12, video->getDuration());
283 success = query.exec();
284 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
288 query = QSqlQuery(db);
289 query.prepare("update subscriptions set updated=? where user_id=?");
290 query.bindValue(0, published);
291 query.bindValue(1, channel->getChannelId());
292 success = query.exec();
293 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
296 void ChannelAggregator::markAllAsWatched() {
297 uint now = QDateTime::currentDateTimeUtc().toTime_t();
299 QSqlDatabase db = Database::instance().getConnection();
301 query.prepare("update subscriptions set watched=?, notify_count=0");
302 query.bindValue(0, now);
303 bool success = query.exec();
304 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
307 const auto &channels = YTChannel::getCachedChannels();
308 for (YTChannel *channel : channels) {
309 channel->setWatched(now);
310 channel->setNotifyCount(0);
313 emit unwatchedCountChanged(0);
316 void ChannelAggregator::videoWatched(Video *video) {
317 if (!Database::exists()) return;
318 QSqlDatabase db = Database::instance().getConnection();
320 query.prepare("update subscriptions_videos set watched=? where video_id=?");
321 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
322 query.bindValue(1, video->getId());
323 bool success = query.exec();
324 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
325 if (query.numRowsAffected() > 0) {
326 YTChannel *channel = YTChannel::forId(video->getChannelId());
327 channel->updateNotifyCount();
331 void ChannelAggregator::cleanup() {
332 const int maxVideos = 1000;
333 const int maxDeletions = 1000;
334 if (!Database::exists()) return;
335 QSqlDatabase db = Database::instance().getConnection();
338 bool success = query.exec("select count(*) from subscriptions_videos");
339 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
340 if (!query.next()) return;
341 int count = query.value(0).toInt();
342 if (count <= maxVideos) return;
344 query = QSqlQuery(db);
345 query.prepare("delete from subscriptions_videos where id in "
346 "(select id from subscriptions_videos "
347 "order by published desc limit ?,?)");
348 query.bindValue(0, maxVideos);
349 query.bindValue(1, maxDeletions);
350 success = query.exec();
351 if (!success) qWarning() << query.lastQuery() << query.lastError().text();