3 This file is part of Minitube.
4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
6 Minitube is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
11 Minitube is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with Minitube. If not, see <http://www.gnu.org/licenses/>.
21 #include "channelaggregator.h"
23 #include "searchparams.h"
25 #include "ytchannel.h"
31 #include "httputils.h"
33 ChannelAggregator::ChannelAggregator(QObject *parent)
34 : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
37 timer = new QTimer(this);
38 timer->setInterval(60000 * 5);
39 connect(timer, SIGNAL(timeout()), SLOT(run()));
42 ChannelAggregator *ChannelAggregator::instance() {
43 static ChannelAggregator *i = new ChannelAggregator();
47 void ChannelAggregator::start() {
49 updateUnwatchedCount();
50 QTimer::singleShot(0, this, SLOT(run()));
51 if (!timer->isActive()) timer->start();
54 void ChannelAggregator::stop() {
59 YTChannel *ChannelAggregator::getChannelToCheck() {
60 if (stopped) return 0;
61 QSqlDatabase db = Database::instance().getConnection();
63 query.prepare("select user_id from subscriptions where checked<? "
64 "order by checked limit 1");
65 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
66 bool success = query.exec();
67 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
68 if (query.next()) return YTChannel::forId(query.value(0).toString());
72 void ChannelAggregator::run() {
74 if (!Database::exists()) return;
77 updatedChannels.clear();
78 updatedChannels.squeeze();
80 if (!Database::instance().getConnection().transaction())
81 qWarning() << "Transaction failed" << __PRETTY_FUNCTION__;
86 void ChannelAggregator::processNextChannel() {
91 YTChannel *channel = getChannelToCheck();
93 checkWebPage(channel);
98 void ChannelAggregator::checkWebPage(YTChannel *channel) {
99 currentChannel = channel;
100 QString url = "https://www.youtube.com/channel/" + channel->getChannelId() + "/videos";
101 QObject *reply = HttpUtils::yt().get(url);
103 connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
104 connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
107 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
108 bool hasNewVideos = true;
109 QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
110 if (re.indexIn(bytes) != -1) {
111 QString videoId = re.cap(1);
112 QString latestVideoId = currentChannel->latestVideoId();
113 qDebug() << "Comparing" << videoId << latestVideoId;
114 hasNewVideos = videoId != latestVideoId;
117 if (currentChannel) {
118 reallyProcessChannel(currentChannel);
122 currentChannel->updateChecked();
124 processNextChannel();
128 void ChannelAggregator::errorWebPage(const QString &message) {
130 reallyProcessChannel(currentChannel);
134 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
135 SearchParams *params = new SearchParams();
136 params->setChannelId(channel->getChannelId());
137 params->setSortBy(SearchParams::SortByNewest);
138 params->setTransient(true);
139 params->setPublishedAfter(channel->getChecked());
140 YTSearch *videoSource = new YTSearch(params);
141 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)), SLOT(videosLoaded(QVector<Video *>)));
142 videoSource->loadVideos(50, 1);
144 channel->updateChecked();
147 void ChannelAggregator::finish() {
150 QSqlDatabase db = Database::instance().getConnection();
151 if (!db.commit()) qWarning() << "Commit failed" << __PRETTY_FUNCTION__;
154 if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
155 QString channelNames;
156 const int total = updatedChannels.size();
157 for (int i = 0; i < total; ++i) {
158 YTChannel *channel = updatedChannels.at(i);
159 channelNames += channel->getDisplayName();
160 if (i < total - 1) channelNames.append(", ");
162 channelNames = tr("By %1").arg(channelNames);
163 int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
164 mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
172 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
173 sender()->deleteLater();
175 for (Video *video : videos) {
177 qApp->processEvents();
180 if (!videos.isEmpty()) {
181 YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
182 channel->updateNotifyCount();
183 emit channelChanged(channel);
184 updateUnwatchedCount();
185 for (Video *video : videos)
186 video->deleteLater();
189 QTimer::singleShot(0, this, SLOT(processNextChannel()));
192 void ChannelAggregator::updateUnwatchedCount() {
193 if (!Database::exists()) return;
194 QSqlDatabase db = Database::instance().getConnection();
196 query.prepare("select sum(notify_count) from subscriptions");
197 bool success = query.exec();
198 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
199 if (!query.next()) return;
200 int newUnwatchedCount = query.value(0).toInt();
201 if (newUnwatchedCount != unwatchedCount) {
202 unwatchedCount = newUnwatchedCount;
203 emit unwatchedCountChanged(unwatchedCount);
207 void ChannelAggregator::addVideo(Video *video) {
208 QSqlDatabase db = Database::instance().getConnection();
211 query.prepare("select count(*) from subscriptions_videos where video_id=?");
212 query.bindValue(0, video->getId());
213 bool success = query.exec();
214 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
215 if (!query.next()) return;
216 int count = query.value(0).toInt();
217 if (count > 0) return;
219 // qDebug() << "Inserting" << video->author() << video->title();
221 YTChannel *channel = YTChannel::forId(video->getChannelId());
223 qWarning() << "channelId not present in db" << video->getChannelId()
224 << video->getChannelTitle();
228 if (!updatedChannels.contains(channel)) updatedChannels << channel;
230 uint now = QDateTime::currentDateTimeUtc().toTime_t();
231 uint published = video->getPublished().toTime_t();
232 if (published > now) {
233 qDebug() << "fixing publish time";
237 query = QSqlQuery(db);
238 query.prepare("insert into subscriptions_videos "
239 "(video_id,channel_id,published,added,watched,"
240 "title,author,user_id,description,url,thumb_url,views,duration) "
241 "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
242 query.bindValue(0, video->getId());
243 query.bindValue(1, channel->getId());
244 query.bindValue(2, published);
245 query.bindValue(3, now);
246 query.bindValue(4, 0);
247 query.bindValue(5, video->getTitle());
248 query.bindValue(6, video->getChannelTitle());
249 query.bindValue(7, video->getChannelId());
250 query.bindValue(8, video->getDescription());
251 query.bindValue(9, video->getWebpage());
252 query.bindValue(10, video->getThumbnailUrl());
253 query.bindValue(11, video->getViewCount());
254 query.bindValue(12, video->getDuration());
255 success = query.exec();
256 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
260 query = QSqlQuery(db);
261 query.prepare("update subscriptions set updated=? where user_id=?");
262 query.bindValue(0, published);
263 query.bindValue(1, channel->getChannelId());
264 success = query.exec();
265 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
268 void ChannelAggregator::markAllAsWatched() {
269 uint now = QDateTime::currentDateTimeUtc().toTime_t();
271 QSqlDatabase db = Database::instance().getConnection();
273 query.prepare("update subscriptions set watched=?, notify_count=0");
274 query.bindValue(0, now);
275 bool success = query.exec();
276 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
279 const auto &channels = YTChannel::getCachedChannels();
280 for (YTChannel *channel : channels) {
281 channel->setWatched(now);
282 channel->setNotifyCount(0);
285 emit unwatchedCountChanged(0);
288 void ChannelAggregator::videoWatched(Video *video) {
289 if (!Database::exists()) return;
290 QSqlDatabase db = Database::instance().getConnection();
292 query.prepare("update subscriptions_videos set watched=? where video_id=?");
293 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
294 query.bindValue(1, video->getId());
295 bool success = query.exec();
296 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
297 if (query.numRowsAffected() > 0) {
298 YTChannel *channel = YTChannel::forId(video->getChannelId());
299 channel->updateNotifyCount();
303 void ChannelAggregator::cleanup() {
304 const int maxVideos = 1000;
305 const int maxDeletions = 1000;
306 if (!Database::exists()) return;
307 QSqlDatabase db = Database::instance().getConnection();
310 bool success = query.exec("select count(*) from subscriptions_videos");
311 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
312 if (!query.next()) return;
313 int count = query.value(0).toInt();
314 if (count <= maxVideos) return;
316 query = QSqlQuery(db);
317 query.prepare("delete from subscriptions_videos where id in "
318 "(select id from subscriptions_videos "
319 "order by published desc limit ?,?)");
320 query.bindValue(0, maxVideos);
321 query.bindValue(1, maxDeletions);
322 success = query.exec();
323 if (!success) qWarning() << query.lastQuery() << query.lastError().text();