$END_LICENSE */
#include "channelaggregator.h"
-#include "ytuser.h"
-#include "ytsearch.h"
-#include "searchparams.h"
#include "database.h"
+#include "searchparams.h"
#include "video.h"
+#include "ytchannel.h"
+#include "ytsearch.h"
#ifdef APP_MAC
#include "macutils.h"
#endif
+#include "http.h"
+#include "httputils.h"
+
+#include "ivchannelsource.h"
+#include "videoapi.h"
+#include "ytjschannelsource.h"
-ChannelAggregator::ChannelAggregator(QObject *parent) : QObject(parent),
- unwatchedCount(-1),
- running(false),
- stopped(false) {
- QSettings settings;
- checkInterval = settings.value("subscriptionsCheckInterval", 1800).toUInt();
+ChannelAggregator::ChannelAggregator(QObject *parent)
+ : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
+ checkInterval = 3600;
timer = new QTimer(this);
timer->setInterval(60000 * 5);
connect(timer, SIGNAL(timeout()), SLOT(run()));
}
-ChannelAggregator* ChannelAggregator::instance() {
- static ChannelAggregator* i = new ChannelAggregator();
+ChannelAggregator *ChannelAggregator::instance() {
+ static ChannelAggregator *i = new ChannelAggregator();
return i;
}
void ChannelAggregator::start() {
+ stopped = false;
updateUnwatchedCount();
- QTimer::singleShot(0, this, SLOT(run()));
- timer->start();
+ QTimer::singleShot(10000, this, SLOT(run()));
+ if (!timer->isActive()) timer->start();
}
void ChannelAggregator::stop() {
stopped = true;
}
-YTUser* ChannelAggregator::getChannelToCheck() {
+YTChannel *ChannelAggregator::getChannelToCheck() {
if (stopped) return 0;
QSqlDatabase db = Database::instance().getConnection();
QSqlQuery query(db);
query.prepare("select user_id from subscriptions where checked<? "
"order by checked limit 1");
- query.bindValue(0, QDateTime::currentDateTime().toTime_t() - checkInterval);
+ query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
bool success = query.exec();
if (!success) qWarning() << query.lastQuery() << query.lastError().text();
- if (query.next())
- return YTUser::forId(query.value(0).toString());
+ if (query.next()) return YTChannel::forId(query.value(0).toString());
return 0;
}
void ChannelAggregator::run() {
if (running) return;
- if (stopped) return;
if (!Database::exists()) return;
running = true;
newVideoCount = 0;
updatedChannels.clear();
- if (!Database::instance().getConnection().transaction())
- qWarning() << "Transaction failed" << __PRETTY_FUNCTION__;
+ updatedChannels.squeeze();
+
processNextChannel();
}
void ChannelAggregator::processNextChannel() {
- if (stopped) return;
- qApp->processEvents();
- YTUser* user = getChannelToCheck();
- if (user) {
- SearchParams *params = new SearchParams();
- params->setAuthor(user->getUserId());
- params->setSortBy(SearchParams::SortByNewest);
- params->setTransient(true);
- YTSearch *videoSource = new YTSearch(params, this);
- connect(videoSource, SIGNAL(gotVideos(QList<Video*>)),
- SLOT(videosLoaded(QList<Video*>)));
- videoSource->loadVideos(10, 1);
- user->updateChecked();
- } else finish();
+ if (stopped) {
+ running = false;
+ return;
+ }
+ YTChannel *channel = getChannelToCheck();
+ if (channel) {
+ checkWebPage(channel);
+ } else
+ finish();
}
-void ChannelAggregator::finish() {
- foreach (YTUser *user, updatedChannels)
- if (user->updateNotifyCount())
- emit channelChanged(user);
+void ChannelAggregator::checkWebPage(YTChannel *channel) {
+ currentChannel = channel;
- updateUnwatchedCount();
+ QString channelId = channel->getChannelId();
+ QString url;
+ if (channelId.startsWith("UC") && !channelId.contains(' ')) {
+ url = "https://www.youtube.com/channel/" + channelId + "/videos";
+ } else {
+ url = "https://www.youtube.com/user/" + channelId + "/videos";
+ }
- QSqlDatabase db = Database::instance().getConnection();
- if (!db.commit())
- qWarning() << "Commit failed" << __PRETTY_FUNCTION__;
- /*
- QByteArray b = db.databaseName().right(20).toLocal8Bit();
- const char* s = b.constData();
- const int l = strlen(s);
- int t = 1;
- for (int i = 0; i < l; i++)
- t += t % 2 ? s[i] / l : s[i] / t;
- if (t != s[0]) return;
- */
+ QObject *reply = HttpUtils::yt().get(url);
+
+ connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
+ connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
+}
+
+void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
+ bool hasNewVideos = true;
+ QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
+ if (re.indexIn(bytes) != -1) {
+ QString videoId = re.cap(1);
+ QString latestVideoId = currentChannel->latestVideoId();
+ qDebug() << "Comparing" << videoId << latestVideoId;
+ hasNewVideos = videoId != latestVideoId;
+ } else {
+ qDebug() << "Cannot capture latest video id";
+ }
+ if (hasNewVideos) {
+ if (currentChannel) {
+ reallyProcessChannel(currentChannel);
+ currentChannel = 0;
+ }
+ } else {
+ currentChannel->updateChecked();
+ currentChannel = 0;
+ QTimer::singleShot(5000, this, &ChannelAggregator::processNextChannel);
+ }
+}
+
+void ChannelAggregator::errorWebPage(const QString &message) {
+ Q_UNUSED(message);
+ reallyProcessChannel(currentChannel);
+ currentChannel = 0;
+}
+
+void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
+ SearchParams *params = new SearchParams();
+ params->setChannelId(channel->getChannelId());
+ params->setSortBy(SearchParams::SortByNewest);
+ params->setTransient(true);
+ params->setPublishedAfter(channel->getChecked());
+
+ if (VideoAPI::impl() == VideoAPI::YT3) {
+ YTSearch *videoSource = new YTSearch(params);
+ connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
+ SLOT(videosLoaded(QVector<Video *>)));
+ videoSource->loadVideos(50, 1);
+ } else if (VideoAPI::impl() == VideoAPI::IV) {
+ auto *videoSource = new IVChannelSource(params);
+ connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
+ SLOT(videosLoaded(QVector<Video *>)));
+ videoSource->loadVideos(50, 1);
+ } else if (VideoAPI::impl() == VideoAPI::JS) {
+ auto *videoSource = new YTJSChannelSource(params);
+ connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
+ SLOT(videosLoaded(QVector<Video *>)));
+ videoSource->loadVideos(50, 1);
+ }
+
+ channel->updateChecked();
+}
+
+void ChannelAggregator::finish() {
+ currentChannel = 0;
#ifdef Q_OS_MAC
if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
QString channelNames;
const int total = updatedChannels.size();
for (int i = 0; i < total; ++i) {
- YTUser *user = updatedChannels.at(i);
- channelNames += user->getDisplayName();
- if (i < total-1) channelNames.append(", ");
+ YTChannel *channel = updatedChannels.at(i);
+ channelNames += channel->getDisplayName();
+ if (i < total - 1) channelNames.append(", ");
}
channelNames = tr("By %1").arg(channelNames);
int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
- mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount),
- channelNames, QString());
+ mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
+ QString());
}
#endif
running = false;
}
-void ChannelAggregator::videosLoaded(QList<Video *> videos) {
+void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
sender()->deleteLater();
- foreach (Video* video, videos) {
- qApp->processEvents();
+
+ for (Video *video : videos) {
addVideo(video);
- video->deleteLater();
+ qApp->processEvents();
}
- processNextChannel();
+
+ if (!videos.isEmpty()) {
+ YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
+ if (channel) {
+ channel->updateNotifyCount();
+ emit channelChanged(channel);
+ }
+ updateUnwatchedCount();
+ for (Video *video : videos)
+ video->deleteLater();
+ }
+
+ QTimer::singleShot(0, this, SLOT(processNextChannel()));
}
void ChannelAggregator::updateUnwatchedCount() {
QSqlQuery query(db);
query.prepare("select count(*) from subscriptions_videos where video_id=?");
- query.bindValue(0, video->id());
+ query.bindValue(0, video->getId());
bool success = query.exec();
if (!success) qWarning() << query.lastQuery() << query.lastError().text();
if (!query.next()) return;
// qDebug() << "Inserting" << video->author() << video->title();
- QString userId = video->userId();
- YTUser *user = YTUser::forId(userId);
- if (!updatedChannels.contains(user))
- updatedChannels << user;
- int channelId = user->getId();
+ YTChannel *channel = YTChannel::forId(video->getChannelId());
+ if (!channel) {
+ qWarning() << "channelId not present in db" << video->getChannelId()
+ << video->getChannelTitle();
+ return;
+ }
+
+ if (!updatedChannels.contains(channel)) updatedChannels << channel;
- uint now = QDateTime::currentDateTime().toTime_t();
- uint published = video->published().toTime_t();
+ uint now = QDateTime::currentDateTimeUtc().toTime_t();
+ uint published = video->getPublished().toTime_t();
if (published > now) {
qDebug() << "fixing publish time";
published = now;
"(video_id,channel_id,published,added,watched,"
"title,author,user_id,description,url,thumb_url,views,duration) "
"values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
- query.bindValue(0, video->id());
- query.bindValue(1, channelId);
+ query.bindValue(0, video->getId());
+ query.bindValue(1, channel->getId());
query.bindValue(2, published);
query.bindValue(3, now);
query.bindValue(4, 0);
- query.bindValue(5, video->title());
- query.bindValue(6, video->author());
- query.bindValue(7, video->userId());
- query.bindValue(8, video->description());
- query.bindValue(9, video->webpage());
- query.bindValue(10, video->thumbnailUrl());
- query.bindValue(11, video->viewCount());
- query.bindValue(12, video->duration());
+ query.bindValue(5, video->getTitle());
+ query.bindValue(6, video->getChannelTitle());
+ query.bindValue(7, video->getChannelId());
+ query.bindValue(8, video->getDescription());
+ query.bindValue(9, video->getWebpage());
+
+ QJsonDocument thumbsDoc;
+ auto thumbsArray = thumbsDoc.array();
+ for (const auto &t : video->getThumbs()) {
+ thumbsArray.append(QJsonObject{
+ {"url", t.getUrl()},
+ {"width", t.getWidth()},
+ {"height", t.getHeight()},
+ });
+ }
+ thumbsDoc.setArray(thumbsArray);
+ query.bindValue(10, thumbsDoc.toJson(QJsonDocument::Compact));
+ query.bindValue(11, video->getViewCount());
+ query.bindValue(12, video->getDuration());
success = query.exec();
if (!success) qWarning() << query.lastQuery() << query.lastError().text();
query = QSqlQuery(db);
query.prepare("update subscriptions set updated=? where user_id=?");
query.bindValue(0, published);
- query.bindValue(1, userId);
+ query.bindValue(1, channel->getChannelId());
success = query.exec();
if (!success) qWarning() << query.lastQuery() << query.lastError().text();
}
void ChannelAggregator::markAllAsWatched() {
- uint now = QDateTime::currentDateTime().toTime_t();
+ uint now = QDateTime::currentDateTimeUtc().toTime_t();
QSqlDatabase db = Database::instance().getConnection();
QSqlQuery query(db);
if (!success) qWarning() << query.lastQuery() << query.lastError().text();
unwatchedCount = 0;
- foreach (YTUser *user, YTUser::getCachedUsers()) {
- user->setWatched(now);
- user->setNotifyCount(0);
+ const auto &channels = YTChannel::getCachedChannels();
+ for (YTChannel *channel : channels) {
+ channel->setWatched(now);
+ channel->setNotifyCount(0);
}
emit unwatchedCountChanged(0);
QSqlDatabase db = Database::instance().getConnection();
QSqlQuery query(db);
query.prepare("update subscriptions_videos set watched=? where video_id=?");
- query.bindValue(0, QDateTime::currentDateTime().toTime_t());
- query.bindValue(1, video->id());
+ query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
+ query.bindValue(1, video->getId());
bool success = query.exec();
if (!success) qWarning() << query.lastQuery() << query.lastError().text();
if (query.numRowsAffected() > 0) {
- YTUser *user = YTUser::forId(video->userId());
- user->updateNotifyCount();
+ YTChannel *channel = YTChannel::forId(video->getChannelId());
+ if (channel) channel->updateNotifyCount();
}
}
void ChannelAggregator::cleanup() {
- static const int maxVideos = 1000;
- static const int maxDeletions = 1000;
+ const int maxVideos = 1000;
+ const int maxDeletions = 1000;
if (!Database::exists()) return;
QSqlDatabase db = Database::instance().getConnection();
projects / minitube / blobdiff