From 265cfcde765624d338aa90542b01295b0ac1bd5f Mon Sep 17 00:00:00 2001 From: Jakob Haufe Date: Tue, 29 Sep 2020 10:49:45 +0200 Subject: [PATCH] New upstream version 3.5.1 --- .github/FUNDING.yml | 2 + README.md | 9 +-- lib/http/src/cachedhttp.cpp | 36 +++++++++-- lib/http/src/cachedhttp.h | 11 +++- minitube.pro | 4 +- src/invidious/invidious.cpp | 91 ++++++++++++++++++++------- src/invidious/invidious.h | 1 + src/invidious/invidious.pri | 6 +- src/invidious/ivchannelsource.cpp | 20 +++--- src/invidious/ivchannelsource.h | 8 +-- src/invidious/ivsearch.cpp | 25 +++----- src/invidious/ivsearch.h | 9 +-- src/invidious/ivsinglevideosource.cpp | 17 ++--- src/invidious/ivsinglevideosource.h | 9 +-- src/invidious/ivvideolist.cpp | 19 +++--- src/invidious/ivvideolist.h | 8 +-- src/invidious/ivvideosource.cpp | 29 +++++++++ src/invidious/ivvideosource.h | 32 ++++++++++ src/yt3.cpp | 2 +- 19 files changed, 224 insertions(+), 114 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 src/invidious/ivvideosource.cpp create mode 100644 src/invidious/ivvideosource.h diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..136a229 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +github: flaviotordini +custom: https://flavio.tordini.org/donate diff --git a/README.md b/README.md index 4106a0e..8dc2650 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,6 @@ Minitube is a YouTube desktop application. It is written in C++ using the Qt fra Translations are done at https://www.transifex.com/flaviotordini/minitube/ Just register and apply for a language team. Please don't request translation merges on GitHub. -## Google API Key -Google is now requiring an API key in order to access YouTube Data web services. -Create a "Browser Key" at https://console.developers.google.com and enable the Youtube Data API. - -The key must be specified at compile time as shown below. -Alternatively Minitube can read an API key from the GOOGLE_API_KEY environment variable. - ## Build instructions Clone from Github: @@ -29,7 +22,7 @@ To be able to build on a Debian (or derivative) system: Compiling: - qmake "DEFINES += APP_GOOGLE_API_KEY=YourAPIKeyHere" + qmake make Running: diff --git a/lib/http/src/cachedhttp.cpp b/lib/http/src/cachedhttp.cpp index e055342..2b29fda 100644 --- a/lib/http/src/cachedhttp.cpp +++ b/lib/http/src/cachedhttp.cpp @@ -1,5 +1,6 @@ #include "cachedhttp.h" #include "localcache.h" +#include CachedHttpReply::CachedHttpReply(const QByteArray &body, const HttpRequest &req) : bytes(body), req(req) { @@ -16,8 +17,11 @@ void CachedHttpReply::emitSignals() { deleteLater(); } -WrappedHttpReply::WrappedHttpReply(LocalCache *cache, const QByteArray &key, HttpReply *httpReply) - : HttpReply(httpReply), cache(cache), key(key), httpReply(httpReply) { +WrappedHttpReply::WrappedHttpReply(CachedHttp &cachedHttp, + LocalCache *cache, + const QByteArray &key, + HttpReply *httpReply) + : HttpReply(httpReply), cachedHttp(cachedHttp), cache(cache), key(key), httpReply(httpReply) { connect(httpReply, SIGNAL(data(QByteArray)), SIGNAL(data(QByteArray))); connect(httpReply, SIGNAL(error(QString)), SIGNAL(error(QString))); connect(httpReply, SIGNAL(finished(HttpReply)), SLOT(originFinished(HttpReply))); @@ -25,7 +29,31 @@ WrappedHttpReply::WrappedHttpReply(LocalCache *cache, const QByteArray &key, Htt void WrappedHttpReply::originFinished(const HttpReply &reply) { qDebug() << reply.statusCode() << reply.url(); - if (reply.isSuccessful()) cache->insert(key, reply.body()); + bool doCache = reply.isSuccessful(); + + if (doCache) { + const auto &validators = cachedHttp.getValidators(); + if (!validators.isEmpty()) { + const QByteArray &mime = reply.header("Content-Type"); + auto i = validators.constFind(mime); + if (i != validators.constEnd()) { + auto validator = i.value(); + doCache = validator(reply); + } else { + i = validators.constFind("*"); + if (i != validators.constEnd()) { + auto validator = i.value(); + doCache = validator(reply); + } + } + } + } + + if (doCache) + cache->insert(key, reply.body()); + else + qDebug() << "Not caching" << reply.statusCode() << reply.url(); + emit finished(reply); } @@ -54,7 +82,7 @@ HttpReply *CachedHttp::request(const HttpRequest &req) { return new CachedHttpReply(value, req); } qDebug() << "MISS" << key << req.url; - return new WrappedHttpReply(cache, key, http.request(req)); + return new WrappedHttpReply(*this, cache, key, http.request(req)); } QByteArray CachedHttp::requestHash(const HttpRequest &req) { diff --git a/lib/http/src/cachedhttp.h b/lib/http/src/cachedhttp.h index 01fef13..4ce5e12 100644 --- a/lib/http/src/cachedhttp.h +++ b/lib/http/src/cachedhttp.h @@ -12,6 +12,7 @@ public: void setMaxSize(uint maxSize); void setCachePostRequests(bool value) { cachePostRequests = value; } void setIgnoreHostname(bool value) { ignoreHostname = value; } + auto &getValidators() { return validators; }; HttpReply *request(const HttpRequest &req); private: @@ -21,6 +22,10 @@ private: LocalCache *cache; bool cachePostRequests; bool ignoreHostname = false; + + /// Mapping is MIME -> validating function + /// Use * as MIME to define a catch-all validator + QMap> validators; }; class CachedHttpReply : public HttpReply { @@ -44,7 +49,10 @@ class WrappedHttpReply : public HttpReply { Q_OBJECT public: - WrappedHttpReply(LocalCache *cache, const QByteArray &key, HttpReply *httpReply); + WrappedHttpReply(CachedHttp &cachedHttp, + LocalCache *cache, + const QByteArray &key, + HttpReply *httpReply); QUrl url() const { return httpReply->url(); } int statusCode() const { return httpReply->statusCode(); } QByteArray body() const { return httpReply->body(); } @@ -53,6 +61,7 @@ private slots: void originFinished(const HttpReply &reply); private: + CachedHttp &cachedHttp; LocalCache *cache; QByteArray key; HttpReply *httpReply; diff --git a/minitube.pro b/minitube.pro index 187db44..5598e04 100644 --- a/minitube.pro +++ b/minitube.pro @@ -1,7 +1,7 @@ CONFIG += c++17 exceptions_off rtti_off optimize_full object_parallel_to_source TEMPLATE = app -VERSION = 3.5 +VERSION = 3.5.1 DEFINES += APP_VERSION="$$VERSION" APP_NAME = Minitube @@ -43,6 +43,8 @@ include(lib/media/media.pri) include(src/qtsingleapplication/qtsingleapplication.pri) include(src/invidious/invidious.pri) +INCLUDEPATH += $$PWD/src + HEADERS += src/video.h \ src/messagebar.h \ src/spacer.h \ diff --git a/src/invidious/invidious.cpp b/src/invidious/invidious.cpp index 38faf9a..572b50e 100644 --- a/src/invidious/invidious.cpp +++ b/src/invidious/invidious.cpp @@ -3,8 +3,23 @@ #include "cachedhttp.h" #include "http.h" #include "httputils.h" +#include "jsfunctions.h" #include "throttledhttp.h" +#ifdef APP_EXTRA +#include "extra.h" +#endif + +namespace { +QStringList fallbackServers{"https://invidious.snopyta.org"}; +QStringList preferredServers; + +void shuffle(QStringList &sl) { + std::shuffle(sl.begin(), sl.end(), *QRandomGenerator::global()); +} + +} // namespace + Invidious &Invidious::instance() { static Invidious i; return i; @@ -23,68 +38,96 @@ Http &Invidious::http() { Http &Invidious::cachedHttp() { static Http *h = [] { ThrottledHttp *throttledHttp = new ThrottledHttp(http()); - throttledHttp->setMilliseconds(300); + throttledHttp->setMilliseconds(500); CachedHttp *cachedHttp = new CachedHttp(*throttledHttp, "iv"); cachedHttp->setMaxSeconds(86400); cachedHttp->setIgnoreHostname(true); + + cachedHttp->getValidators().insert("application/json", [](const auto &reply) -> bool { + const auto body = reply.body(); + if (body.isEmpty() || body == "[]" || body == "{}") return false; + return true; + }); + return cachedHttp; }(); return *h; } -Invidious::Invidious(QObject *parent) : QObject(parent) {} +Invidious::Invidious(QObject *parent) : QObject(parent) { +#ifdef APP_EXTRA + preferredServers << Extra::extraFunctions()->stringArray("ivPreferred()"); + shuffle(preferredServers); +#endif + fallbackServers = JsFunctions::instance()->stringArray("ivFallback()"); + shuffle(fallbackServers); +} void Invidious::initServers() { - servers.clear(); - QUrl url("https://instances.invidio.us/instances.json?sort_by=type,health,users"); - auto reply = HttpUtils::yt().get(url); + if (!servers.isEmpty()) shuffleServers(); + + QString instanceApi = JsFunctions::instance()->string("ivInstances()"); + if (instanceApi.isEmpty()) return; + + auto reply = http().get(instanceApi); connect(reply, &HttpReply::finished, this, [this](auto &reply) { if (reply.isSuccessful()) { + servers.clear(); + QSettings settings; QStringList keywords = settings.value("recentKeywords").toStringList(); QString testKeyword = keywords.isEmpty() ? "test" : keywords.first(); - bool haveEnoughServers = false; QJsonDocument doc = QJsonDocument::fromJson(reply.body()); for (const auto &v : doc.array()) { + if (servers.size() > 3) break; + auto serverArray = v.toArray(); QString host = serverArray.first().toString(); QJsonObject serverObj = serverArray.at(1).toObject(); if (serverObj["type"] == "https") { QString url = "https://" + host; - if (haveEnoughServers) break; QUrl testUrl(url + "/api/v1/search?q=" + testKeyword); auto reply = http().get(testUrl); - connect(reply, &HttpReply::finished, this, - [this, url, &haveEnoughServers](auto &reply) { - if (!haveEnoughServers && reply.isSuccessful()) { - QJsonDocument doc = QJsonDocument::fromJson(reply.body()); - if (!doc.array().isEmpty()) { - servers << url; - if (servers.size() > 4) { - haveEnoughServers = true; - std::shuffle(servers.begin(), servers.end(), - *QRandomGenerator::global()); - qDebug() << servers; - emit serversInitialized(); - } + connect(reply, &HttpReply::finished, this, [this, url](auto &reply) { + if (reply.isSuccessful()) { + QJsonDocument doc = QJsonDocument::fromJson(reply.body()); + if (!doc.array().isEmpty()) { + if (servers.size() < 3) { + servers << url; + if (servers.size() == 3) { + shuffleServers(); + for (const auto &s : qAsConst(preferredServers)) + servers.prepend(s); + qDebug() << servers; + emit serversInitialized(); } } - }); + } + } + }); } } } }); } +void Invidious::shuffleServers() { + shuffle(servers); +} + QString Invidious::baseUrl() { QString host; - if (servers.isEmpty()) - host = "https://invidious.snopyta.org"; - else + if (servers.isEmpty()) { + if (preferredServers.isEmpty()) + host = fallbackServers.first(); + else + host = preferredServers.first(); + } else host = servers.first(); + QString url = host + QLatin1String("/api/v1/"); return url; } diff --git a/src/invidious/invidious.h b/src/invidious/invidious.h index 65de4c0..f6fe09a 100644 --- a/src/invidious/invidious.h +++ b/src/invidious/invidious.h @@ -15,6 +15,7 @@ public: explicit Invidious(QObject *parent = nullptr); void initServers(); + void shuffleServers(); QString baseUrl(); QUrl method(const QString &name); diff --git a/src/invidious/invidious.pri b/src/invidious/invidious.pri index b6256bd..400937d 100644 --- a/src/invidious/invidious.pri +++ b/src/invidious/invidious.pri @@ -7,7 +7,8 @@ HEADERS += $$PWD/invidious.h \ $$PWD/ivlistparser.h \ $$PWD/ivsearch.h \ $$PWD/ivsinglevideosource.h \ - $$PWD/ivvideolist.h + $$PWD/ivvideolist.h \ + $$PWD/ivvideosource.h SOURCES += $$PWD/invidious.cpp \ $$PWD/ivchannel.cpp \ @@ -15,5 +16,6 @@ SOURCES += $$PWD/invidious.cpp \ $$PWD/ivlistparser.cpp \ $$PWD/ivsearch.cpp \ $$PWD/ivsinglevideosource.cpp \ - $$PWD/ivvideolist.cpp + $$PWD/ivvideolist.cpp \ + $$PWD/ivvideosource.cpp diff --git a/src/invidious/ivchannelsource.cpp b/src/invidious/ivchannelsource.cpp index f486517..ff30965 100644 --- a/src/invidious/ivchannelsource.cpp +++ b/src/invidious/ivchannelsource.cpp @@ -13,13 +13,11 @@ int invidiousFixedMax = 20; } IVChannelSource::IVChannelSource(SearchParams *searchParams, QObject *parent) - : VideoSource(parent), searchParams(searchParams) { + : IVVideoSource(parent), searchParams(searchParams) { searchParams->setParent(this); } -void IVChannelSource::loadVideos(int max, int startIndex) { - aborted = false; - +void IVChannelSource::reallyLoadVideos(int max, int startIndex) { QUrl url = Invidious::instance().method("channels/videos/"); url.setPath(url.path() + searchParams->channelId()); @@ -43,6 +41,11 @@ void IVChannelSource::loadVideos(int max, int startIndex) { connect(reply, &HttpReply::data, this, [this](auto data) { QJsonDocument doc = QJsonDocument::fromJson(data); const QJsonArray items = doc.array(); + if (items.isEmpty()) { + handleError("No videos"); + return; + } + IVListParser parser(items); const QVector