6 #include "jsfunctions.h"
8 #include "videodefinition.h"
16 static const QString jsNameChars = "a-zA-Z0-9\\$_";
19 YTVideo::YTVideo(const QString &videoId, QObject *parent)
20 : QObject(parent), videoId(videoId), definitionCode(0), elIndex(0), ageGate(false),
21 loadingStreamUrl(false) {}
23 void YTVideo::loadStreamUrl() {
24 if (loadingStreamUrl) {
25 qDebug() << "Already loading stream URL for" << videoId;
28 loadingStreamUrl = true;
31 webPageLoaded = false;
37 void YTVideo::getVideoInfo() {
38 static const QStringList elTypes = {"&el=embedded", "&el=detailpage", "&el=vevo", ""};
41 if (elIndex == elTypes.size()) {
42 qDebug() << "Trying special embedded el param";
43 url = QUrl("https://www.youtube.com/get_video_info");
45 q.addQueryItem("video_id", videoId);
46 q.addQueryItem("el", "embedded");
47 q.addQueryItem("gl", "US");
48 q.addQueryItem("hl", "en");
49 q.addQueryItem("eurl", "https://youtube.googleapis.com/v/" + videoId);
50 q.addQueryItem("asv", "3");
51 q.addQueryItem("sts", "1588");
53 } else if (elIndex > elTypes.size() - 1) {
54 qDebug() << "Cannot get video info";
56 // no video info file, but we can try loading the "urlmap" from the web page
59 emitError("Cannot get video info");
63 // qDebug() << "Trying el param:" << elTypes.at(elIndex) << elIndex;
64 url = QUrl(QString("https://www.youtube.com/"
65 "get_video_info?video_id=%1%2&ps=default&eurl=&gl=US&hl=en")
66 .arg(videoId, elTypes.at(elIndex)));
69 QObject *reply = HttpUtils::stealthAndNotCached().get(url);
70 connect(reply, SIGNAL(data(QByteArray)), SLOT(gotVideoInfo(QByteArray)));
71 connect(reply, SIGNAL(error(QString)), SLOT(emitError(QString)));
73 // see you in gotVideoInfo...
76 void YTVideo::gotVideoInfo(const QByteArray &bytes) {
77 QString videoInfo = QString::fromUtf8(bytes);
78 // qDebug() << "videoInfo" << videoInfo;
80 // get player_response
81 static const QRegExp playerResponseRE("&player_response=([^&]+)");
82 if (playerResponseRE.indexIn(videoInfo) != -1) {
83 QString playerResponse = playerResponseRE.cap(1);
84 QByteArray playerResponseUtf8 = QByteArray::fromPercentEncoding(playerResponse.toUtf8());
85 // qDebug() << "player_response" << playerResponseUtf8;
86 QJsonDocument doc = QJsonDocument::fromJson(playerResponseUtf8);
87 QJsonObject obj = doc.object();
88 if (obj.contains("streamingData")) {
89 auto parseFormats = [this](const QJsonArray &formats) {
90 for (const QJsonValue &format : formats) {
91 QJsonObject formatObj = format.toObject();
92 int itag = formatObj["itag"].toInt();
93 QString url = formatObj["url"].toString();
95 QString cipher = formatObj["cipher"].toString();
96 if (cipher.isEmpty()) cipher = formatObj["signatureCipher"].toString();
98 qDebug() << "Cipher is " << q.toString();
99 url = q.queryItemValue("url").trimmed();
100 // while (url.contains('%'))
101 url = QByteArray::fromPercentEncoding(url.toUtf8());
102 if (q.hasQueryItem("s")) {
103 QString s = q.queryItemValue("s");
104 qDebug() << "s is" << s;
105 s = decryptSignature(s);
107 qDebug() << "Added signature" << s;
113 // qDebug() << "player_response format" << itag << url;
114 if (!url.isEmpty()) urlMap.insert(itag, url);
117 QJsonObject streamingDataObj = obj["streamingData"].toObject();
118 // qDebug() << "Found streamingData" << streamingDataObj;
119 parseFormats(streamingDataObj["formats"].toArray());
120 parseFormats(streamingDataObj["adaptiveFormats"].toArray());
126 static const QRegExp videoTokeRE(JsFunctions::instance()->videoTokenRE());
127 if (videoTokeRE.indexIn(videoInfo) == -1) {
128 qDebug() << "Cannot get token. Trying next el param" << videoTokeRE.pattern() << videoInfo;
129 // Don't panic! We're gonna try another magic "el" param
135 QString videoToken = videoTokeRE.cap(1);
136 while (videoToken.contains('%'))
137 videoToken = QByteArray::fromPercentEncoding(videoToken.toLatin1());
138 qDebug() << "videoToken" << videoToken;
139 this->videoToken = videoToken;
142 static const QRegExp fmtMapRE(JsFunctions::instance()->videoInfoFmtMapRE());
143 if (fmtMapRE.indexIn(videoInfo) == -1) {
144 qDebug() << "Cannot get urlMap. Trying next el param";
145 // Don't panic! We're gonna try another magic "el" param
150 QString fmtUrlMap = fmtMapRE.cap(1);
151 // qDebug() << "got fmtUrlMap" << fmtUrlMap;
152 fmtUrlMap = QByteArray::fromPercentEncoding(fmtUrlMap.toUtf8());
155 if (urlMap.isEmpty()) {
156 qDebug() << "empty urlMap, trying next el";
162 qDebug() << "Got token and urlMap" << elIndex << videoToken << fmtUrlMap;
163 parseFmtUrlMap(fmtUrlMap);
166 void YTVideo::parseFmtUrlMap(const QString &fmtUrlMap) {
168 const VideoDefinition &definition = YT3::instance().maxVideoDefinition();
170 // qDebug() << "fmtUrlMap" << fmtUrlMap;
171 const QVector<QStringRef> formatUrls = fmtUrlMap.splitRef(',', QString::SkipEmptyParts);
173 for (const QStringRef &formatUrl : formatUrls) {
174 // qDebug() << "formatUrl" << formatUrl;
175 const QVector<QStringRef> urlParams = formatUrl.split('&', QString::SkipEmptyParts);
176 // qDebug() << "urlParams" << urlParams;
182 for (const QStringRef &urlParam : urlParams) {
183 qDebug() << "urlParam" << urlParam;
184 if (sp.isNull() && urlParam.startsWith(QLatin1String("sp"))) {
185 int separator = urlParam.indexOf('=');
186 sp = urlParam.mid(separator + 1);
188 if (urlParam.startsWith(QLatin1String("itag="))) {
189 int separator = urlParam.indexOf('=');
190 format = urlParam.mid(separator + 1).toInt();
191 } else if (urlParam.startsWith(QLatin1String("url="))) {
192 int separator = urlParam.indexOf('=');
193 url = QByteArray::fromPercentEncoding(urlParam.mid(separator + 1).toUtf8());
194 } else if (urlParam.startsWith(QLatin1String("sig="))) {
195 int separator = urlParam.indexOf('=');
196 sig = QByteArray::fromPercentEncoding(urlParam.mid(separator + 1).toUtf8());
197 } else if (urlParam.startsWith(QLatin1String("s="))) {
198 if (webPageLoaded || ageGate) {
199 int separator = urlParam.indexOf('=');
200 sig = QByteArray::fromPercentEncoding(urlParam.mid(separator + 1).toUtf8());
201 sig = decryptSignature(sig);
202 if (sig.isEmpty()) sig = JsFunctions::instance()->decryptSignature(sig);
203 if (sig.isEmpty()) qWarning() << "Empty signature";
210 if (format == -1 || url.isNull()) continue;
212 if (!sig.isEmpty()) {
214 url += QLatin1String("&signature=") + sig;
216 url += '&' + sp + '=' + sig;
219 if (!url.contains(QLatin1String("ratebypass"))) url += QLatin1String("&ratebypass=yes");
222 if (format == definition.getCode()) {
223 qDebug() << "Found format" << format;
224 if (definition.hasAudio()) {
225 // we found the exact match with an audio/video stream
226 saveDefinitionForUrl(url, definition);
229 videoFormat = format;
231 urlMap.insert(format, url);
234 if (!webPageLoaded && !ageGate) {
239 if (videoFormat != 0) {
240 // exact match with video stream was found
241 const VideoDefinition &definition = VideoDefinition::forCode(videoFormat);
242 saveDefinitionForUrl(urlMap.value(videoFormat), definition);
246 qDebug() << "available formats" << urlMap.keys();
247 const QVector<VideoDefinition> &definitions = VideoDefinition::getDefinitions();
248 int previousIndex = std::max(definitions.indexOf(definition), 0);
249 for (; previousIndex >= 0; previousIndex--) {
250 const VideoDefinition &previousDefinition = definitions.at(previousIndex);
251 qDebug() << "Testing format" << previousDefinition.getCode();
252 if (urlMap.contains(previousDefinition.getCode())) {
253 qDebug() << "Found format" << previousDefinition.getCode();
254 saveDefinitionForUrl(urlMap.value(previousDefinition.getCode()), previousDefinition);
259 emit errorStreamUrl(tr("Cannot get video stream for %1").arg(videoId));
262 void YTVideo::loadWebPage() {
263 QUrl url("https://www.youtube.com/watch");
265 q.addQueryItem("v", videoId);
266 q.addQueryItem("gl", "US");
267 q.addQueryItem("hl", "en");
268 q.addQueryItem("has_verified", "1");
269 q.addQueryItem("bpctr", "9999999999");
272 qDebug() << "Loading webpage" << url;
273 QObject *reply = HttpUtils::yt().get(url);
274 connect(reply, SIGNAL(data(QByteArray)), SLOT(scrapeWebPage(QByteArray)));
275 connect(reply, SIGNAL(error(QString)), SLOT(emitError(QString)));
276 // see you in scrapWebPage(QByteArray)
279 void YTVideo::loadEmbedPage() {
280 QUrl url("https://www.youtube.com/embed/" + videoId);
281 auto reply = HttpUtils::yt().get(url);
282 connect(reply, &HttpReply::finished, this, [this](auto &reply) {
283 if (!reply.isSuccessful()) {
287 static const QRegExp embedRE("\"sts\"\\s*:\\s*(\\d+)");
289 if (embedRE.indexIn(reply.body()) == -1) {
290 // qDebug() << "Cannot get sts" << reply.body();
292 sts = embedRE.cap(1);
293 qDebug() << "sts" << sts;
296 q.addQueryItem("video_id", videoId);
297 q.addQueryItem("eurl", "https://youtube.googleapis.com/v/" + videoId);
298 q.addQueryItem("sts", sts);
299 QUrl url = QUrl("https://www.youtube.com/get_video_info");
301 HttpReply *r = HttpUtils::stealthAndNotCached().get(url);
302 connect(r, &HttpReply::data, this, [this](const QByteArray &bytes) {
303 QByteArray decodedBytes = QByteArray::fromPercentEncoding(bytes);
304 gotVideoInfo(decodedBytes);
306 connect(r, &HttpReply::error, this, &YTVideo::emitError);
310 void YTVideo::emitError(const QString &message) {
311 qWarning() << message;
312 emit errorStreamUrl(message);
315 void YTVideo::scrapeWebPage(const QByteArray &bytes) {
316 webPageLoaded = true;
318 const QString html = QString::fromUtf8(bytes);
319 // qDebug() << "scrapeWebPage" << html;
321 static const QRegExp ageGateRE(JsFunctions::instance()->ageGateRE());
322 if (ageGateRE.indexIn(html) != -1 || html.contains("desktopLegacyAgeGateReason")) {
323 qDebug() << "Found ageGate";
331 // "\"url_encoded_fmt_stream_map\":\s*\"([^\"]+)\""
332 static const QRegExp fmtMapRE(JsFunctions::instance()->webPageFmtMapRE());
333 if (fmtMapRE.indexIn(html) != -1) {
334 fmtUrlMap = fmtMapRE.cap(1);
335 fmtUrlMap.replace("\\u0026", "&");
338 QRegExp adaptiveFormatsRE("\"adaptive_fmts\":\\s*\"([^\"]+)\"");
339 if (adaptiveFormatsRE.indexIn(html) != -1) {
340 qDebug() << "Found adaptive_fmts";
341 if (!fmtUrlMap.isEmpty()) fmtUrlMap += ',';
342 fmtUrlMap += adaptiveFormatsRE.cap(1).replace("\\u0026", "&");
345 if (fmtUrlMap.isEmpty() && urlMap.isEmpty()) {
346 qDebug() << "Cannot get fmtUrlMap from video page. Trying next el";
352 static const QRegExp jsPlayerRe(JsFunctions::instance()->jsPlayerRE());
353 if (jsPlayerRe.indexIn(html) != -1) {
354 QString jsPlayerUrl = jsPlayerRe.cap(1);
355 jsPlayerUrl.remove('\\');
356 if (jsPlayerUrl.startsWith(QLatin1String("//"))) {
357 jsPlayerUrl = QLatin1String("https:") + jsPlayerUrl;
358 } else if (jsPlayerUrl.startsWith("/")) {
359 jsPlayerUrl = QLatin1String("https://youtube.com") + jsPlayerUrl;
361 // qDebug() << "jsPlayerUrl" << jsPlayerUrl;
363 QRegExp jsPlayerIdRe("-(.+)\\.js");
364 jsPlayerIdRe.indexIn(jsPlayerUrl);
365 QString jsPlayerId = jsPlayerRe.cap(1);
367 QObject *reply = HttpUtils::stealthAndNotCached().get(jsPlayerUrl);
368 connect(reply, SIGNAL(data(QByteArray)), SLOT(parseJsPlayer(QByteArray)));
369 connect(reply, SIGNAL(error(QString)), SLOT(emitError(QString)));
371 qDebug() << "Cannot find jsPlayer";
375 void YTVideo::parseJsPlayer(const QByteArray &bytes) {
376 jsPlayer = QString::fromUtf8(bytes);
377 // qDebug() << "jsPlayer" << jsPlayer;
379 // QRegExp funcNameRe("[\"']signature[\"']\\s*,\\s*([" + jsNameChars + "]+)\\(");
380 static const QVector<QRegExp> funcNameRes = [] {
381 QVector<QRegExp> res;
382 for (const QString &s : JsFunctions::instance()->signatureFunctionNameREs()) {
383 res << QRegExp(s.arg(jsNameChars));
387 for (const QRegExp &funcNameRe : funcNameRes) {
388 if (funcNameRe.indexIn(jsPlayer) == -1) {
389 qDebug() << "Cannot capture signature function name" << funcNameRe;
392 sigFuncName = funcNameRe.cap(1);
393 qDebug() << "Captures" << funcNameRe.captureCount() << funcNameRe.capturedTexts();
394 if (sigFuncName.isEmpty()) {
395 qDebug() << "Empty capture for" << funcNameRe;
398 captureFunction(sigFuncName, jsPlayer);
399 qDebug() << sigFunctions << sigObjects;
403 if (sigFuncName.isEmpty()) qDebug() << "Empty signature function name" << jsPlayer;
405 // parseFmtUrlMap(fmtUrlMap, true);
409 void YTVideo::captureFunction(const QString &name, const QString &js) {
410 qDebug() << __PRETTY_FUNCTION__ << name;
411 const QString argsAndBody =
412 QLatin1String("\\s*\\([") + jsNameChars + QLatin1String(",\\s]*\\)\\s*\\{[^\\}]+\\}");
414 QRegExp funcRe(QLatin1String("function\\s+") + QRegExp::escape(name) + argsAndBody);
415 if (funcRe.indexIn(js) != -1) {
416 func = funcRe.cap(0);
418 // try var foo = function(bar) { };
419 funcRe = QRegExp(QLatin1String("var\\s+") + QRegExp::escape(name) +
420 QLatin1String("\\s*=\\s*function") + argsAndBody);
421 if (funcRe.indexIn(js) != -1) {
422 func = funcRe.cap(0);
424 // try ,gr= function(bar) { };
425 funcRe = QRegExp(QLatin1String("[,\\s;}\\.\\)](") + QRegExp::escape(name) +
426 QLatin1String("\\s*=\\s*function") + argsAndBody + ")");
427 if (funcRe.indexIn(js) != -1) {
428 func = funcRe.cap(1);
430 qWarning() << "Cannot capture function" << name;
435 sigFunctions.insert(name, func);
437 // capture inner functions
438 static const QRegExp invokedFuncRe(QLatin1String("[\\s=;\\(]([") + jsNameChars +
439 QLatin1String("]+)\\s*\\([") + jsNameChars +
440 QLatin1String(",\\s]+\\)"));
441 int pos = name.length() + 9;
442 while ((pos = invokedFuncRe.indexIn(func, pos)) != -1) {
443 QString funcName = invokedFuncRe.cap(1);
444 if (!sigFunctions.contains(funcName)) captureFunction(funcName, js);
445 pos += invokedFuncRe.matchedLength();
448 // capture referenced objects
449 static const QRegExp objRe(QLatin1String("[\\s=;\\(]([") + jsNameChars +
450 QLatin1String("]+)\\.[") + jsNameChars + QLatin1String("]+"));
451 pos = name.length() + 9;
452 while ((pos = objRe.indexIn(func, pos)) != -1) {
453 QString objName = objRe.cap(1);
454 if (!sigObjects.contains(objName)) captureObject(objName, js);
455 pos += objRe.matchedLength();
459 void YTVideo::captureObject(const QString &name, const QString &js) {
460 QRegExp re(QLatin1String("var\\s+") + QRegExp::escape(name) +
461 QLatin1String("\\s*=\\s*\\{.*\\}\\s*;"));
463 if (re.indexIn(js) == -1) {
464 qWarning() << "Cannot capture object" << name;
467 QString obj = re.cap(0);
468 sigObjects.insert(name, obj);
471 QString YTVideo::decryptSignature(const QString &s) {
472 qDebug() << "decryptSignature" << sigFuncName << sigFunctions << sigObjects;
473 if (sigFuncName.isEmpty()) return QString();
475 for (const QString &f : sigObjects) {
476 QJSValue value = engine.evaluate(f);
477 if (value.isError()) qWarning() << "Error in" << f << value.toString();
479 for (const QString &f : sigFunctions) {
480 QJSValue value = engine.evaluate(f);
481 if (value.isError()) qWarning() << "Error in" << f << value.toString();
483 QString js = sigFuncName + "('" + s + "');";
484 QJSValue value = engine.evaluate(js);
486 if (value.isUndefined()) {
487 qWarning() << "Undefined result for" << js;
490 if (value.isError()) {
491 qWarning() << "Error in" << js << value.toString();
496 engine2.evaluate(jsPlayer);
497 value = engine2.evaluate(js);
498 if (value.isUndefined()) {
499 qWarning() << "Undefined result for" << js;
502 if (value.isError()) {
503 qWarning() << "Error in" << js << value.toString();
507 if (error) return QString();
508 return value.toString();
511 void YTVideo::saveDefinitionForUrl(const QString &url, const VideoDefinition &definition) {
512 qDebug() << "Selected video format" << definition.getCode() << definition.getName()
513 << definition.hasAudio();
515 definitionCode = definition.getCode();
518 if (!definition.hasAudio()) {
519 qDebug() << "Finding audio format";
520 static const QVector<int> audioFormats({251, 171, 140});
521 for (int audioFormat : audioFormats) {
522 qDebug() << "Trying audio format" << audioFormat;
523 auto i = urlMap.constFind(audioFormat);
524 if (i != urlMap.constEnd()) {
525 qDebug() << "Found audio format" << i.value();
526 audioUrl = i.value();
532 loadingStreamUrl = false;
533 emit gotStreamUrl(url, audioUrl);