git.sur5r.net Git - minitube/blob - src/video.cpp

   1 /* $BEGIN_LICENSE
   2
   3 This file is part of Minitube.
   4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
   5
   6 Minitube is free software: you can redistribute it and/or modify
   7 it under the terms of the GNU General Public License as published by
   8 the Free Software Foundation, either version 3 of the License, or
   9 (at your option) any later version.
  10
  11 Minitube is distributed in the hope that it will be useful,
  12 but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Minitube.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 $END_LICENSE */
  20
  21 #include "video.h"
  22 #include "networkaccess.h"
  23 #include <QtNetwork>
  24 #include "videodefinition.h"
  25 #include "jsfunctions.h"
  26 #include "temporary.h"
  27
  28 namespace The {
  29 NetworkAccess* http();
  30 }
  31
  32 namespace {
  33 static const QString jsNameChars = "a-zA-Z0-9\\$_";
  34 }
  35
  36 Video::Video() : m_duration(0),
  37     m_viewCount(-1),
  38     definitionCode(0),
  39     elIndex(0),
  40     ageGate(false),
  41     m_license(LicenseYouTube),
  42     loadingStreamUrl(false),
  43     loadingThumbnail(false)
  44 { }
  45
  46 Video* Video::clone() {
  47     Video* cloneVideo = new Video();
  48     cloneVideo->m_title = m_title;
  49     cloneVideo->m_description = m_description;
  50     cloneVideo->m_author = m_author;
  51     cloneVideo->m_userId = m_userId;
  52     cloneVideo->m_webpage = m_webpage;
  53     cloneVideo->m_streamUrl = m_streamUrl;
  54     cloneVideo->m_thumbnail = m_thumbnail;
  55     cloneVideo->m_thumbnailUrl = m_thumbnailUrl;
  56     cloneVideo->m_mediumThumbnailUrl = m_mediumThumbnailUrl;
  57     cloneVideo->m_duration = m_duration;
  58     cloneVideo->m_published = m_published;
  59     cloneVideo->m_viewCount = m_viewCount;
  60     cloneVideo->videoId = videoId;
  61     cloneVideo->videoToken = videoToken;
  62     cloneVideo->definitionCode = definitionCode;
  63     return cloneVideo;
  64 }
  65
  66 void Video::setWebpage(QUrl webpage) {
  67     m_webpage = webpage;
  68
  69     // Get Video ID
  70     // youtube-dl line 428
  71     // QRegExp re("^((?:http://)?(?:\\w+\\.)?youtube\\.com/(?:(?:v/)|(?:(?:watch(?:\\.php)?)?\\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$");
  72     QRegExp re("^https?://www\\.youtube\\.com/watch\\?v=([0-9A-Za-z_-]+).*");
  73     bool match = re.exactMatch(m_webpage.toString());
  74     if (!match) {
  75         qWarning() << QString("Cannot get video id for %1").arg(m_webpage.toString());
  76         // emit errorStreamUrl(QString("Cannot get video id for %1").arg(m_webpage.toString()));
  77         // loadingStreamUrl = false;
  78         return;
  79     }
  80     videoId = re.cap(1);
  81 }
  82
  83 void Video::loadThumbnail() {
  84     if (m_thumbnailUrl.isEmpty() || loadingThumbnail) return;
  85     loadingThumbnail = true;
  86     QObject *reply = The::http()->get(m_thumbnailUrl);
  87     connect(reply, SIGNAL(data(QByteArray)), SLOT(setThumbnail(QByteArray)));
  88 }
  89
  90 void Video::setThumbnail(QByteArray bytes) {
  91     loadingThumbnail = false;
  92     m_thumbnail.loadFromData(bytes);
  93     if (m_thumbnail.width() > 160)
  94         m_thumbnail = m_thumbnail.scaledToWidth(160, Qt::SmoothTransformation);
  95     emit gotThumbnail();
  96 }
  97
  98 void Video::loadMediumThumbnail() {
  99     if (m_mediumThumbnailUrl.isEmpty()) return;
 100     QObject *reply = The::http()->get(m_mediumThumbnailUrl);
 101     connect(reply, SIGNAL(data(QByteArray)), SIGNAL(gotMediumThumbnail(QByteArray)));
 102 }
 103
 104 void Video::loadStreamUrl() {
 105     if (loadingStreamUrl) {
 106         qDebug() << "Already loading stream URL for" << this->title();
 107         return;
 108     }
 109     loadingStreamUrl = true;
 110     elIndex = 0;
 111     ageGate = false;
 112
 113     getVideoInfo();
 114 }
 115
 116 void  Video::getVideoInfo() {
 117     static const QStringList elTypes = QStringList() << "&el=embedded" << "&el=detailpage" << "&el=vevo" << "";
 118
 119     QUrl url;
 120
 121     if (elIndex == elTypes.size()) {
 122         // qDebug() << "Trying special embedded el param";
 123         url = QUrl("http://www.youtube.com/get_video_info");
 124
 125 #if QT_VERSION >= 0x050000
 126         {
 127             QUrl &u = url;
 128             QUrlQuery url;
 129 #endif
 130             url.addQueryItem("video_id", videoId);
 131             url.addQueryItem("el", "embedded");
 132             url.addQueryItem("gl", "US");
 133             url.addQueryItem("hl", "en");
 134             url.addQueryItem("eurl", "https://youtube.googleapis.com/v/" + videoId);
 135             url.addQueryItem("asv", "3");
 136             url.addQueryItem("sts", "1588");
 137 #if QT_VERSION >= 0x050000
 138             u.setQuery(url);
 139         }
 140 #endif
 141
 142     } else if (elIndex > elTypes.size() - 1) {
 143         qWarning() << "Cannot get video info";
 144         loadingStreamUrl = false;
 145         emit errorStreamUrl("Cannot get video info");
 146         return;
 147     } else {
 148         // qDebug() << "Trying el param:" << elTypes.at(elIndex) << elIndex;
 149         url = QUrl(QString(
 150                        "http://www.youtube.com/get_video_info?video_id=%1%2&ps=default&eurl=&gl=US&hl=en"
 151                        ).arg(videoId, elTypes.at(elIndex)));
 152     }
 153
 154     QObject *reply = The::http()->get(url);
 155     connect(reply, SIGNAL(data(QByteArray)), SLOT(gotVideoInfo(QByteArray)));
 156     connect(reply, SIGNAL(error(QNetworkReply*)), SLOT(errorVideoInfo(QNetworkReply*)));
 157
 158     // see you in gotVideoInfo...
 159 }
 160
 161 void  Video::gotVideoInfo(QByteArray data) {
 162     QString videoInfo = QString::fromUtf8(data);
 163     // qDebug() << "videoInfo" << videoInfo;
 164
 165     // get video token
 166     QRegExp re = QRegExp("^.*&token=([^&]+).*$");
 167     bool match = re.exactMatch(videoInfo);
 168     // handle regexp failure
 169     if (!match) {
 170         // qDebug() << "Cannot get token. Trying next el param";
 171         // Don't panic! We're gonna try another magic "el" param
 172         elIndex++;
 173         getVideoInfo();
 174         return;
 175     }
 176
 177     QString videoToken = re.cap(1);
 178     while (videoToken.contains('%'))
 179         videoToken = QByteArray::fromPercentEncoding(videoToken.toLatin1());
 180     // qDebug() << "videoToken" << videoToken;
 181     this->videoToken = videoToken;
 182
 183     // get fmt_url_map
 184     re = QRegExp("^.*&url_encoded_fmt_stream_map=([^&]+).*$");
 185     match = re.exactMatch(videoInfo);
 186     // handle regexp failure
 187     if (!match) {
 188         // qDebug() << "Cannot get urlMap. Trying next el param";
 189         // Don't panic! We're gonna try another magic "el" param
 190         elIndex++;
 191         getVideoInfo();
 192         return;
 193     }
 194
 195     // qDebug() << "Got token and urlMap" << elIndex;
 196
 197     QString fmtUrlMap = re.cap(1);
 198     fmtUrlMap = QByteArray::fromPercentEncoding(fmtUrlMap.toUtf8());
 199     parseFmtUrlMap(fmtUrlMap);
 200 }
 201
 202 void Video::parseFmtUrlMap(const QString &fmtUrlMap, bool fromWebPage) {
 203     QSettings settings;
 204     QString definitionName = settings.value("definition", "360p").toString();
 205     int definitionCode = VideoDefinition::getDefinitionCode(definitionName);
 206
 207     // qDebug() << "fmtUrlMap" << fmtUrlMap;
 208     QStringList formatUrls = fmtUrlMap.split(',', QString::SkipEmptyParts);
 209     QHash<int, QString> urlMap;
 210     foreach(QString formatUrl, formatUrls) {
 211         // qDebug() << "formatUrl" << formatUrl;
 212         QStringList urlParams = formatUrl.split('&', QString::SkipEmptyParts);
 213         // qDebug() << "urlParams" << urlParams;
 214
 215         int format = -1;
 216         QString url;
 217         QString sig;
 218         foreach(QString urlParam, urlParams) {
 219             // qWarning() << urlParam;
 220             if (urlParam.startsWith("itag=")) {
 221                 int separator = urlParam.indexOf("=");
 222                 format = urlParam.mid(separator + 1).toInt();
 223             } else if (urlParam.startsWith("url=")) {
 224                 int separator = urlParam.indexOf("=");
 225                 url = urlParam.mid(separator + 1);
 226                 url = QByteArray::fromPercentEncoding(url.toUtf8());
 227             } else if (urlParam.startsWith("sig=")) {
 228                 int separator = urlParam.indexOf("=");
 229                 sig = urlParam.mid(separator + 1);
 230                 sig = QByteArray::fromPercentEncoding(sig.toUtf8());
 231             } else if (urlParam.startsWith("s=")) {
 232                 if (fromWebPage || ageGate) {
 233                     int separator = urlParam.indexOf("=");
 234                     sig = urlParam.mid(separator + 1);
 235                     sig = QByteArray::fromPercentEncoding(sig.toUtf8());
 236                     if (ageGate)
 237                         sig = JsFunctions::instance()->decryptAgeSignature(sig);
 238                     else {
 239                         sig = decryptSignature(sig);
 240                         if (sig.isEmpty())
 241                             sig = JsFunctions::instance()->decryptSignature(sig);
 242                     }
 243                 } else {
 244                     // qDebug() << "Loading webpage";
 245                     QUrl url("http://www.youtube.com/watch");
 246
 247 #if QT_VERSION >= 0x050000
 248                     {
 249                         QUrl &u = url;
 250                         QUrlQuery url;
 251 #endif
 252                         url.addQueryItem("v", videoId);
 253                         url.addQueryItem("gl", "US");
 254                         url.addQueryItem("hl", "en");
 255                         url.addQueryItem("has_verified", "1");
 256 #if QT_VERSION >= 0x050000
 257                         u.setQuery(url);
 258                     }
 259 #endif
 260                     QObject *reply = The::http()->get(url);
 261                     connect(reply, SIGNAL(data(QByteArray)), SLOT(scrapeWebPage(QByteArray)));
 262                     connect(reply, SIGNAL(error(QNetworkReply*)), SLOT(errorVideoInfo(QNetworkReply*)));
 263                     // see you in scrapWebPage(QByteArray)
 264                     return;
 265                 }
 266             }
 267         }
 268         if (format == -1 || url.isNull()) continue;
 269
 270         url += "&signature=" + sig;
 271
 272         if (!url.contains("ratebypass"))
 273             url += "&ratebypass=yes";
 274
 275         // qWarning() << url;
 276
 277         if (format == definitionCode) {
 278             qDebug() << "Found format" << definitionCode;
 279             QUrl videoUrl = QUrl::fromEncoded(url.toUtf8(), QUrl::StrictMode);
 280             m_streamUrl = videoUrl;
 281             this->definitionCode = definitionCode;
 282             emit gotStreamUrl(videoUrl);
 283             loadingStreamUrl = false;
 284             return;
 285         }
 286
 287         urlMap.insert(format, url);
 288     }
 289
 290     QList<int> definitionCodes = VideoDefinition::getDefinitionCodes();
 291     int currentIndex = definitionCodes.indexOf(definitionCode);
 292     int previousIndex = 0;
 293     while (currentIndex >= 0) {
 294         previousIndex = currentIndex - 1;
 295         if (previousIndex < 0) previousIndex = 0;
 296         int definitionCode = definitionCodes.at(previousIndex);
 297         if (urlMap.contains(definitionCode)) {
 298             qDebug() << "Found format" << definitionCode;
 299             QString url = urlMap.value(definitionCode);
 300             QUrl videoUrl = QUrl::fromEncoded(url.toUtf8(), QUrl::StrictMode);
 301             m_streamUrl = videoUrl;
 302             this->definitionCode = definitionCode;
 303             emit gotStreamUrl(videoUrl);
 304             loadingStreamUrl = false;
 305             return;
 306         }
 307         currentIndex--;
 308     }
 309
 310     emit errorStreamUrl(tr("Cannot get video stream for %1").arg(m_webpage.toString()));
 311 }
 312
 313 void Video::foundVideoUrl(QString videoToken, int definitionCode) {
 314     // qDebug() << "foundVideoUrl" << videoToken << definitionCode;
 315
 316     QUrl videoUrl = QUrl(QString(
 317                              "http://www.youtube.com/get_video?video_id=%1&t=%2&eurl=&el=&ps=&asv=&fmt=%3"
 318                              ).arg(videoId, videoToken, QString::number(definitionCode)));
 319
 320     m_streamUrl = videoUrl;
 321     loadingStreamUrl = false;
 322     emit gotStreamUrl(videoUrl);
 323 }
 324
 325 void Video::errorVideoInfo(QNetworkReply *reply) {
 326     loadingStreamUrl = false;
 327     emit errorStreamUrl(tr("Network error: %1 for %2").arg(reply->errorString(), reply->url().toString()));
 328 }
 329
 330 void Video::scrapeWebPage(QByteArray data) {
 331     QString html = QString::fromUtf8(data);
 332     // qWarning() << html;
 333
 334     if (html.contains("player-age-gate-content\"")) {
 335         // qDebug() << "Found ageGate";
 336         ageGate = true;
 337         elIndex = 4;
 338         getVideoInfo();
 339         return;
 340     }
 341
 342     QRegExp re(".*\"url_encoded_fmt_stream_map\":\\s+\"([^\"]+)\".*");
 343     bool match = re.exactMatch(html);
 344     // on regexp failure, stop and report error
 345     if (!match) {
 346         qWarning() << "Error parsing video page";
 347         // emit errorStreamUrl("Error parsing video page");
 348         // loadingStreamUrl = false;
 349         elIndex++;
 350         getVideoInfo();
 351         return;
 352     }
 353     fmtUrlMap = re.cap(1);
 354     fmtUrlMap.replace("\\u0026", "&");
 355     // parseFmtUrlMap(fmtUrlMap, true);
 356
 357 #ifdef APP_DASH
 358     QSettings settings;
 359     QString definitionName = settings.value("definition", "360p").toString();
 360     if (definitionName == QLatin1String("1080p") {
 361         QRegExp dashManifestRe("\"dashmpd\":\\s*\"([^\"]+)\"");
 362         if (dashManifestRe.indexIn(html) != -1) {
 363             dashManifestUrl = dashManifestRe.cap(1);
 364             dashManifestUrl.remove('\\');
 365             qDebug() << "dashManifestUrl" << dashManifestUrl;
 366         }
 367     }
 368 #endif
 369
 370     QRegExp jsPlayerRe("\"assets\":.+\"js\":\\s*\"([^\"]+)\"");
 371     if (jsPlayerRe.indexIn(html) != -1) {
 372         QString jsPlayerUrl = jsPlayerRe.cap(1);
 373         jsPlayerUrl.remove('\\');
 374         jsPlayerUrl = "http:" + jsPlayerUrl;
 375         // qDebug() << "jsPlayerUrl" << jsPlayerUrl;
 376         /*
 377         QRegExp jsPlayerIdRe("-(.+)\\.js");
 378         jsPlayerIdRe.indexIn(jsPlayerUrl);
 379         QString jsPlayerId = jsPlayerRe.cap(1);
 380         */
 381         QObject *reply = The::http()->get(jsPlayerUrl);
 382         connect(reply, SIGNAL(data(QByteArray)), SLOT(parseJsPlayer(QByteArray)));
 383         connect(reply, SIGNAL(error(QNetworkReply*)), SLOT(errorVideoInfo(QNetworkReply*)));
 384     }
 385 }
 386
 387 void Video::gotHeadHeaders(QNetworkReply* reply) {
 388     int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
 389     // qDebug() << "gotHeaders" << statusCode;
 390     if (statusCode == 200) {
 391         foundVideoUrl(videoToken, definitionCode);
 392     } else {
 393
 394         // try next (lower quality) definition
 395         /*
 396         QStringList definitionNames = VideoDefinition::getDefinitionNames();
 397         int currentIndex = definitionNames.indexOf(currentDefinition);
 398         int previousIndex = 0;
 399         if (currentIndex > 0) {
 400             previousIndex = currentIndex - 1;
 401         }
 402         if (previousIndex > 0) {
 403             QString nextDefinitionName = definitionNames.at(previousIndex);
 404             findVideoUrl(nextDefinitionName);
 405         } else {
 406             foundVideoUrl(videoToken, 18);
 407         }*/
 408
 409
 410         QList<int> definitionCodes = VideoDefinition::getDefinitionCodes();
 411         int currentIndex = definitionCodes.indexOf(definitionCode);
 412         int previousIndex = 0;
 413         if (currentIndex > 0) {
 414             previousIndex = currentIndex - 1;
 415             int definitionCode = definitionCodes.at(previousIndex);
 416             if (definitionCode == 18) {
 417                 // This is assumed always available
 418                 foundVideoUrl(videoToken, 18);
 419             } else {
 420                 findVideoUrl(definitionCode);
 421             }
 422
 423         } else {
 424             foundVideoUrl(videoToken, 18);
 425         }
 426
 427     }
 428 }
 429
 430 void Video::parseJsPlayer(QByteArray bytes) {
 431     QString js = QString::fromUtf8(bytes);
 432     // qWarning() << "jsPlayer" << js;
 433     QRegExp funcNameRe("signature=([" + jsNameChars + "]+)");
 434     if (funcNameRe.indexIn(js) == -1) {
 435         qWarning() << "Cannot capture signature function name";
 436     } else {
 437         sigFuncName = funcNameRe.cap(1);
 438         captureFunction(sigFuncName, js);
 439         // qWarning() << sigFunctions;
 440     }
 441
 442 #ifdef APP_DASH
 443     if (!dashManifestUrl.isEmpty()) {
 444         QRegExp sigRe("/s/([\\w\\.]+)");
 445         if (sigRe.indexIn(dashManifestUrl) != -1) {
 446             qDebug() << "Decrypting signature for dash manifest";
 447             QString sig = sigRe.cap(1);
 448             sig = decryptSignature(sig);
 449             dashManifestUrl.replace(sigRe, "/signature/" + sig);
 450             qDebug() << dashManifestUrl;
 451
 452             m_streamUrl = dashManifestUrl;
 453             this->definitionCode = 37;
 454             emit gotStreamUrl(m_streamUrl);
 455             loadingStreamUrl = false;
 456
 457             /*
 458             QObject *reply = The::http()->get(QUrl::fromEncoded(dashManifestUrl.toUtf8()));
 459             connect(reply, SIGNAL(data(QByteArray)), SLOT(parseDashManifest(QByteArray)));
 460             connect(reply, SIGNAL(error(QNetworkReply*)), SLOT(errorVideoInfo(QNetworkReply*)));
 461             */
 462
 463             return;
 464         }
 465     }
 466 #endif
 467
 468     parseFmtUrlMap(fmtUrlMap, true);
 469 }
 470
 471 void Video::parseDashManifest(QByteArray bytes) {
 472     QFile file(Temporary::filename());
 473     if (!file.open(QIODevice::WriteOnly))
 474         qWarning() << file.errorString() << file.fileName();
 475     QDataStream stream(&file);
 476     stream.writeRawData(bytes.constData(), bytes.size());
 477
 478     m_streamUrl = "file://" + file.fileName();
 479     this->definitionCode = 37;
 480     emit gotStreamUrl(m_streamUrl);
 481     loadingStreamUrl = false;
 482 }
 483
 484 void Video::captureFunction(const QString &name, const QString &js) {
 485     QRegExp funcRe("function\\s+" + QRegExp::escape(name) + "\\s*\\([" + jsNameChars + ",\\s]*\\)\\s*\\{[^\\}]+\\}");
 486     if (funcRe.indexIn(js) == -1) {
 487         qWarning() << "Cannot capture function" << name;
 488         return;
 489     }
 490     QString func = funcRe.cap(0);
 491     sigFunctions.insert(name, func);
 492
 493     // capture inner functions
 494     QRegExp invokedFuncRe("[\\s=;\\(]([" + jsNameChars + "]+)\\s*\\([" + jsNameChars + ",\\s]+\\)");
 495     int pos = name.length() + 9;
 496     while ((pos = invokedFuncRe.indexIn(func, pos)) != -1) {
 497         QString funcName = invokedFuncRe.cap(1);
 498         if (!sigFunctions.contains(funcName))
 499             captureFunction(funcName, js);
 500         pos += invokedFuncRe.matchedLength();
 501     }
 502
 503     // capture referenced objects
 504     QRegExp objRe("[\\s=;\\(]([" + jsNameChars + "]+)\\.[" + jsNameChars + "]+");
 505     pos = name.length() + 9;
 506     while ((pos = objRe.indexIn(func, pos)) != -1) {
 507         QString objName = objRe.cap(1);
 508         if (!sigObjects.contains(objName))
 509             captureObject(objName, js);
 510         pos += objRe.matchedLength();
 511     }
 512 }
 513
 514 void Video::captureObject(const QString &name, const QString &js) {
 515     QRegExp re("var\\s+" + QRegExp::escape(name) + "\\s*=\\s*\\{.+\\}\\s*;");
 516     re.setMinimal(true);
 517     if (re.indexIn(js) == -1) {
 518         qWarning() << "Cannot capture object" << name;
 519         return;
 520     }
 521     QString obj = re.cap(0);
 522     sigObjects.insert(name, obj);
 523 }
 524
 525 QString Video::decryptSignature(const QString &s) {
 526     if (sigFuncName.isEmpty()) return QString();
 527     QScriptEngine engine;
 528     foreach (QString f, sigObjects.values()) {
 529         QScriptValue value = engine.evaluate(f);
 530         if (value.isError())
 531             qWarning() << "Error in" << f << value.toString();
 532     }
 533     foreach (QString f, sigFunctions.values()) {
 534         QScriptValue value = engine.evaluate(f);
 535         if (value.isError())
 536             qWarning() << "Error in" << f << value.toString();
 537     }
 538     QString js = sigFuncName + "('" + s + "');";
 539     QScriptValue value = engine.evaluate(js);
 540     if (value.isUndefined()) {
 541         qWarning() << "Undefined result for" << js;
 542         return QString();
 543     }
 544     if (value.isError()) {
 545         qWarning() << "Error in" << js << value.toString();
 546         return QString();
 547     }
 548     return value.toString();
 549 }
 550
 551 void Video::findVideoUrl(int definitionCode) {
 552     this->definitionCode = definitionCode;
 553
 554     QUrl videoUrl = QUrl(QString(
 555                              "http://www.youtube.com/get_video?video_id=%1&t=%2&eurl=&el=&ps=&asv=&fmt=%3"
 556                              ).arg(videoId, videoToken, QString::number(definitionCode)));
 557
 558     QObject *reply = The::http()->head(videoUrl);
 559     connect(reply, SIGNAL(finished(QNetworkReply*)), SLOT(gotHeadHeaders(QNetworkReply*)));
 560     // connect(reply, SIGNAL(error(QNetworkReply*)), SLOT(errorVideoInfo(QNetworkReply*)));
 561
 562     // see you in gotHeadHeaders()
 563 }
 564
 565 QString Video::formattedDuration() const {
 566     QString format = m_duration > 3600 ? "h:mm:ss" : "m:ss";
 567     return QTime().addSecs(m_duration).toString(format);
 568 }