Skip to content

Commit

Permalink
[fix/ISSUE-83] Detect unplayable videos in embedded extractor
Browse files Browse the repository at this point in the history
  • Loading branch information
@azihassan
azihassan committed Dec 22, 2024
1 parent 083e5cf commit 29a4899
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 11 deletions.
8 changes: 4 additions & 4 deletions source/cache.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import std.zlib : UnCompress;
import std.json : JSONValue;

import helpers : StdoutLogger, parseID, parseQueryString, parseBaseJSKey, formatTitle, formatSuccess, formatWarning;
import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor, PlayerYoutubeVideoURLExtractor, parseYoutubeConfig;
import parsers : parseBaseJSURL, YoutubeVideoURLExtractor, SimpleYoutubeVideoURLExtractor, AdvancedYoutubeVideoURLExtractor, EmbeddedSimpleYoutubeVideoURLExtractor, parseYoutubeConfig;

string formatPlayerRequest(string videoId, string poToken, string clientPlayerNonce)
{
Expand Down Expand Up @@ -245,7 +245,7 @@ struct Cache
{
if(player != "")
{
return new PlayerYoutubeVideoURLExtractor(html, baseJS, player, poToken, clientPlayerNonce, logger);
return new EmbeddedSimpleYoutubeVideoURLExtractor(html, baseJS, player, poToken, clientPlayerNonce, logger);
}
immutable urlRegex = ctRegex!`"itag":\d+,"url":"(.*?)"`;
if(!html.matchFirst(urlRegex).empty)
Expand Down Expand Up @@ -421,7 +421,7 @@ unittest

unittest
{
writeln("Given PlayerYoutubeVideoURLExtractor, when cache is fresh, should not download HTML and player".formatTitle());
writeln("Given EmbeddedSimpleYoutubeVideoURLExtractor, when cache is fresh, should not download HTML and player".formatTitle());
scope(success) writeln("OK\n".formatSuccess());

SysTime tomorrow = Clock.currTime() + 1.days;
Expand All @@ -445,7 +445,7 @@ unittest

unittest
{
writeln("Given PlayerYoutubeVideoURLExtractor, when cache is stale, should download HTML and player".formatTitle());
writeln("Given EmbeddedSimpleYoutubeVideoURLExtractor, when cache is stale, should download HTML and player".formatTitle());
scope(success) writeln("OK\n".formatSuccess());

//base.js is already available in tests/ so no need to check it
Expand Down
41 changes: 34 additions & 7 deletions source/parsers.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ abstract class YoutubeVideoURLExtractor
}
}

abstract class HTMLYoutubeVideoURLExtractor : YoutubeVideoURLExtractor
abstract class WebYoutubeVideoURLExtractor : YoutubeVideoURLExtractor
{
override public void failIfUnplayable()
{
Expand Down Expand Up @@ -163,7 +163,7 @@ abstract class HTMLYoutubeVideoURLExtractor : YoutubeVideoURLExtractor
}
}

class SimpleYoutubeVideoURLExtractor : HTMLYoutubeVideoURLExtractor
class SimpleYoutubeVideoURLExtractor : WebYoutubeVideoURLExtractor
{
this(string html, StdoutLogger logger)
{
Expand Down Expand Up @@ -319,7 +319,7 @@ unittest
assert(YoutubeFormat(140, 9371359, "unknown", `foobar`, [AudioVisual.VIDEO]).extension == "mp4");
}

class AdvancedYoutubeVideoURLExtractor : HTMLYoutubeVideoURLExtractor
class AdvancedYoutubeVideoURLExtractor : WebYoutubeVideoURLExtractor
{
this(string html, string baseJS, StdoutLogger logger)
{
Expand Down Expand Up @@ -786,7 +786,7 @@ unittest
assert(expected == actual, expected ~ " != " ~ actual);
}

class PlayerYoutubeVideoURLExtractor : YoutubeVideoURLExtractor
class EmbeddedSimpleYoutubeVideoURLExtractor : YoutubeVideoURLExtractor
{
private JSONValue json;
private string clientPlaybackNonce;
Expand Down Expand Up @@ -844,8 +844,16 @@ class PlayerYoutubeVideoURLExtractor : YoutubeVideoURLExtractor

override public void failIfUnplayable()
{
//todo add me
logger.display("failIfUnplayable not implemented yet for embedded players".formatWarning());
if("playabilityStatus" !in this.json)
{
logger.display("Warning: playability status could not be parsed".formatWarning);
return;
}
auto playabilityStatus = this.json["playabilityStatus"];
if(playabilityStatus["status"].str != "OK")
{
throw new Exception("Video is unplayable because of status " ~ playabilityStatus["status"].str ~ " and reason: " ~ playabilityStatus["reason"].str);
}
}

override public string getTitle()
Expand All @@ -863,6 +871,25 @@ class PlayerYoutubeVideoURLExtractor : YoutubeVideoURLExtractor
return this.json["streamingData"];
}
}

unittest
{
import std.exception : collectExceptionMsg;
writeln("When embedded video is unplayable, should fail gracefully".formatTitle());
scope(success) writeln("OK\n".formatSuccess());

string html = readText("tests/cvDVjwMXiCs.html");
string baseJS = readText("tests/4e23410d.js");
string player = readText("tests/unplayable.json");
string cpn = generateClientPlayerNonce();
string poToken = "MnTzqqeGiL40LvOS8qO2zA4oPs9hKB03p_jFCpNuAOAzaPVsQNzkqKOokO8z4cu6Az_afF7dFchYJ_YHINMszhIrrmGEzU7E1sYY-fp78SP5me0kAWQ1nGt5Hgc0NiJZQdUtQMod6_9roD2TTmmLn6xTv1N2Vw==";

string exceptionMessage = collectExceptionMsg(new EmbeddedSimpleYoutubeVideoURLExtractor(html, baseJS, player, poToken, cpn, new StdoutLogger()));

string expectedExceptionMessage = "Video is unplayable because of status UNPLAYABLE and reason: Video unavailable";
assert(exceptionMessage == expectedExceptionMessage, "Expected message " ~ expectedExceptionMessage ~ " but got " ~ exceptionMessage);
}

unittest
{
writeln("When video is embedded, should parse from player JSON response".formatTitle());
Expand All @@ -871,7 +898,7 @@ unittest
string baseJS = readText("tests/4e23410d.js");
string player = readText("tests/cvDVjwMXiCs.json");
string cpn = generateClientPlayerNonce();
auto extractor = new PlayerYoutubeVideoURLExtractor(html, baseJS, player, "MnTzqqeGiL40LvOS8qO2zA4oPs9hKB03p_jFCpNuAOAzaPVsQNzkqKOokO8z4cu6Az_afF7dFchYJ_YHINMszhIrrmGEzU7E1sYY-fp78SP5me0kAWQ1nGt5Hgc0NiJZQdUtQMod6_9roD2TTmmLn6xTv1N2Vw==", cpn, new StdoutLogger());
auto extractor = new EmbeddedSimpleYoutubeVideoURLExtractor(html, baseJS, player, "MnTzqqeGiL40LvOS8qO2zA4oPs9hKB03p_jFCpNuAOAzaPVsQNzkqKOokO8z4cu6Az_afF7dFchYJ_YHINMszhIrrmGEzU7E1sYY-fp78SP5me0kAWQ1nGt5Hgc0NiJZQdUtQMod6_9roD2TTmmLn6xTv1N2Vw==", cpn, new StdoutLogger());

assert(extractor.getID() == "cvDVjwMXiCs");
assert(extractor.getTitle() == "A Cat Is More Terrifying Than a 200lbs Caucasian Shepherd Dog 😂");
Expand Down
1 change: 1 addition & 0 deletions tests/unplayable.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"responseContext":{"serviceTrackingParams":[{"service":"GFEEDBACK","params":[{"key":"is_viewed_live","value":"False"},{"key":"ipcc","value":"0"},{"key":"is_alc_surface","value":"false"},{"key":"wh_paused","value":"0"},{"key":"logged_in","value":"0"},{"key":"e","value":"23804281,23966208,24004644,24077241,24181174,24241378,24407446,24439361,24499534,24542367,24548629,24566687,51009781,51010235,51017346,51020570,51021189,51022792,51025415,51028055,51030103,51037342,51037353,51050361,51053689,51057848,51057853,51063643,51064835,51098299,51105630,51111738,51115184,51117319,51124104,51129210,51131427,51133103,51134506,51144925,51152050,51157411,51157841,51158514,51160545,51165467,51169118,51176511,51178320,51178331,51178346,51178351,51178982,51182850,51183910,51195231,51199253,51204329,51213773,51217504,51221152,51222382,51222973,51223962,51226936,51227037,51227778,51228350,51230241,51230478,51231814,51237842,51239093,51241028,51242448,51242767,51243940,51248255,51248734,51251836,51255676,51255680,51255743,51256074,51256084,51257897,51257902,51257911,51257914,51258066,51260456,51265345,51265364,51265367,51266454,51272663,51273608,51274583,51275785,51276557,51276565,51281227,51282073,51282086,51285417,51285717,51287196,51287500,51288345,51289483,51289924,51289933,51289938,51289954,51289967,51289976,51294322,51295132,51295576,51296439,51298019,51298020,51299626,51299710,51299724,51299973,51300005,51300016,51300699,51302492,51302680,51303667,51303670,51303789,51304155,51304660,51305531,51305839,51307502,51308045,51308060,51309313,51310323,51310742,51311027,51311040,51312146,51312688,51313149,51313767,51314679,51314694,51314707,51314714,51314727,51315041,51315914,51315919,51315924,51315935,51315942,51315945,51315954,51315963,51315968,51315979,51316745,51317749,51318207,51318844,51323297,51323366,51324817,51325576,51326281,51326641,51326760,51326932,51327140,51327163,51327186,51327616,51328144,51329146,51329227,51329505,51330194,51330475,51331487,51331504,51331520,51331531,51331542,51331545,51331554,51331559,51331692,51333739,51333878,51337140,51337187,51337350,51339163,51339747,51340613,51341226,51341758,51342093,51343109,51343244,51343368,51345126,51345230"}]},{"service":"CSI","params":[{"key":"c","value":"WEB_EMBEDDED_PLAYER"},{"key":"cver","value":"1.20241029.01.00"},{"key":"yt_li","value":"0"},{"key":"GetPlayer_rid","value":"0x05b636e5558b795f"}]},{"service":"GUIDED_HELP","params":[{"key":"logged_in","value":"0"}]},{"service":"ECATCHER","params":[{"key":"client.version","value":"20241029"},{"key":"client.name","value":"WEB_EMBEDDED_PLAYER"}]}],"maxAgeSeconds":0},"playabilityStatus":{"status":"UNPLAYABLE","reason":"Video unavailable","errorScreen":{"playerErrorMessageRenderer":{"reason":{"runs":[{"text":"Video unavailable"}]},"proceedButton":{"buttonRenderer":{"style":"STYLE_DEFAULT","size":"SIZE_DEFAULT","isDisabled":false,"text":{"simpleText":"Watch on YouTube"},"navigationEndpoint":{"clickTrackingParams":"CAEQ8FsiEwiEn6LstNCJAxVVgbEDHX4vE2c=","urlEndpoint":{"url":"http://www.youtube.com/watch?v=dQw4w9WgXcQ","target":"TARGET_NEW_WINDOW"}},"trackingParams":"CAEQ8FsiEwiEn6LstNCJAxVVgbEDHX4vE2c="}},"thumbnail":{"thumbnails":[{"url":"//s.ytimg.com/yts/img/meh7-vflGevej7.png","width":140,"height":100}]},"icon":{"iconType":"ERROR_OUTLINE"}}},"contextParams":"Q0FFU0FnZ0I="},"videoDetails":{"videoId":"dQw4w9WgXcQ","title":"Rick Astley - Never Gonna Give You Up (Official Music Video)","lengthSeconds":"212","keywords":["rick astley","Never Gonna Give You Up","nggyu","never gonna give you up lyrics","rick rolled","Rick Roll","rick astley official","rickrolled","Fortnite song","Fortnite event","Fortnite dance","fortnite never gonna give you up","rick roll","rickrolling","rick rolling","never gonna give you up","80s music","rick astley new","animated video","rickroll","meme songs","never gonna give u up lyrics","Rick Astley 2022","never gonna let you down","animated","rick rolls 2022","never gonna give you up karaoke"],"channelId":"UCuAXFkgsw1L7xaCfnd5JJOw","isOwnerViewing":false,"shortDescription":"The official video for “Never Gonna Give You Up” by Rick Astley. \n\nNever: The Autobiography 📚 OUT NOW! \nFollow this link to get your copy and listen to Rick’s ‘Never’ playlist ❤️ #RickAstleyNever\nhttps://linktr.ee/rickastleynever\n\n“Never Gonna Give You Up” was a global smash on its release in July 1987, topping the charts in 25 countries including Rick’s native UK and the US Billboard Hot 100. It also won the Brit Award for Best single in 1988. Stock Aitken and Waterman wrote and produced the track which was the lead-off single and lead track from Rick’s debut LP “Whenever You Need Somebody”. The album was itself a UK number one and would go on to sell over 15 million copies worldwide.\n\nThe legendary video was directed by Simon West – who later went on to make Hollywood blockbusters such as Con Air, Lara Croft – Tomb Raider and The Expendables 2. The video passed the 1bn YouTube views milestone on 28 July 2021.\n\nSubscribe to the official Rick Astley YouTube channel: https://RickAstley.lnk.to/YTSubID\n\nFollow Rick Astley:\nFacebook: https://RickAstley.lnk.to/FBFollowID \nTwitter: https://RickAstley.lnk.to/TwitterID \nInstagram: https://RickAstley.lnk.to/InstagramID \nWebsite: https://RickAstley.lnk.to/storeID \nTikTok: https://RickAstley.lnk.to/TikTokID\n\nListen to Rick Astley:\nSpotify: https://RickAstley.lnk.to/SpotifyID \nApple Music: https://RickAstley.lnk.to/AppleMusicID \nAmazon Music: https://RickAstley.lnk.to/AmazonMusicID \nDeezer: https://RickAstley.lnk.to/DeezerID \n\nLyrics:\nWe’re no strangers to love\nYou know the rules and so do I\nA full commitment’s what I’m thinking of\nYou wouldn’t get this from any other guy\n\nI just wanna tell you how I’m feeling\nGotta make you understand\n\nNever gonna give you up\nNever gonna let you down\nNever gonna run around and desert you\nNever gonna make you cry\nNever gonna say goodbye\nNever gonna tell a lie and hurt you\n\nWe’ve known each other for so long\nYour heart’s been aching but you’re too shy to say it\nInside we both know what’s been going on\nWe know the game and we’re gonna play it\n\nAnd if you ask me how I’m feeling\nDon’t tell me you’re too blind to see\n\nNever gonna give you up\nNever gonna let you down\nNever gonna run around and desert you\nNever gonna make you cry\nNever gonna say goodbye\nNever gonna tell a lie and hurt you\n\n#RickAstley #NeverGonnaGiveYouUp #WheneverYouNeedSomebody #OfficialMusicVideo","isCrawlable":true,"thumbnail":{"thumbnails":[{"url":"https://i.ytimg.com/vi_webp/dQw4w9WgXcQ/default.webp","width":120,"height":90},{"url":"https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg?sqp=-oaymwEWCKgBEF5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLBf45ycDkHw60O072DkeCeAw0TV-w","width":168,"height":94},{"url":"https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg?sqp=-oaymwEWCMQBEG5IWvKriqkDCQgBFQAAiEIYAQ==&rs=AOn4CLC4kE2xK3nLa36ObM_eu_KdLSXlVQ","width":196,"height":110},{"url":"https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg?sqp=-oaymwEXCPYBEIoBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBQZfoSOXTWVau7H6ztlvfGyeB9nQ","width":246,"height":138},{"url":"https://i.ytimg.com/vi_webp/dQw4w9WgXcQ/mqdefault.webp","width":320,"height":180},{"url":"https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg?sqp=-oaymwEXCNACELwBSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLB56MmlqspsJNcQGtiZLu0tjiSolQ","width":336,"height":188},{"url":"https://i.ytimg.com/vi_webp/dQw4w9WgXcQ/hqdefault.webp","width":480,"height":360},{"url":"https://i.ytimg.com/vi_webp/dQw4w9WgXcQ/sddefault.webp","width":640,"height":480},{"url":"https://i.ytimg.com/vi_webp/dQw4w9WgXcQ/maxresdefault.webp","width":1920,"height":1080},{"url":"https://i.ytimg.com/vi_webp/dQw4w9WgXcQ/maxresdefault.webp","width":1920,"height":1080}]},"allowRatings":true,"viewCount":"1590129868","author":"Rick Astley","isPrivate":false,"isUnpluggedCorpus":false,"isLiveContent":false},"trackingParams":"CAAQu2kiEwiEn6LstNCJAxVVgbEDHX4vE2c=","adBreakHeartbeatParams":"Q0FBJTNE"}

0 comments on commit 29a4899

Please sign in to comment.