[fix/ISSUE-83] Parse adaptive format video length from JSON instead o…

…f HEAD request
azihassan · Oct 20, 2024 · 6ec1c6d · 6ec1c6d
1 parent 085e995
commit 6ec1c6d
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 13 deletions.
diff --git a/source/app.d b/source/app.d
@@ -172,7 +172,7 @@ void handleURL(string url, int itag, StdoutLogger logger, bool displayFormats, b
     {
         logger.display("Using RegularDownloader");
         bool finished = false;
-        downloader = new RegularDownloader(logger, (size_t total, size_t current) {
+        downloader = new RegularDownloader(logger, youtubeFormat, (size_t total, size_t current) {
             if(current == 0 || total == 0 || finished)
             {
                 return 0;

diff --git a/source/downloaders.d b/source/downloaders.d
@@ -21,12 +21,14 @@ class RegularDownloader : Downloader
     private StdoutLogger logger;
     private int delegate(ulong length, ulong currentLength) onProgress;
     private bool progress;
+    private YoutubeFormat youtubeFormat;
 
-    this(StdoutLogger logger, int delegate(ulong length, ulong currentLength) onProgress, bool progress = true)
+    this(StdoutLogger logger, YoutubeFormat youtubeFormat, int delegate(ulong length, ulong currentLength) onProgress, bool progress = true)
     {
         this.logger = logger;
         this.onProgress = onProgress;
         this.progress = progress;
+        this.youtubeFormat = youtubeFormat;
     }
 
     public void download(string destination, string url, string referer)
@@ -47,8 +49,8 @@ class RegularDownloader : Downloader
 
         http.verbose(logger.verbose);
         auto curl = http.handle();
-        ulong length = url.getContentLength();
-        logger.displayVerbose("Length = ", length);
+        ulong length = url.getContentLength(youtubeFormat);
+        logger.display("Length = ", length);
         if(destination.exists() && destination.getSize() == length)
         {
             logger.display("Done !".formatSuccess());
@@ -98,10 +100,14 @@ unittest
     server.listen(1);
 
     task!(() {
-        new RegularDownloader(new StdoutLogger(), (ulong length, ulong currentLength) { return 0; }).download(
-                "destination.mp4",
-                "http://127.0.0.1:1234/destination.mp4",
-                "Random referer"
+        new RegularDownloader(
+                new StdoutLogger(true),
+                YoutubeFormat(18, 0L, "360p", "video/mp4", []),
+                (ulong length, ulong currentLength) { return 0; }
+        ).download(
+            "destination.mp4",
+            "http://127.0.0.1:1234/destination.mp4?",
+            "Random referer"
         );
     }).executeInNewThread();
 
@@ -157,7 +163,7 @@ class ParallelDownloader : Downloader
 
     public void download(string destination, string url, string referer)
     {
-        ulong length = url.getContentLength();
+        ulong length = url.getContentLength(youtubeFormat);
         logger.displayVerbose("Length = ", length);
         if(destination.exists() && destination.getSize() == length)
         {
@@ -187,7 +193,7 @@ class ParallelDownloader : Downloader
                 logger.displayVerbose(partialDestination, " already has ", partialDestination.getSize(), " bytes, skipping");
                 continue;
             }
-            new RegularDownloader(logger, (ulong _, ulong __) {
+            new RegularDownloader(logger, youtubeFormat, (ulong _, ulong __) {
                 if(length == 0)
                 {
                     return 0;

diff --git a/source/helpers.d b/source/helpers.d
@@ -4,11 +4,32 @@ import std.regex : ctRegex, matchFirst, escaper, regex, Captures;
 import std.algorithm : filter;
 import std.conv : to;
 import std.net.curl : HTTP;
-import std.string : split, indexOf, startsWith;
+import std.string : split, indexOf, startsWith, endsWith;
 import std.format : formattedRead;
 
-ulong getContentLength(string url)
+import parsers : YoutubeFormat, AudioVisual;
+
+ulong getContentLength(string url, YoutubeFormat youtubeFormat)
 {
+    writeln("url = ", url);
+    writeln("youtubeFormat = ", youtubeFormat.length);
+    if(youtubeFormat.length != 0)
+    {
+        writeln("return ", youtubeFormat.length);
+        return youtubeFormat.length;
+    }
+
+    writeln("queryString = ");
+    string[string] queryString = url.parseQueryString();
+    writeln("queryString = ", queryString);
+    if("range" in queryString && !queryString["range"].endsWith("-"))
+    {
+        string[] limits = queryString["range"].split("-");
+        writeln("return ", limits);
+        return limits[1].to!ulong - limits[0].to!ulong;
+    }
+
+    writeln("Sending head request");
     auto http = HTTP(url);
     http.method = HTTP.Method.head;
     http.addRequestHeader("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0");
@@ -17,9 +38,16 @@ ulong getContentLength(string url)
     {
         throw new Exception("Failed with status " ~ http.statusLine.code.to!string);
     }
+    writeln("return ", http.responseHeaders["content-length"]);
     return http.responseHeaders["content-length"].to!ulong;
 }
 
+unittest
+{
+    assert(1337L == "".getContentLength(YoutubeFormat(232, 1337L, "360p", "video/mp4", [AudioVisual.VIDEO])));
+    assert(42L == "https://www.youtube.com/video.mp4?foo=bar&range=10-52&bar=baz".getContentLength(YoutubeFormat(18, 0L, "360p", "video/mp4", [AudioVisual.VIDEO])));
+}
+
 string sanitizePath(string path)
 {
     bool[dchar] reserved = [
@@ -38,12 +66,12 @@ string sanitizePath(string path)
 
 string[string] parseQueryString(string input)
 {
+    string[string] result;
     auto questionMarkIndex = input.indexOf("?");
     if(questionMarkIndex != -1)
     {
         input = input[questionMarkIndex + 1 .. $];
     }
-    string[string] result;
     foreach(params; input.split("&"))
     {
         string[] parts = params.split("=");