From 0c266c1fb5ffe85e6d93d5bac762f4a1bfcfcd51 Mon Sep 17 00:00:00 2001 From: Oliver Gassner Date: Thu, 24 Oct 2024 21:32:02 +0200 Subject: [PATCH] [IMPROVEMENT] Prevent 429 error with bandcamp api * add random wait between requests * increases runtime but handles my > 5000 releases collection fine. --- .../bandcampcollectiondownloader/core/BandcampAPIConnector.kt | 3 +++ .../core/BandcampCollectionDownloader.kt | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/bandcampcollectiondownloader/core/BandcampAPIConnector.kt b/src/main/kotlin/bandcampcollectiondownloader/core/BandcampAPIConnector.kt index ba9d518..bb13ccb 100644 --- a/src/main/kotlin/bandcampcollectiondownloader/core/BandcampAPIConnector.kt +++ b/src/main/kotlin/bandcampcollectiondownloader/core/BandcampAPIConnector.kt @@ -7,6 +7,8 @@ import org.jsoup.HttpStatusException import org.jsoup.Jsoup import java.util.* import java.util.regex.Pattern +import java.util.concurrent.TimeUnit + class BandcampAPIConnector constructor( private val bandcampUser: String, @@ -166,6 +168,7 @@ class BandcampAPIConnector constructor( // Append download pages from this api endpoint as well val theRest = util.retry({ + TimeUnit.SECONDS.sleep((1 + Random().nextInt(2)).toLong()) Jsoup.connect("https://bandcamp.com/api/fancollection/1/${collectionName}") .ignoreContentType(true) .timeout(timeout) diff --git a/src/main/kotlin/bandcampcollectiondownloader/core/BandcampCollectionDownloader.kt b/src/main/kotlin/bandcampcollectiondownloader/core/BandcampCollectionDownloader.kt index 1a8bd80..f5dba3e 100644 --- a/src/main/kotlin/bandcampcollectiondownloader/core/BandcampCollectionDownloader.kt +++ b/src/main/kotlin/bandcampcollectiondownloader/core/BandcampCollectionDownloader.kt @@ -14,6 +14,7 @@ import java.time.temporal.ChronoField import java.util.* import java.util.concurrent.* import java.util.stream.Collectors +import java.util.concurrent.TimeUnit class BandcampCollectionDownloader( private val args: Command, @@ -150,7 +151,7 @@ class BandcampCollectionDownloader( } private fun manageDownloadPage(connector: BandcampAPIConnector, saleItemId: String, cache: Cache) { - + TimeUnit.SECONDS.sleep((1 + Random().nextInt(2)).toLong()) val digitalItem = connector.retrieveDigitalItemData(saleItemId) // If null, then the download page is simply invalid and not usable anymore, therefore it can be added to the cache