Skip to content

Commit

Permalink
Use SourceReader when parsing HTML to fetch feed link
Browse files Browse the repository at this point in the history
  • Loading branch information
msasikanth committed Oct 1, 2024
1 parent 16cbe35 commit 95e02b8
Showing 1 changed file with 13 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
package dev.sasikanth.rss.reader.core.network.fetcher

import com.fleeksoft.ksoup.Ksoup
import com.fleeksoft.ksoup.io.SourceReader
import com.fleeksoft.ksoup.io.from
import dev.sasikanth.rss.reader.core.network.parser.FeedParser
import dev.sasikanth.rss.reader.core.network.parser.FeedParser.Companion.ATOM_MEDIA_TYPE
import dev.sasikanth.rss.reader.core.network.parser.FeedParser.Companion.ATTR_HREF
Expand All @@ -26,13 +28,13 @@ import io.ktor.client.HttpClient
import io.ktor.client.request.get
import io.ktor.client.statement.HttpResponse
import io.ktor.client.statement.bodyAsChannel
import io.ktor.client.statement.bodyAsText
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.http.URLBuilder
import io.ktor.http.URLProtocol
import io.ktor.http.Url
import io.ktor.http.contentType
import io.ktor.utils.io.ByteReadChannel
import korlibs.io.lang.Charset
import korlibs.io.lang.Charsets
import me.tatarka.inject.annotations.Inject
Expand Down Expand Up @@ -99,7 +101,8 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
return FeedFetchResult.Success(feedPayload)
}

val feedUrl = fetchFeedLinkFromHtmlIfExists(response.bodyAsText(), url)
val feedUrl =
fetchFeedLinkFromHtmlIfExists(response = response.bodyAsChannel(), originalUrl = url)

if (feedUrl != url && !feedUrl.isNullOrBlank()) {
return fetch(url = feedUrl, transformUrl = false, redirectCount = redirectCount + 1)
Expand Down Expand Up @@ -141,10 +144,16 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
}
}

private fun fetchFeedLinkFromHtmlIfExists(htmlContent: String, originalUrl: String): String? {
private fun fetchFeedLinkFromHtmlIfExists(
response: ByteReadChannel,
originalUrl: String
): String? {
val document =
try {
Ksoup.parse(htmlContent)
Ksoup.parse(
sourceReader = SourceReader.from(response),
baseUri = originalUrl,
)
} catch (t: Throwable) {
return null
}
Expand Down

0 comments on commit 95e02b8

Please sign in to comment.