diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index a15d3ed012..0c1e8110aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -27,6 +27,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicTrendingExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; @@ -35,6 +36,7 @@ import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeTrendingExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeCommentsLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeMusicTrendingLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubePlaylistLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; @@ -140,6 +142,16 @@ public KioskList getKioskList() throws ExtractionException { // add kiosks here e.g.: try { + list.addKioskEntry( + (streamingService, url, id) -> new YoutubeMusicTrendingExtractor( + YoutubeService.this, + new YoutubeMusicTrendingLinkHandlerFactory().fromUrl(url), + id + ), + new YoutubeMusicTrendingLinkHandlerFactory(), + "Trending Music" + ); + list.addKioskEntry( (streamingService, url, id) -> new YoutubeTrendingExtractor( YoutubeService.this, @@ -155,6 +167,7 @@ public KioskList getKioskList() throws ExtractionException { } return list; + } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicTrendingExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicTrendingExtractor.java new file mode 100644 index 0000000000..e4359901e1 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicTrendingExtractor.java @@ -0,0 +1,158 @@ +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2018 + * YoutubeTrendingExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + +package org.schabi.newpipe.extractor.services.youtube.extractors; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextAtKey; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; + +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.kiosk.KioskExtractor; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class YoutubeMusicTrendingExtractor extends KioskExtractor { + private JsonObject initialData; + + public YoutubeMusicTrendingExtractor(final StreamingService service, + final ListLinkHandler linkHandler, + final String kioskId) { + super(service, linkHandler, kioskId); + } + + @Override + public void onFetchPage(@Nonnull final Downloader downloader) + throws IOException, ExtractionException { + // @formatter:off + final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(), + getExtractorContentCountry()) + .value("browseId", "MUtrending") + .done()) + .getBytes(UTF_8); + // @formatter:on + + initialData = getJsonPostResponse("browse", body, getExtractorLocalization()); + } + + @Override + public InfoItemsPage getPage(final Page page) { + return InfoItemsPage.emptyPage(); + } + + @Nonnull + @Override + public String getName() throws ParsingException { + final JsonObject header = initialData.getObject("header"); + String name = null; + if (header.has("feedTabbedHeaderRenderer")) { + name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title"); + } else if (header.has("c4TabbedHeaderRenderer")) { + name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title"); + } + + if (isNullOrEmpty(name)) { + throw new ParsingException("Could not get Trending name"); + } + return name; + } + + @Nonnull + @Override + public InfoItemsPage getInitialPage() throws ParsingException { + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final TimeAgoParser timeAgoParser = getTimeAgoParser(); + final JsonObject tabContent = getTrendingTabContent(); + + if (tabContent.has("richGridRenderer")) { + tabContent.getObject("richGridRenderer") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + // Filter Trending shorts and Recently trending sections + .filter(content -> content.has("richItemRenderer")) +// .filter(shelfRenderer -> !shelfRenderer.has("music")) + .map(content -> content.getObject("richItemRenderer") + .getObject("content") + .getObject("videoRenderer")) + .forEachOrdered(videoRenderer -> collector.commit( + new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); + } else if (tabContent.has("sectionListRenderer")) { + tabContent.getObject("sectionListRenderer") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .flatMap(content -> content.getObject("itemSectionRenderer") + .getArray("contents") + .stream()) + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(content -> content.getObject("shelfRenderer")) + // Filter Trending shorts and Recently trending sections which have a title, + // contrary to normal trends + .filter(shelfRenderer -> !shelfRenderer.has("title")) +// .filter(shelfRenderer -> !shelfRenderer.has("music")) + .flatMap(shelfRenderer -> shelfRenderer.getObject("content") + .getObject("expandedShelfContentsRenderer") + .getArray("items") + .stream()) + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(item -> item.getObject("videoRenderer")) + .forEachOrdered(videoRenderer -> collector.commit( + new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser))); + } + + return new InfoItemsPage<>(collector, null); + } + + private JsonObject getTrendingTabContent() throws ParsingException { + return initialData.getObject("contents") + .getObject("twoColumnBrowseResultsRenderer") + .getArray("tabs") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(tab -> tab.getObject("tabRenderer")) + .filter(tabRenderer -> tabRenderer.getBoolean("selected")) + .filter(tabRenderer -> tabRenderer.has("content")) + // There should be at most one tab selected + .findFirst() + .orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab")) + .getObject("content"); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeMusicTrendingLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeMusicTrendingLinkHandlerFactory.java new file mode 100644 index 0000000000..d5122f07ed --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeMusicTrendingLinkHandlerFactory.java @@ -0,0 +1,43 @@ +package org.schabi.newpipe.extractor.services.youtube.linkHandler; + +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isInvidioURL; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isYoutubeURL; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; +import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.Utils; + +public final class YoutubeMusicTrendingLinkHandlerFactory extends ListLinkHandlerFactory { + + public String getUrl(final String id, + final List contentFilters, + final String sortFilter) { + return "https://www.youtube.com/feed/trending/music"; + } + + @Override + public String getId(final String url) { + return "Trending"; + } + + @Override + public boolean onAcceptUrl(final String url) { + final URL urlObj; + try { + urlObj = Utils.stringToURL(url); + } catch (final MalformedURLException e) { + return false; + } + + final String urlPath = urlObj.getPath(); + return Utils.isHTTP(urlObj) && (isYoutubeURL(urlObj) || isInvidioURL(urlObj)) + && urlPath.equals("/feed/trending/music"); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingKioskInfoTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingKioskInfoTest.java new file mode 100644 index 0000000000..ca554ed513 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingKioskInfoTest.java @@ -0,0 +1,69 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingKioskInfoTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.schabi.newpipe.downloader.DownloaderFactory; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.kiosk.KioskInfo; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; + +/** + * Test for {@link KioskInfo} + */ +public class YoutubeMusicTrendingKioskInfoTest { + + private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "kiosk"; + + static KioskInfo kioskInfo; + + @BeforeAll + public static void setUp() + throws Exception { + YoutubeTestsUtils.ensureStateless(); + NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH)); + LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending"); + + kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl()); + } + + @Test + public void getStreams() { + assertFalse(kioskInfo.getRelatedItems().isEmpty()); + } + + @Test + public void getId() { + assertTrue(kioskInfo.getId().equals("Trending Music") + || kioskInfo.getId().equals("Trends")); + } + + @Test + public void getName() { + assertFalse(kioskInfo.getName().isEmpty()); + } +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingLinkHandlerFactoryTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingLinkHandlerFactoryTest.java new file mode 100644 index 0000000000..3f936eb472 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMusicTrendingLinkHandlerFactoryTest.java @@ -0,0 +1,87 @@ +package org.schabi.newpipe.extractor.services.youtube; + +/* + * Created by Christian Schabesberger on 12.08.17. + * + * Copyright (C) Christian Schabesberger 2017 + * YoutubeTrendingLinkHandlerFactoryTest.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.schabi.newpipe.extractor.ServiceList.YouTube; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.schabi.newpipe.downloader.DownloaderTestImpl; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeTrendingLinkHandlerFactory; + +/** + * Test for {@link YoutubeTrendingLinkHandlerFactory} + */ +public class YoutubeMusicTrendingLinkHandlerFactoryTest { + private static LinkHandlerFactory LinkHandlerFactory; + + @BeforeAll + public static void setUp() throws Exception { + LinkHandlerFactory = YouTube.getKioskList().getListLinkHandlerFactoryByType("Trending Music"); + NewPipe.init(DownloaderTestImpl.getInstance()); + } + + @Test + public void getUrl() + throws Exception { + assertEquals(LinkHandlerFactory.fromId("").getUrl(), "https://www.youtube.com/feed/trending/music"); + } + + @Test + public void getId() + throws Exception { + assertEquals(LinkHandlerFactory.fromUrl("https://www.youtube.com/feed/trending/music").getId(), "Trending"); + } + + @Test + public void acceptUrl() throws ParsingException { + assertTrue(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music")); + assertTrue(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music/?adsf=fjaj#fhe")); + assertTrue(LinkHandlerFactory.acceptUrl("http://www.youtube.com/feed/trending/music/")); + assertTrue(LinkHandlerFactory.acceptUrl("www.youtube.com/feed/trending/music")); + assertTrue(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music")); + assertTrue(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music/?akdsakjf=dfije&kfj=dkjak")); + assertTrue(LinkHandlerFactory.acceptUrl("https://youtube.com/feed/trending/music")); + assertTrue(LinkHandlerFactory.acceptUrl("m.youtube.com/feed/trending/music")); + + assertTrue(LinkHandlerFactory.acceptUrl("https://www.invidio.us/feed/trending/music")); + assertTrue(LinkHandlerFactory.acceptUrl("https://invidio.us/feed/trending/music")); + assertTrue(LinkHandlerFactory.acceptUrl("invidio.us/feed/trending/music")); + + assertFalse(LinkHandlerFactory.acceptUrl("https://youtu.be/feed/trending/music")); + assertFalse(LinkHandlerFactory.acceptUrl("kdskjfiiejfia")); + assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/bullshit/feed/trending/music")); + assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music/bullshit")); + assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/bullshit/trending/music")); + assertFalse(LinkHandlerFactory.acceptUrl("peter klaut aepferl youtube.com/feed/trending/music")); + assertFalse(LinkHandlerFactory.acceptUrl("youtube.com/feed/trending/music askjkf")); + assertFalse(LinkHandlerFactory.acceptUrl("askdjfi youtube.com/feed/trending/music askjkf")); + assertFalse(LinkHandlerFactory.acceptUrl(" youtube.com/feed/trending/music")); + assertFalse(LinkHandlerFactory.acceptUrl("https://www.youtube.com/feed/trending/music.html")); + assertFalse(LinkHandlerFactory.acceptUrl("")); + } +}