From fa3440664526ab5be9f855b2ef499717466bb7fa Mon Sep 17 00:00:00 2001 From: Harmon Date: Fri, 28 Jul 2023 02:04:19 -0500 Subject: [PATCH] [Units] Return list of WikiArticle results for search_wiki function [Discord] Add and use utilities.views and WikiArticlesView to show and allow selection of multiple article results for commands that search wikis --- Discord/cogs/runescape.py | 26 ++-- Discord/cogs/search.py | 107 ++++++++++------- Discord/utilities/views.py | 62 ++++++++++ units/wikis.py | 239 +++++++++++++++++++++---------------- 4 files changed, 278 insertions(+), 156 deletions(-) create mode 100644 Discord/utilities/views.py diff --git a/Discord/cogs/runescape.py b/Discord/cogs/runescape.py index c0586eaaaa..e3491d1270 100644 --- a/Discord/cogs/runescape.py +++ b/Discord/cogs/runescape.py @@ -6,6 +6,7 @@ import sys from utilities import checks +from utilities.views import WikiArticlesView sys.path.insert(0, "..") from units.runescape import get_ge_data, get_item_id, get_monster_data @@ -162,21 +163,26 @@ async def wiki(self, ctx, *, query): """ await ctx.defer() try: - article = await search_wiki( + articles = await search_wiki( "https://runescape.wiki/", query, aiohttp_session = ctx.bot.aiohttp_session ) except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") - else: - await ctx.embed_reply( - title = article.title, - title_url = article.url, - description = article.extract, - image_url = article.image_url, - footer_icon_url = article.wiki.logo, - footer_text = article.wiki.name - ) + return + + view = WikiArticlesView(articles) + message = await ctx.reply( + "", + embed = view.initial_embed(ctx), + view = view + ) + + if ctx.interaction: + # Fetch Message, as InteractionMessage token expires after 15 min. + message = await message.fetch() + view.message = message + ctx.bot.views.append(view) @runescape.command(hidden = True, with_app_command = False) async def zybez(self, ctx): diff --git a/Discord/cogs/search.py b/Discord/cogs/search.py index a86019fba2..be225e2d23 100644 --- a/Discord/cogs/search.py +++ b/Discord/cogs/search.py @@ -13,6 +13,7 @@ from utilities import checks from utilities.menu_sources import WolframAlphaSource from utilities.paginators import ButtonPaginator +from utilities.views import WikiArticlesView sys.path.insert(0, "..") from units.wikis import search_wiki @@ -209,21 +210,26 @@ async def startpage(self, ctx, *search: str): async def uesp(self, ctx, *, search: str): """Look something up on the Unofficial Elder Scrolls Pages""" try: - article = await search_wiki( + articles = await search_wiki( "https://en.uesp.net/", search, aiohttp_session = ctx.bot.aiohttp_session ) except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") - else: - await ctx.embed_reply( - title = article.title, - title_url = article.url, - description = article.extract, - image_url = article.image_url, - footer_icon_url = article.wiki.logo, - footer_text = article.wiki.name - ) + return + + view = WikiArticlesView(articles) + message = await ctx.reply( + "", + embed = view.initial_embed(ctx), + view = view + ) + + if ctx.interaction: + # Fetch Message, as InteractionMessage token expires after 15 min. + message = await message.fetch() + view.message = message + ctx.bot.views.append(view) @uesp.command(name = "random") async def uesp_random(self, ctx): @@ -233,7 +239,7 @@ async def uesp_random(self, ctx): ''' # Note: random uesp command invokes this command try: - article = await search_wiki( + articles = await search_wiki( "https://en.uesp.net/", None, aiohttp_session = ctx.bot.aiohttp_session, random = True, @@ -244,6 +250,7 @@ async def uesp_random(self, ctx): except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") else: + article = articles[0] await ctx.embed_reply( title = article.title, title_url = article.url, @@ -260,21 +267,26 @@ async def uesp_random(self, ctx): async def wikipedia(self, ctx, *, query: str): """Search for an article on Wikipedia""" try: - article = await search_wiki( + articles = await search_wiki( "https://en.wikipedia.org/", query, aiohttp_session = ctx.bot.aiohttp_session ) except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") - else: - await ctx.embed_reply( - title = article.title, - title_url = article.url, - description = article.extract, - image_url = article.image_url, - footer_icon_url = article.wiki.logo, - footer_text = article.wiki.name - ) + return + + view = WikiArticlesView(articles) + message = await ctx.reply( + "", + embed = view.initial_embed(ctx), + view = view + ) + + if ctx.interaction: + # Fetch Message, as InteractionMessage token expires after 15 min. + message = await message.fetch() + view.message = message + ctx.bot.views.append(view) @wikipedia.command(name = "random") async def wikipedia_random(self, ctx): @@ -282,7 +294,7 @@ async def wikipedia_random(self, ctx): # Note: random wikipedia command invokes this command await ctx.defer() try: - article = await search_wiki( + articles = await search_wiki( "https://en.wikipedia.org/", None, aiohttp_session = ctx.bot.aiohttp_session, random = True @@ -290,6 +302,7 @@ async def wikipedia_random(self, ctx): except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") else: + article = articles[0] await ctx.embed_reply( title = article.title, title_url = article.url, @@ -325,41 +338,51 @@ async def fandom(self, ctx): async def lotr(self, ctx, *, query: str): """Search for an article on The Lord of The Rings Wiki""" try: - article = await search_wiki( + articles = await search_wiki( "https://lotr.fandom.com/", query, aiohttp_session = ctx.bot.aiohttp_session ) except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") - else: - await ctx.embed_reply( - title = article.title, - title_url = article.url, - description = article.extract, - image_url = article.image_url, - footer_icon_url = article.wiki.logo, - footer_text = article.wiki.name - ) + return + + view = WikiArticlesView(articles) + message = await ctx.reply( + "", + embed = view.initial_embed(ctx), + view = view + ) + + if ctx.interaction: + # Fetch Message, as InteractionMessage token expires after 15 min. + message = await message.fetch() + view.message = message + ctx.bot.views.append(view) @commands.command() async def tolkien(self, ctx, *, query: str): """Search for an article on Tolkien Gateway""" try: - article = await search_wiki( + articles = await search_wiki( "https://tolkiengateway.net/", query, aiohttp_session = ctx.bot.aiohttp_session ) except ValueError as e: await ctx.embed_reply(f"{ctx.bot.error_emoji} {e}") - else: - await ctx.embed_reply( - title = article.title, - title_url = article.url, - description = article.extract, - image_url = article.image_url, - footer_icon_url = article.wiki.logo, - footer_text = article.wiki.name - ) + return + + view = WikiArticlesView(articles) + message = await ctx.reply( + "", + embed = view.initial_embed(ctx), + view = view + ) + + if ctx.interaction: + # Fetch Message, as InteractionMessage token expires after 15 min. + message = await message.fetch() + view.message = message + ctx.bot.views.append(view) @commands.group( aliases = ["wa", "wolfram_alpha"], diff --git a/Discord/utilities/views.py b/Discord/utilities/views.py new file mode 100644 index 0000000000..d7dadb3be5 --- /dev/null +++ b/Discord/utilities/views.py @@ -0,0 +1,62 @@ + +import discord + + +class WikiArticlesView(discord.ui.View): + + def __init__(self, articles): + super().__init__(timeout = None) + # TODO: Timeout? + + self.articles = articles + + for number, article in enumerate(articles): + self.article.add_option(label = article.title, value = number) + + self.article.options[0].default = True + + def initial_embed(self, ctx): + article = self.articles[0] + return discord.Embed( + color = ctx.bot.bot_color, + title = article.title, + url = article.url, + description = article.extract + ).set_image( + url = article.image_url + ).set_footer( + icon_url = article.wiki.logo, + text = article.wiki.name + ) + + @discord.ui.select() + async def article(self, interaction, select): + for option in select.options: + option.default = False + + selected = int(select.values[0]) + article = self.articles[selected] + + embed = discord.Embed( + color = interaction.client.bot_color, + title = article.title, + url = article.url, + description = article.extract + ).set_image( + url = article.image_url + ).set_footer( + icon_url = article.wiki.logo, + text = article.wiki.name + ) + + select.options[selected].default = True + + await interaction.response.edit_message(embed = embed, view = self) + + async def stop(self): + self.article.disabled = True + + await self.message.edit(view = self) + + super().stop() + diff --git a/units/wikis.py b/units/wikis.py index 9fdc052c33..f5379e885a 100644 --- a/units/wikis.py +++ b/units/wikis.py @@ -85,7 +85,7 @@ async def search_wiki( # https://en.wikipedia.org/wiki/Wikipedia:Namespace # https://community.fandom.com/wiki/Help:Namespaces redirect: bool = True -) -> WikiArticle: +) -> list[WikiArticle]: # TODO: Add User-Agent # TODO: Use textwrap async with ensure_session(aiohttp_session) as aiohttp_session: @@ -105,21 +105,23 @@ async def search_wiki( ) as resp: # https://www.mediawiki.org/wiki/API:Random data = await resp.json() - search = data["query"]["random"][0]["title"] + titles = [data["query"]["random"][0]["title"]] else: async with aiohttp_session.get( api_url, params = { "action": "query", "list": "search", "srsearch": search, - "srinfo": "suggestion", "srlimit": 1, "format": "json" - } + "srinfo": "suggestion", "srlimit": 20, "format": "json" + } # max exlimit is 20 ) as resp: # https://www.mediawiki.org/wiki/API:Search data = await resp.json() - if search := data["query"]["search"]: - search = search[0]["title"] - elif not ( - search := data["query"].get("searchinfo", {}).get("suggestion") + if results := data["query"]["search"]: + titles = [result["title"] for result in results] + elif suggestion := data["query"].get("searchinfo", {}).get( + "suggestion" ): + titles = [suggestion] + else: raise ValueError("Page not found") async with aiohttp_session.get( @@ -127,7 +129,7 @@ async def search_wiki( # https://www.mediawiki.org/wiki/API:Query "action": "query", "prop": "info|extracts|pageimages|revisions", - "titles": search, + "titles": '|'.join(titles), "redirects": "", # https://www.mediawiki.org/wiki/API:Info "inprop": "url", @@ -148,117 +150,146 @@ async def search_wiki( ) as resp: data = await resp.json() + wiki_info_data = data["query"]["general"] + + logo = wiki_info_data["logo"] + if logo.startswith("//"): + logo = "https:" + logo + + wiki_info = WikiInfo( + name = wiki_info_data["sitename"], + logo = logo + ) + if "pages" not in data["query"]: raise ValueError("Error") # TODO: More descriptive error - page = list(data["query"]["pages"].values())[0] + articles = {} + invalid_pages = [] + + for page in data["query"]["pages"].values(): + if "missing" in page: + continue + if "invalid" in page: + invalid_pages.append(page) + continue + + title = page["title"] + + if "extract" in page: + extract = re.sub(r"\s+ \s+", ' ', page["extract"]) + else: + continue # TODO: Handle no extracts efficiently + # https://www.mediawiki.org/wiki/API:Parsing_wikitext + async with aiohttp_session.get( + api_url, params = { + "action": "parse", "page": search, "prop": "text", + "format": "json" + } + ) as resp: + data = await resp.json() + + p = BeautifulSoup( + data["parse"]["text"]['*'], "lxml" + ).body.div.find_all( + 'p', recursive = False + ) - if "missing" in page: - raise ValueError("Page not found") - if "invalid" in page: - raise ValueError(page["invalidreason"]) + first_p = p[0] + if first_p.aside: + first_p.aside.clear() + extract = first_p.get_text() - if redirect and "redirects" in data["query"]: - return await search_wiki( - url, data["query"]["redirects"][-1]["to"], - aiohttp_session = aiohttp_session, - redirect = False - ) - # TODO: Handle section links/tofragments + if len(p) > 1: + second_p = p[1] + extract += '\n' + second_p.get_text() - wiki_info_data = data["query"]["general"] + extract = re.sub(r"\n\s*\n", "\n\n", extract) - if "extract" in page: - extract = re.sub(r"\s+ \s+", ' ', page["extract"]) - else: - # https://www.mediawiki.org/wiki/API:Parsing_wikitext - async with aiohttp_session.get( - api_url, params = { - "action": "parse", "page": search, "prop": "text", - "format": "json" - } - ) as resp: - data = await resp.json() + extract = extract if len(extract) <= 512 else extract[:512] + '…' + # TODO: Update character limit?, Discord now uses 350 - p = BeautifulSoup( - data["parse"]["text"]['*'], "lxml" - ).body.div.find_all( - 'p', recursive = False - ) + article_path = wiki_info_data["articlepath"] + url = url.rstrip('/') + replacement_texts = {} - first_p = p[0] - if first_p.aside: - first_p.aside.clear() - extract = first_p.get_text() + # https://www.mediawiki.org/wiki/Help:Links + for link in re.finditer( + ( + r"\[\[([^\[\]]+?)\|([^\[\]]+?)\]\]" + r'|' + + r"\[\[([^\|]+?)\]\]" + r'|' + + r"(? 1: - second_p = p[1] - extract += '\n' + second_p.get_text() + # TODO: Handle bold (''' -> **) and italics ('' -> *) - extract = re.sub(r"\n\s*\n", "\n\n", extract) + if (thumbnail := page.get("thumbnail")): + thumbnail = thumbnail["source"].replace( + f"{thumbnail['width']}px", "1200px" + ) - extract = extract if len(extract) <= 512 else extract[:512] + '…' - # TODO: Update character limit?, Discord now uses 350 + articles[title] = WikiArticle( + title = title, + url = page["fullurl"], # TODO: Use canonicalurl? + extract = extract, + image_url = thumbnail, + wiki = wiki_info + ) - article_path = wiki_info_data["articlepath"] - url = url.rstrip('/') - replacement_texts = {} - - # https://www.mediawiki.org/wiki/Help:Links - for link in re.finditer( - ( - r"\[\[([^\[\]]+?)\|([^\[\]]+?)\]\]" + r'|' + - r"\[\[([^\|]+?)\]\]" + r'|' + - r"(? **) and italics ('' -> *) + else: + raise ValueError("Page not found") - if (thumbnail := page.get("thumbnail")): - thumbnail = thumbnail["source"].replace( - f"{thumbnail['width']}px", "1200px" + if redirect and "redirects" in data["query"]: + # TODO: Handle redirects + """ + return await search_wiki( + url, data["query"]["redirects"][-1]["to"], + aiohttp_session = aiohttp_session, + redirect = False ) + # TODO: Handle section links/tofragments + """ - logo = wiki_info_data["logo"] - if logo.startswith("//"): - logo = "https:" + logo - - wiki_info = WikiInfo( - name = wiki_info_data["sitename"], - logo = logo - ) + ordered_articles = [] + for title in titles: + try: + ordered_articles.append(articles[title]) + except KeyError: + pass # TODO: Handle? - return WikiArticle( - title = page["title"], - url = page["fullurl"], # TODO: Use canonicalurl? - extract = extract, - image_url = thumbnail, - wiki = wiki_info - ) + return ordered_articles