From 4598e1ba509adec830d39d5d5a34541020092bed Mon Sep 17 00:00:00 2001 From: hfz Date: Sat, 22 Jun 2024 19:56:49 -0400 Subject: [PATCH] feat: add chat export command - fix(dependencies): pin numpy version to 1.26.4 - feat: add chat exporter - fix(Dockerfile): prevent caching - typo fix - bug: fix UserResponse model - Add *.pem to .gitignore - Update README.md --- .gitignore | 2 + .ssh/config | 7 ++++ Dockerfile | 25 +++++++++++++ README.md | 13 +++++++ chat_exporter | 28 ++++++++++++++ docker-compose.yml | 3 ++ eruditus/app_commands/ctf/__init__.py | 53 +++++++++++++++++++++++++++ eruditus/eruditus.py | 6 +-- eruditus/lib/validators/ctfd.py | 14 +++---- requirements.txt | 1 + 10 files changed, 142 insertions(+), 10 deletions(-) create mode 100644 .ssh/config create mode 100755 chat_exporter diff --git a/.gitignore b/.gitignore index 935db6a..af81b39 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ **/__pycache__ **/venv **/.idea +**/.vscode +**/*.pem diff --git a/.ssh/config b/.ssh/config new file mode 100644 index 0000000..9607a15 --- /dev/null +++ b/.ssh/config @@ -0,0 +1,7 @@ +Host github.com + User git + HostName github.com + Port 22 + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + IdentityFile ~/.ssh/privkey.pem diff --git a/Dockerfile b/Dockerfile index 922678b..e87f00e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,14 @@ FROM python:3.10 +RUN wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb \ + -O packages-microsoft-prod.deb \ + && dpkg -i packages-microsoft-prod.deb \ + && rm packages-microsoft-prod.deb + +RUN apt-get update && apt-get install -y dotnet-sdk-8.0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + COPY requirements.txt . RUN pip3 install -r requirements.txt && \ @@ -13,6 +22,22 @@ WORKDIR /eruditus RUN useradd -m user && \ chown -R user:user . +COPY .ssh /home/user/.ssh/ +COPY chat_exporter /usr/bin/ +RUN chmod a+x /usr/bin/chat_exporter && \ + chown -R user:user /home/user + USER user +# Prevent caching the subsequent "git clone" layer. +# https://github.com/moby/moby/issues/1996#issuecomment-1152463036 +ADD http://date.jsontest.com /etc/builddate +RUN git clone https://github.com/hfz1337/DiscordChatExporter ~/DiscordChatExporter + +ARG CHATLOGS_REPO=git@github.com:username/repo + +RUN git clone $CHATLOGS_REPO ~/chatlogs +RUN git config --global user.email "eruditus@localhost" && \ + git config --global user.name "eruditus" + ENTRYPOINT ["python3", "-u", "eruditus.py"] diff --git a/README.md b/README.md index 2837f4d..3903828 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Here's a list of the currently supported commands: /ctf createctf (Create a new CTF) /ctf renamectf (Rename a CTF) /ctf archivectf (Archive a CTF's channels) +/ctf exportchat (Export CTF chat logs to a static site) /ctf deletectf (Delete a CTF as well as its channels) /ctf setprivacy (Toggle CTF privacy between public and private) /ctf join (Join a specific CTF channels) @@ -99,6 +100,18 @@ Here's a list of the currently supported commands: ## Installation +Before proceeding with the installation, you may want to setup a repository to host Discord chat logs for archived CTFs that you no longer want to have on your Discord guild. Follow these instructions if you wish to do so: +1. Create a private GitHub repository to host your chat logs. +2. Clone [this project](https://github.com/hfz1337/discord-oauth2-webapp) into your previously create GitHub repository. +3. Modify line 37 of the [Dockerfile](./Dockerfile) to point to your private GitHub repository that will host the chat logs. +4. Prepare an SSH key pair to access your private GitHub repository, and put the private key under [.ssh/privkey.pem](./.ssh). As for the public key, under your repository settings in the `Deploy keys` section, click on `Add deploy key` and paste your SSH public key (make sure to tick the `Allow write access` box). + +The [sample GitHub repository](https://github.com/hfz1337/discord-oauth2-webapp) already has a workflow for publishing the website to Azure App Service, but you're free to host it somewhere else, or simply keep it in the GitHub repository. If you're willing to use Azure, make sure to add the necessary secrets and variables referenced inside the [workflow](https://github.com/hfz1337/discord-oauth2-webapp/blob/main/.github/workflows/publish.yml). + +--- + +Follow the instructions below to deploy the bot: + 1. Go to the [Discord Developer Portal](https://discord.com/developers/applications). 2. Create a new application. 3. Go to the **Bot** pane and add a bot for your application. diff --git a/chat_exporter b/chat_exporter new file mode 100755 index 0000000..15e9b5b --- /dev/null +++ b/chat_exporter @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 " + exit 1 +fi + +CHANNELS_FILE="$1" +OUTPUT_DIR="/home/user/chatlogs/static/$2" + +# Ensure output directory exists +mkdir -p "$OUTPUT_DIR" + +# Load the bot token +export $(cat /eruditus/.env | grep 'DISCORD_TOKEN' | xargs) + +# Export the chat logs +cd ~/DiscordChatExporter +dotnet run --project DiscordChatExporter.Cli export \ + --token "$DISCORD_TOKEN" \ + --file "$CHANNELS_FILE" \ + --output "$OUTPUT_DIR" + +# Commit & push +cd ~/chatlogs +git add . && git commit -m "Export operation ($(date))" +git push origin main \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ada5428..5d4c9aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,9 @@ services: image: eruditus:latest container_name: eruditus restart: unless-stopped + environment: + - DOTNET_CLI_TELEMETRY_OPTOUT=1 + - DOTNET_NOLOGO=true depends_on: - mongodb mongodb: diff --git a/eruditus/app_commands/ctf/__init__.py b/eruditus/app_commands/ctf/__init__.py index 420c449..3633be7 100644 --- a/eruditus/app_commands/ctf/__init__.py +++ b/eruditus/app_commands/ctf/__init__.py @@ -1,4 +1,7 @@ +import asyncio import logging +import subprocess +import tempfile from datetime import datetime from typing import Callable, Optional @@ -1416,3 +1419,53 @@ async def register( return await interaction.response.send_modal(form) + + @app_commands.checks.bot_has_permissions(manage_channels=True, manage_roles=True) + @app_commands.checks.has_permissions(manage_channels=True, manage_roles=True) + @app_commands.command() + @_in_ctf_channel() + async def exportchat(self, interaction: discord.Interaction) -> None: + """Export CTF chat logs to a static site. + + Args: + interaction: The interaction that triggered this command. + """ + + async def _handle_process(process: asyncio.subprocess.Process): + _, _ = await process.communicate() + _log.info("Chat export task finished successfully.") + + await interaction.response.defer() + + guild_category = interaction.channel.category + exportable = set() + for channel in guild_category.text_channels: + exportable.add(channel.id) + + for thread in channel.threads: + exportable.add(thread.id) + + async for thread in channel.archived_threads(private=True, limit=None): + exportable.add(thread.id) + + tmp = tempfile.mktemp() + output_dirname = ( + f"[{guild_category.id}] {guild_category.name.replace('/', '_')}" + ) + with open(tmp, "w", encoding="utf-8") as f: + f.write("\n".join(map(str, exportable))) + + asyncio.create_task( + _handle_process( + await asyncio.create_subprocess_exec( + "chat_exporter", + tmp, + output_dirname, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + ) + ) + await interaction.followup.send( + "Export task started, chat logs will be available shortly.", ephemeral=True + ) diff --git a/eruditus/eruditus.py b/eruditus/eruditus.py index 5d6ab4f..bb35dc7 100644 --- a/eruditus/eruditus.py +++ b/eruditus/eruditus.py @@ -777,7 +777,7 @@ async def challenge_puller(self) -> None: @tasks.loop(minutes=1, reconnect=True) async def scoreboard_updater(self) -> None: """Periodically update the scoreboard for all running CTFs.""" - # Wait until the bot internal cache is ready. + # Wait until the bot's internal cache is ready. await self.wait_until_ready() # The bot is supposed to be part of a single guild. @@ -788,7 +788,7 @@ async def scoreboard_updater(self) -> None: @tasks.loop(minutes=15, reconnect=True) async def ctftime_team_tracking(self) -> None: - # Wait until the bot internal cache is ready. + # Wait until the bot's internal cache is ready. await self.wait_until_ready() # Disable the feature if some of the related config vars are missing. @@ -897,7 +897,7 @@ async def ctftime_team_tracking(self) -> None: @tasks.loop(minutes=15, reconnect=True) async def ctftime_leaderboard_tracking(self) -> None: - # Wait until the bot internal cache is ready. + # Wait until the bot's internal cache is ready. await self.wait_until_ready() # Disable the feature if some of the related config vars are missing. diff --git a/eruditus/lib/validators/ctfd.py b/eruditus/lib/validators/ctfd.py index b44d96c..cfc6c69 100644 --- a/eruditus/lib/validators/ctfd.py +++ b/eruditus/lib/validators/ctfd.py @@ -161,18 +161,18 @@ class UserResponse(BaseValidResponse): """Response schema returned by `/api/v1/teams/me`.""" class Data(BaseModel): - website: Optional[str] + website: Optional[str] = None id: int members: list[int] - oauth_id: Optional[Union[str, int]] - email: Optional[str] - country: Optional[str] + oauth_id: Optional[Union[str, int]] = None + email: Optional[str] = None + country: Optional[str] = None captain_id: int fields: list[dict] - affiliation: Optional[str] - bracket: Optional[Any] + affiliation: Optional[str] = None + bracket: Optional[Any] = None name: str - place: Optional[str] + place: Optional[str] = None score: int def convert(self) -> Team: diff --git a/requirements.txt b/requirements.txt index e246750..f0c2630 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ discord==2.2.0 +numpy==1.26.4 pydantic==2.3 markdownify==0.11.6 matplotlib==3.6.3