feat: add chat export command

- fix(dependencies): pin numpy version to 1.26.4
- feat: add chat exporter
- fix(Dockerfile): prevent caching
- typo fix
- bug: fix UserResponse model
- Add *.pem to .gitignore
- Update README.md

.gitignore

@@ -3,3 +3,5 @@
 **/__pycache__
 **/venv
 **/.idea
+**/.vscode
+**/*.pem

.ssh/config

@@ -0,0 +1,7 @@
+Host github.com
+    User git
+    HostName github.com
+    Port 22
+    StrictHostKeyChecking no
+    UserKnownHostsFile=/dev/null
+    IdentityFile ~/.ssh/privkey.pem

Dockerfile

@@ -1,5 +1,14 @@
 FROM python:3.10
 
+RUN wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb \
+    -O packages-microsoft-prod.deb \
+    && dpkg -i packages-microsoft-prod.deb \
+    && rm packages-microsoft-prod.deb
+
+RUN apt-get update && apt-get install -y dotnet-sdk-8.0 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
 COPY requirements.txt .
 
 RUN pip3 install -r requirements.txt && \
@@ -13,6 +22,22 @@ WORKDIR /eruditus
 RUN useradd -m user && \
     chown -R user:user .
 
+COPY .ssh /home/user/.ssh/
+COPY chat_exporter /usr/bin/
+RUN chmod a+x /usr/bin/chat_exporter && \
+    chown -R user:user /home/user
+
 USER user
 
+# Prevent caching the subsequent "git clone" layer.
+# https://github.com/moby/moby/issues/1996#issuecomment-1152463036
+ADD http://date.jsontest.com /etc/builddate
+RUN git clone https://github.com/hfz1337/DiscordChatExporter ~/DiscordChatExporter
+
+ARG [email protected]:username/repo
+
+RUN git clone $CHATLOGS_REPO ~/chatlogs
+RUN git config --global user.email "eruditus@localhost" && \
+    git config --global user.name "eruditus"
+
 ENTRYPOINT ["python3", "-u", "eruditus.py"]

README.md

@@ -59,6 +59,7 @@ Here's a list of the currently supported commands:
 /ctf createctf                                       (Create a new CTF)
 /ctf renamectf                                       (Rename a CTF)
 /ctf archivectf                                      (Archive a CTF's channels)
+/ctf exportchat                                      (Export CTF chat logs to a static site)
 /ctf deletectf                                       (Delete a CTF as well as its channels)
 /ctf setprivacy                                      (Toggle CTF privacy between public and private)
 /ctf join                                            (Join a specific CTF channels)
@@ -99,6 +100,18 @@ Here's a list of the currently supported commands:
 
 ## Installation
 
+Before proceeding with the installation, you may want to setup a repository to host Discord chat logs for archived CTFs that you no longer want to have on your Discord guild. Follow these instructions if you wish to do so:
+1. Create a private GitHub repository to host your chat logs.
+2. Clone [this project](https://github.com/hfz1337/discord-oauth2-webapp) into your previously create GitHub repository.
+3. Modify line 37 of the [Dockerfile](./Dockerfile) to point to your private GitHub repository that will host the chat logs.
+4. Prepare an SSH key pair to access your private GitHub repository, and put the private key under [.ssh/privkey.pem](./.ssh). As for the public key, under your repository settings in the `Deploy keys` section, click on `Add deploy key` and paste your SSH public key (make sure to tick the `Allow write access` box).
+
+The [sample GitHub repository](https://github.com/hfz1337/discord-oauth2-webapp) already has a workflow for publishing the website to Azure App Service, but you're free to host it somewhere else, or simply keep it in the GitHub repository. If you're willing to use Azure, make sure to add the necessary secrets and variables referenced inside the [workflow](https://github.com/hfz1337/discord-oauth2-webapp/blob/main/.github/workflows/publish.yml).
+
+---
+
+Follow the instructions below to deploy the bot:
+
 1. Go to the [Discord Developer Portal](https://discord.com/developers/applications).
 2. Create a new application.
 3. Go to the **Bot** pane and add a bot for your application.

chat_exporter

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+if [[ $# -lt 2 ]]; then
+    echo "Usage: $0 <channel_file> <output_dirname>"
+    exit 1
+fi
+
+CHANNELS_FILE="$1"
+OUTPUT_DIR="/home/user/chatlogs/static/$2"
+
+# Ensure output directory exists
+mkdir -p "$OUTPUT_DIR"
+
+# Load the bot token
+export $(cat /eruditus/.env | grep 'DISCORD_TOKEN' | xargs)
+
+# Export the chat logs
+cd ~/DiscordChatExporter
+dotnet run --project DiscordChatExporter.Cli export \
+  --token "$DISCORD_TOKEN" \
+  --file "$CHANNELS_FILE" \
+  --output "$OUTPUT_DIR"
+
+# Commit & push
+cd ~/chatlogs
+git add . && git commit -m "Export operation ($(date))"
+git push origin main

docker-compose.yml

@@ -6,6 +6,9 @@ services:
     image: eruditus:latest
     container_name: eruditus
     restart: unless-stopped
+    environment:
+      - DOTNET_CLI_TELEMETRY_OPTOUT=1
+      - DOTNET_NOLOGO=true
     depends_on:
       - mongodb
   mongodb:

eruditus/app_commands/ctf/__init__.py

@@ -1,4 +1,7 @@
+import asyncio
 import logging
+import subprocess
+import tempfile
 from datetime import datetime
 from typing import Callable, Optional
 
@@ -1416,3 +1419,53 @@ async def register(
             return
 
         await interaction.response.send_modal(form)
+
+    @app_commands.checks.bot_has_permissions(manage_channels=True, manage_roles=True)
+    @app_commands.checks.has_permissions(manage_channels=True, manage_roles=True)
+    @app_commands.command()
+    @_in_ctf_channel()
+    async def exportchat(self, interaction: discord.Interaction) -> None:
+        """Export CTF chat logs to a static site.
+
+        Args:
+            interaction: The interaction that triggered this command.
+        """
+
+        async def _handle_process(process: asyncio.subprocess.Process):
+            _, _ = await process.communicate()
+            _log.info("Chat export task finished successfully.")
+
+        await interaction.response.defer()
+
+        guild_category = interaction.channel.category
+        exportable = set()
+        for channel in guild_category.text_channels:
+            exportable.add(channel.id)
+
+            for thread in channel.threads:
+                exportable.add(thread.id)
+
+            async for thread in channel.archived_threads(private=True, limit=None):
+                exportable.add(thread.id)
+
+        tmp = tempfile.mktemp()
+        output_dirname = (
+            f"[{guild_category.id}] {guild_category.name.replace('/', '_')}"
+        )
+        with open(tmp, "w", encoding="utf-8") as f:
+            f.write("\n".join(map(str, exportable)))
+
+        asyncio.create_task(
+            _handle_process(
+                await asyncio.create_subprocess_exec(
+                    "chat_exporter",
+                    tmp,
+                    output_dirname,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+            )
+        )
+        await interaction.followup.send(
+            "Export task started, chat logs will be available shortly.", ephemeral=True
+        )

eruditus/eruditus.py

@@ -777,7 +777,7 @@ async def challenge_puller(self) -> None:
     @tasks.loop(minutes=1, reconnect=True)
     async def scoreboard_updater(self) -> None:
         """Periodically update the scoreboard for all running CTFs."""
-        # Wait until the bot internal cache is ready.
+        # Wait until the bot's internal cache is ready.
         await self.wait_until_ready()
 
         # The bot is supposed to be part of a single guild.
@@ -788,7 +788,7 @@ async def scoreboard_updater(self) -> None:
 
     @tasks.loop(minutes=15, reconnect=True)
     async def ctftime_team_tracking(self) -> None:
-        # Wait until the bot internal cache is ready.
+        # Wait until the bot's internal cache is ready.
         await self.wait_until_ready()
 
         # Disable the feature if some of the related config vars are missing.
@@ -897,7 +897,7 @@ async def ctftime_team_tracking(self) -> None:
 
     @tasks.loop(minutes=15, reconnect=True)
     async def ctftime_leaderboard_tracking(self) -> None:
-        # Wait until the bot internal cache is ready.
+        # Wait until the bot's internal cache is ready.
         await self.wait_until_ready()
 
         # Disable the feature if some of the related config vars are missing.

eruditus/lib/validators/ctfd.py

@@ -161,18 +161,18 @@ class UserResponse(BaseValidResponse):
     """Response schema returned by `/api/v1/teams/me`."""
 
     class Data(BaseModel):
-        website: Optional[str]
+        website: Optional[str] = None
         id: int
         members: list[int]
-        oauth_id: Optional[Union[str, int]]
-        email: Optional[str]
-        country: Optional[str]
+        oauth_id: Optional[Union[str, int]] = None
+        email: Optional[str] = None
+        country: Optional[str] = None
         captain_id: int
         fields: list[dict]
-        affiliation: Optional[str]
-        bracket: Optional[Any]
+        affiliation: Optional[str] = None
+        bracket: Optional[Any] = None
         name: str
-        place: Optional[str]
+        place: Optional[str] = None
         score: int
 
         def convert(self) -> Team:

requirements.txt

@@ -1,4 +1,5 @@
 discord==2.2.0
+numpy==1.26.4
 pydantic==2.3
 markdownify==0.11.6
 matplotlib==3.6.3