Merge pull request #193 from SFTtech/milo/config-env-loading

refactor: read config options from environment variables

.env.example

@@ -1,26 +1,24 @@
-SERVICE_URL=http://localhost:8080
-SERVICE_API_URL=https://localhost:8080/api
-SERVICE_NAME=Abrechnung
-DB_HOST=abrechnung_postgres
-DB_USER=abrechnung
-DB_NAME=abrechnung
-DB_PASSWORD=replaceme # e.g. pwgen -s 64 1
+ABRECHNUNG_SERVICE__URL=http://localhost:8080
+ABRECHNUNG_SERVICE__API_URL=https://localhost:8080/api
+ABRECHNUNG_SERVICE__NAME=Abrechnung
+ABRECHNUNG_DATABASE__HOST=abrechnung_postgres
+ABRECHNUNG_DATABASE__USER=abrechnung
+ABRECHNUNG_DATABASE__DBNAME=abrechnung
+ABRECHNUNG_DATABASE__PASSWORD=replaceme # e.g. pwgen -s 64 1
 
-ABRECHNUNG_SECRET=replaceme # pwgen -s 64 1
-ABRECHNUNG_PORT=8080
-ABRECHNUNG_ID=default
+ABRECHNUNG_API__SECRET_KEY=replaceme # pwgen -s 64 1
+ABRECHNUNG_API__PORT=8080
+ABRECHNUNG_API__ID=default
 
-REGISTRATION_ENABLED=false
-REGISTRATION_VALID_EMAIL_DOMAINS=sft.lol,sft.mx
-REGISTRATION_ALLOW_GUEST_USERS=true
+ABRECHNUNG_REGISTRATION__ENABLED=false
 
-ABRECHNUNG_EMAIL=[email protected]
-SMTP_HOST=mail
-SMTP_PORT=1025
-SMTP_MODE=smtp
-#SMTP_MODE=smtp-starttls # use this in production, remove line above
-SMTP_USER=username
-SMTP_PASSWORD=replaceme
+ABRECHNUNG_EMAIL__ADDRESS=[email protected]
+ABRECHNUNG_EMAIL__HOST=mail
+ABRECHNUNG_EMAIL__PORT=1025
+ABRECHNUNG_EMAIL__MODE=smtp
+#ABRECHNUNG_EMAIL__MODE=smtp-starttls # use this in production, remove line above
+ABRECHNUNG_EMAIL__AUTH__USERNAME=username
+ABRECHNUNG_EMAIL__AUTH__PASSWORD=replaceme
 
 POSTGRES_USER=abrechnung
 POSTGRES_PASSWORD=replaceme # use the same as DB_PASSWORD

abrechnung/application/users.py

@@ -53,6 +53,7 @@ def __init__(
         super().__init__(db_pool=db_pool, config=config)
 
         self.enable_registration = self.cfg.registration.enabled
+        self.require_email_confirmation = self.cfg.registration.require_email_confirmation
         self.allow_guest_users = self.cfg.registration.allow_guest_users
         self.valid_email_domains = self.cfg.registration.valid_email_domains
 
@@ -172,7 +173,7 @@ async def demo_register_user(self, *, conn: Connection, username: str, email: st
     def _validate_email_address(email: str) -> str:
         try:
             valid = validate_email(email)
-            email = valid.email
+            email = valid.normalized
         except EmailNotValidError as e:
             raise InvalidCommand(str(e))
 
@@ -203,6 +204,8 @@ async def register_user(
         if not self.enable_registration:
             raise PermissionError(f"User registrations are disabled on this server")
 
+        requires_email_confirmation = self.require_email_confirmation and requires_email_confirmation
+
         await _check_user_exists(conn=conn, username=username, email=email)
 
         email = self._validate_email_address(email)

abrechnung/config.py

@@ -1,9 +1,14 @@
 from datetime import timedelta
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Literal, Optional, Tuple, Type
 
 import yaml
-from pydantic import BaseModel
+from pydantic import BaseModel, EmailStr
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+)
 
 from abrechnung.framework.database import DatabaseConfig
 
@@ -32,21 +37,24 @@ class RegistrationConfig(BaseModel):
     enabled: bool = False
     allow_guest_users: bool = False
     valid_email_domains: Optional[List[str]] = None
+    require_email_confirmation: bool = True
 
 
 class EmailConfig(BaseModel):
     class AuthConfig(BaseModel):
         username: str
         password: str
 
-    address: str
+    address: EmailStr
     host: str
     port: int
-    mode: str = "smtp"  # oneof "local" "smtp-ssl" "smtp-starttls" "smtp"
+    mode: Literal["local", "smtp-ssl", "smtp", "smtp-starttls"] = "smtp"
     auth: Optional[AuthConfig] = None
 
 
-class Config(BaseModel):
+class Config(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="ABRECHNUNG_", env_nested_delimiter="__")
+
     service: ServiceConfig
     api: ApiConfig
     database: DatabaseConfig
@@ -55,8 +63,20 @@ class Config(BaseModel):
     demo: DemoConfig = DemoConfig()
     registration: RegistrationConfig = RegistrationConfig()
 
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        return env_settings, init_settings, dotenv_settings, file_secret_settings
+
 
 def read_config(config_path: Path) -> Config:
     content = config_path.read_text("utf-8")
-    config = Config(**yaml.safe_load(content))
+    loaded = yaml.safe_load(content)
+    config = Config(**loaded)
     return config

docker-compose.base.yaml

@@ -18,5 +18,3 @@ services:
     command: cron
     healthcheck:
       disable: true
-
-

docker-compose.devel.yaml

@@ -2,13 +2,13 @@ services:
   api:
     build:
       context: .
-      dockerfile: docker/Dockerfile-devel
-      target: api
+      dockerfile: docker/Dockerfile-api
     extends:
       file: docker-compose.base.yaml
       service: api
+    env_file: .env
     environment:
-      DB_HOST: postgres
+      ABRECHNUNG_DATABASE__HOST: postgres
     depends_on:
       postgres:
         condition: service_healthy
@@ -40,30 +40,38 @@ services:
       - "8080:80"
 
   mailer:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile-api
     extends:
       file: docker-compose.base.yaml
       service: mailer
+    env_file: .env
     environment:
-      SMTP_HOST: mail
-      SMTP_PORT: 1025
-      SMTP_MODE: smtp
-      DB_HOST: postgres
-
+      ABRECHNUNG_DATABASE__HOST: postgres
+      ABRECHNUNG_EMAIL__HOST: mail
+      ABRECHNUNG_EMAIL__PORT: 1025
+      ABRECHNUNG_EMAIL__MODE: smtp
     depends_on:
       api:
         condition: service_healthy
     links:
       - postgres
       - "mailhog:mail"
+
   cron:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile-api
     extends:
       file: docker-compose.base.yaml
       service: cron
+    env_file: .env
     environment:
-      SMTP_HOST: mail
-      SMTP_PORT: 1025
-      SMTP_MODE: smtp
-      DB_HOST: postgres
+      ABRECHNUNG_DATABASE__HOST: postgres
+      ABRECHNUNG_EMAIL__HOST: mail
+      ABRECHNUNG_EMAIL__PORT: 1025
+      ABRECHNUNG_EMAIL__MODE: smtp
     links:
       - postgres
       - "mailhog:mail"

docker/Dockerfile-api

@@ -8,7 +8,7 @@ RUN /opt/abrechnung-venv/bin/python3 -m pip install /src
 FROM python:3.10-alpine
 RUN addgroup -S abrechnung && adduser -S abrechnung -G abrechnung && apk add --no-cache curl
 COPY --from=builder /opt/abrechnung-venv/ /opt/abrechnung-venv/
-ADD --chmod=644 --chown=abrechnung:abrechnung config/abrechnung.yaml /etc/abrechnung/abrechnung.yaml
+ADD --chmod=644 --chown=abrechnung:abrechnung docker/abrechnung.yaml /etc/abrechnung/abrechnung.yaml
 ADD --chmod=755 ./docker/entrypoint.py /
 COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung
 USER abrechnung

docker/Dockerfile-devel

@@ -1,13 +1,4 @@
 # syntax=docker/dockerfile:1.3
-FROM python:3.10-alpine as api
-RUN addgroup -S abrechnung && adduser -S abrechnung -G abrechnung \
-  && apk add --no-cache curl
-ADD . /usr/share/abrechnung
-RUN pip install --editable /usr/share/abrechnung
-ADD --chmod=755 ./docker/entrypoint.py /
-COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung
-ENTRYPOINT ["/entrypoint.py"]
-
 FROM node:lts as build
 ADD frontend/ /build/
 RUN cd /build/ && npm install && npx nx build web 

docker/abrechnung.yaml

@@ -0,0 +1,22 @@
+service:
+  url: "https://localhost"
+  api_url: "https://localhost/api"
+  name: "Abrechnung"
+
+database:
+  host: "0.0.0.0"
+  user: "abrechnung"
+  dbname: "abrechnung"
+
+api:
+  host: "0.0.0.0"
+  port: 8080
+  id: default
+
+registration:
+  enabled: false
+
+email:
+  host: "localhost"
+  port: 587
+  mode: "smtp-starttls"

docker/entrypoint.py

@@ -1,73 +1,9 @@
 import subprocess
 import sys
-from os import execlp, execvp, getenv, makedirs
-
-from yaml import dump, safe_load
-
-
-def to_bool(data: str):
-    return data.lower() in [
-        "true",
-        "1",
-        "t",
-        "y",
-        "yes",
-        "on",
-    ]
-
+from os import execlp, execvp
 
 abrechnung_venv_python = "/opt/abrechnung-venv/bin/python3"
 
-print("generating config")
-config = {}
-filename = "/etc/abrechnung/abrechnung.yaml"
-with open(filename, "r", encoding="utf-8") as filehandle:
-    config = safe_load(filehandle)
-
-if not "service" in config:
-    config["service"] = {}
-if not "database" in config:
-    config["database"] = {}
-if not "registration" in config:
-    config["registration"] = {}
-if not "email" in config:
-    config["email"] = {}
-
-config["service"]["url"] = getenv("SERVICE_URL", "https://localhost")
-config["service"]["api_url"] = getenv("SERVICE_API_URL", "https://localhost/api")
-config["service"]["name"] = getenv("SERVICE_NAME", "Abrechnung")
-config["database"]["host"] = getenv("DB_HOST")
-config["database"]["user"] = getenv("DB_USER")
-config["database"]["dbname"] = getenv("DB_NAME")
-config["database"]["password"] = getenv("DB_PASSWORD")
-
-config["api"]["secret_key"] = getenv("ABRECHNUNG_SECRET")
-config["api"]["host"] = "0.0.0.0"
-config["api"]["port"] = int(getenv("ABRECHNUNG_PORT", "8080"))
-config["api"]["id"] = getenv("ABRECHNUNG_ID", "default")
-
-config["registration"]["allow_guest_users"] = to_bool(getenv("REGISTRATION_ALLOW_GUEST_USERS", "false"))
-config["registration"]["enabled"] = to_bool(getenv("REGISTRATION_ENABLED", "false"))
-config["registration"]["valid_email_domains"] = getenv("REGISTRATION_VALID_EMAIL_DOMAINS", "false").split(",")
-
-config["email"]["address"] = getenv("ABRECHNUNG_EMAIL", "")
-config["email"]["host"] = getenv("SMTP_HOST", "localhost")
-config["email"]["port"] = int(getenv("SMTP_PORT", "587"))
-config["email"]["mode"] = getenv("SMTP_MODE", "smtp-starttls")
-
-if getenv("SMTP_USER", None):
-    if not "auth" in config["email"]:
-        config["email"]["auth"] = dict()
-
-    config["email"]["auth"]["username"] = getenv("SMTP_USER", None)
-    config["email"]["auth"]["password"] = getenv("SMTP_PASSWORD", None)
-
-output = dump(config)
-makedirs("/etc/abrechnung/", exist_ok=True)
-with open("/etc/abrechnung/abrechnung.yaml", "w", encoding="utf-8") as file:
-    file.write(output)
-print("config done")
-
 if sys.argv[1] == "api":
     print("migrating ...")
     subprocess.run([abrechnung_venv_python, "-m", "abrechnung", "-vvv", "db", "migrate"], check=True, stdout=sys.stdout)

docs/usage/configuration.rst

@@ -67,8 +67,7 @@ The config will then look like
     port: 8080
     id: default
 
-In most cases there is no need to adjust either the ``host``, ``port`` or ``id`` options. For an overview of all
-possible options see :ref:`abrechnung-config-all-options`.
+In most cases there is no need to adjust either the ``host``, ``port`` or ``id`` options.
 
 E-Mail Delivery
 ---------------
@@ -96,7 +95,51 @@ Currently supported ``mode`` options are
 The ``auth`` section is optional, if omitted the mail delivery daemon will try to connect to the mail server
 without authentication.
 
-.. _abrechnung-config-all-options:
+User Registration
+-----------------
+
+This section allows to configure how users can register at the abrechnung instance.
+By default open registration is disabled.
+
+When enabling registration without any additional settings any user will be able to create an account and use it after
+a successful email confirmation.
+
+E-mail confirmation can be turned of by setting the respective config variable to ``false``.
+
+.. code-block:: yaml
+
+  registration:
+    enabled: true
+    require_email_confirmation: true
+
+Additionally open registration can be restricted adding domains to the ``valid_email_domains`` config variable.
+This will restrict account creation to users who possess an email from one of the configured domains.
+To still allow outside users to take part the ``allow_guest_users`` flag can be set which enables users to create a
+"guest" account when in possession of a valid group invite link.
+Guest users will not be able to create new groups themselves but can take part in groups they are invited to normally.
+
+.. code-block:: yaml
+
+  registration:
+    enabled: true
+    require_email_confirmation: true
+    valid_email_domains: ["some-domain.com"]
+    allow_guest_users: true
+
+Configuration via Environment Variables
+---------------------------------------
+
+All of the configuration options set in the config yaml file can also be set via environment variables.
+The respective environment variable name for a config variable is in the pattern ``ABRECHNUNG_<config section>__<variable name in capslock>``.
+
+E.g. to set the email auth username from the config yaml as below we'd use the environment variable ``ABRECHNUNG_EMAIL__AUTH__USERNAME``.
+
+.. code-block:: yaml
+
+  email:
+    auth:
+      username: "..."
+
 
 Frontend Configuration
 -------------------------