From 651c4a313ab41bf48f9f9c95b1a85c7216885615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Thu, 4 Jan 2024 18:30:57 +0100 Subject: [PATCH 1/2] refactor: read config options from environment variables --- .env.example | 38 ++++++++++----------- abrechnung/config.py | 31 +++++++++++++---- docker-compose.base.yaml | 2 -- docker-compose.devel.yaml | 32 ++++++++++------- docker/Dockerfile-api | 2 +- docker/Dockerfile-devel | 9 ----- docker/abrechnung.yaml | 22 ++++++++++++ docker/entrypoint.py | 66 +----------------------------------- docs/usage/configuration.rst | 15 ++++++++ docs/usage/installation.rst | 15 -------- pyproject.toml | 2 ++ tests/test_config.py | 30 ++++++++++++++++ 12 files changed, 134 insertions(+), 130 deletions(-) create mode 100644 docker/abrechnung.yaml create mode 100644 tests/test_config.py diff --git a/.env.example b/.env.example index fcf655f0..9db5eed5 100644 --- a/.env.example +++ b/.env.example @@ -1,26 +1,24 @@ -SERVICE_URL=http://localhost:8080 -SERVICE_API_URL=https://localhost:8080/api -SERVICE_NAME=Abrechnung -DB_HOST=abrechnung_postgres -DB_USER=abrechnung -DB_NAME=abrechnung -DB_PASSWORD=replaceme # e.g. pwgen -s 64 1 +ABRECHNUNG_SERVICE__URL=http://localhost:8080 +ABRECHNUNG_SERVICE__API_URL=https://localhost:8080/api +ABRECHNUNG_SERVICE__NAME=Abrechnung +ABRECHNUNG_DATABASE__HOST=abrechnung_postgres +ABRECHNUNG_DATABASE__USER=abrechnung +ABRECHNUNG_DATABASE__DBNAME=abrechnung +ABRECHNUNG_DATABASE__PASSWORD=replaceme # e.g. pwgen -s 64 1 -ABRECHNUNG_SECRET=replaceme # pwgen -s 64 1 -ABRECHNUNG_PORT=8080 -ABRECHNUNG_ID=default +ABRECHNUNG_API__SECRET_KEY=replaceme # pwgen -s 64 1 +ABRECHNUNG_API__PORT=8080 +ABRECHNUNG_API__ID=default -REGISTRATION_ENABLED=false -REGISTRATION_VALID_EMAIL_DOMAINS=sft.lol,sft.mx -REGISTRATION_ALLOW_GUEST_USERS=true +ABRECHNUNG_REGISTRATION__ENABLED=false -ABRECHNUNG_EMAIL=abrechnung@sft.lol -SMTP_HOST=mail -SMTP_PORT=1025 -SMTP_MODE=smtp -#SMTP_MODE=smtp-starttls # use this in production, remove line above -SMTP_USER=username -SMTP_PASSWORD=replaceme +ABRECHNUNG_EMAIL__ADDRESS=abrechnung@sft.lol +ABRECHNUNG_EMAIL__HOST=mail +ABRECHNUNG_EMAIL__PORT=1025 +ABRECHNUNG_EMAIL__MODE=smtp +#ABRECHNUNG_EMAIL__MODE=smtp-starttls # use this in production, remove line above +ABRECHNUNG_EMAIL__AUTH__USERNAME=username +ABRECHNUNG_EMAIL__AUTH__PASSWORD=replaceme POSTGRES_USER=abrechnung POSTGRES_PASSWORD=replaceme # use the same as DB_PASSWORD diff --git a/abrechnung/config.py b/abrechnung/config.py index ddbed1a7..679fe107 100644 --- a/abrechnung/config.py +++ b/abrechnung/config.py @@ -1,9 +1,14 @@ from datetime import timedelta from pathlib import Path -from typing import List, Optional +from typing import List, Literal, Optional, Tuple, Type import yaml -from pydantic import BaseModel +from pydantic import BaseModel, EmailStr +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) from abrechnung.framework.database import DatabaseConfig @@ -39,14 +44,16 @@ class AuthConfig(BaseModel): username: str password: str - address: str + address: EmailStr host: str port: int - mode: str = "smtp" # oneof "local" "smtp-ssl" "smtp-starttls" "smtp" + mode: Literal["local", "smtp-ssl", "smtp", "smtp-starttls"] = "smtp" auth: Optional[AuthConfig] = None -class Config(BaseModel): +class Config(BaseSettings): + model_config = SettingsConfigDict(env_prefix="ABRECHNUNG_", env_nested_delimiter="__") + service: ServiceConfig api: ApiConfig database: DatabaseConfig @@ -55,8 +62,20 @@ class Config(BaseModel): demo: DemoConfig = DemoConfig() registration: RegistrationConfig = RegistrationConfig() + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return env_settings, init_settings, dotenv_settings, file_secret_settings + def read_config(config_path: Path) -> Config: content = config_path.read_text("utf-8") - config = Config(**yaml.safe_load(content)) + loaded = yaml.safe_load(content) + config = Config(**loaded) return config diff --git a/docker-compose.base.yaml b/docker-compose.base.yaml index ba78070c..b6579e4e 100644 --- a/docker-compose.base.yaml +++ b/docker-compose.base.yaml @@ -18,5 +18,3 @@ services: command: cron healthcheck: disable: true - - diff --git a/docker-compose.devel.yaml b/docker-compose.devel.yaml index 7c97af59..e1a8ed10 100644 --- a/docker-compose.devel.yaml +++ b/docker-compose.devel.yaml @@ -2,13 +2,13 @@ services: api: build: context: . - dockerfile: docker/Dockerfile-devel - target: api + dockerfile: docker/Dockerfile-api extends: file: docker-compose.base.yaml service: api + env_file: .env environment: - DB_HOST: postgres + ABRECHNUNG_DATABASE__HOST: postgres depends_on: postgres: condition: service_healthy @@ -40,30 +40,38 @@ services: - "8080:80" mailer: + build: + context: . + dockerfile: docker/Dockerfile-api extends: file: docker-compose.base.yaml service: mailer + env_file: .env environment: - SMTP_HOST: mail - SMTP_PORT: 1025 - SMTP_MODE: smtp - DB_HOST: postgres - + ABRECHNUNG_DATABASE__HOST: postgres + ABRECHNUNG_EMAIL__HOST: mail + ABRECHNUNG_EMAIL__PORT: 1025 + ABRECHNUNG_EMAIL__MODE: smtp depends_on: api: condition: service_healthy links: - postgres - "mailhog:mail" + cron: + build: + context: . + dockerfile: docker/Dockerfile-api extends: file: docker-compose.base.yaml service: cron + env_file: .env environment: - SMTP_HOST: mail - SMTP_PORT: 1025 - SMTP_MODE: smtp - DB_HOST: postgres + ABRECHNUNG_DATABASE__HOST: postgres + ABRECHNUNG_EMAIL__HOST: mail + ABRECHNUNG_EMAIL__PORT: 1025 + ABRECHNUNG_EMAIL__MODE: smtp links: - postgres - "mailhog:mail" diff --git a/docker/Dockerfile-api b/docker/Dockerfile-api index 8a09fe39..747a7f21 100644 --- a/docker/Dockerfile-api +++ b/docker/Dockerfile-api @@ -8,7 +8,7 @@ RUN /opt/abrechnung-venv/bin/python3 -m pip install /src FROM python:3.10-alpine RUN addgroup -S abrechnung && adduser -S abrechnung -G abrechnung && apk add --no-cache curl COPY --from=builder /opt/abrechnung-venv/ /opt/abrechnung-venv/ -ADD --chmod=644 --chown=abrechnung:abrechnung config/abrechnung.yaml /etc/abrechnung/abrechnung.yaml +ADD --chmod=644 --chown=abrechnung:abrechnung docker/abrechnung.yaml /etc/abrechnung/abrechnung.yaml ADD --chmod=755 ./docker/entrypoint.py / COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung USER abrechnung diff --git a/docker/Dockerfile-devel b/docker/Dockerfile-devel index e15a9067..63fead8f 100644 --- a/docker/Dockerfile-devel +++ b/docker/Dockerfile-devel @@ -1,13 +1,4 @@ # syntax=docker/dockerfile:1.3 -FROM python:3.10-alpine as api -RUN addgroup -S abrechnung && adduser -S abrechnung -G abrechnung \ - && apk add --no-cache curl -ADD . /usr/share/abrechnung -RUN pip install --editable /usr/share/abrechnung -ADD --chmod=755 ./docker/entrypoint.py / -COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung -ENTRYPOINT ["/entrypoint.py"] - FROM node:lts as build ADD frontend/ /build/ RUN cd /build/ && npm install && npx nx build web diff --git a/docker/abrechnung.yaml b/docker/abrechnung.yaml new file mode 100644 index 00000000..f6fd8c2f --- /dev/null +++ b/docker/abrechnung.yaml @@ -0,0 +1,22 @@ +service: + url: "https://localhost" + api_url: "https://localhost/api" + name: "Abrechnung" + +database: + host: "0.0.0.0" + user: "abrechnung" + dbname: "abrechnung" + +api: + host: "0.0.0.0" + port: 8080 + id: default + +registration: + enabled: false + +email: + host: "localhost" + port: 587 + mode: "smtp-starttls" diff --git a/docker/entrypoint.py b/docker/entrypoint.py index 09c51898..83ef2954 100644 --- a/docker/entrypoint.py +++ b/docker/entrypoint.py @@ -1,73 +1,9 @@ import subprocess import sys -from os import execlp, execvp, getenv, makedirs - -from yaml import dump, safe_load - - -def to_bool(data: str): - return data.lower() in [ - "true", - "1", - "t", - "y", - "yes", - "on", - ] - +from os import execlp, execvp abrechnung_venv_python = "/opt/abrechnung-venv/bin/python3" -print("generating config") -config = {} -filename = "/etc/abrechnung/abrechnung.yaml" -with open(filename, "r", encoding="utf-8") as filehandle: - config = safe_load(filehandle) - -if not "service" in config: - config["service"] = {} -if not "database" in config: - config["database"] = {} -if not "registration" in config: - config["registration"] = {} -if not "email" in config: - config["email"] = {} - -config["service"]["url"] = getenv("SERVICE_URL", "https://localhost") -config["service"]["api_url"] = getenv("SERVICE_API_URL", "https://localhost/api") -config["service"]["name"] = getenv("SERVICE_NAME", "Abrechnung") -config["database"]["host"] = getenv("DB_HOST") -config["database"]["user"] = getenv("DB_USER") -config["database"]["dbname"] = getenv("DB_NAME") -config["database"]["password"] = getenv("DB_PASSWORD") - -config["api"]["secret_key"] = getenv("ABRECHNUNG_SECRET") -config["api"]["host"] = "0.0.0.0" -config["api"]["port"] = int(getenv("ABRECHNUNG_PORT", "8080")) -config["api"]["id"] = getenv("ABRECHNUNG_ID", "default") - -config["registration"]["allow_guest_users"] = to_bool(getenv("REGISTRATION_ALLOW_GUEST_USERS", "false")) -config["registration"]["enabled"] = to_bool(getenv("REGISTRATION_ENABLED", "false")) -config["registration"]["valid_email_domains"] = getenv("REGISTRATION_VALID_EMAIL_DOMAINS", "false").split(",") - -config["email"]["address"] = getenv("ABRECHNUNG_EMAIL", "") -config["email"]["host"] = getenv("SMTP_HOST", "localhost") -config["email"]["port"] = int(getenv("SMTP_PORT", "587")) -config["email"]["mode"] = getenv("SMTP_MODE", "smtp-starttls") - -if getenv("SMTP_USER", None): - if not "auth" in config["email"]: - config["email"]["auth"] = dict() - - config["email"]["auth"]["username"] = getenv("SMTP_USER", None) - config["email"]["auth"]["password"] = getenv("SMTP_PASSWORD", None) - -output = dump(config) -makedirs("/etc/abrechnung/", exist_ok=True) -with open("/etc/abrechnung/abrechnung.yaml", "w", encoding="utf-8") as file: - file.write(output) -print("config done") - if sys.argv[1] == "api": print("migrating ...") subprocess.run([abrechnung_venv_python, "-m", "abrechnung", "-vvv", "db", "migrate"], check=True, stdout=sys.stdout) diff --git a/docs/usage/configuration.rst b/docs/usage/configuration.rst index 25786e39..2bca155b 100644 --- a/docs/usage/configuration.rst +++ b/docs/usage/configuration.rst @@ -98,6 +98,21 @@ without authentication. .. _abrechnung-config-all-options: +Configuration via Environment Variables +---------------------- + +All of the configuration options set in the config yaml file can also be set via environment variables. +The respective environment variable name for a config variable is in the pattern ``ABRECHNUNG___``. + +E.g. to set the email auth username from the config yaml as below we'd use the environment variable ``ABRECHNUNG_EMAIL__AUTH__USERNAME``. + +.. code-block:: yaml + + email: + auth: + username: "..." + + Frontend Configuration ------------------------- diff --git a/docs/usage/installation.rst b/docs/usage/installation.rst index 33f0ddc9..0a474859 100644 --- a/docs/usage/installation.rst +++ b/docs/usage/installation.rst @@ -58,18 +58,3 @@ Then a simple simple :: docker-compose -f docker-compose.prod.yaml up Should suffice to get you up and running. - -Pip ---------------- - -TODO - -From Release Tarball --------------------- - -TODO - -From Source ---------------- - -TODO diff --git a/pyproject.toml b/pyproject.toml index ca8da42e..4dc0b0a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "typer~=0.9.0", "fastapi==0.108.0", "pydantic[email]~=2.4.0", + "pydantic-settings==2.1.0", "uvicorn[standard]~=0.23.0", "python-jose[cryptography]~=3.3.0", "asyncpg~=0.28.0", @@ -35,6 +36,7 @@ dependencies = [ test = [ "aiosmtpd~=1.4", "pytest", + "pytest-asyncio", "pytest-cov", "httpx~=0.23", ] diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..f3f96ccb --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,30 @@ +import os +import tempfile +from pathlib import Path + +from abrechnung.config import read_config + +docker_base_config = (Path(__file__).parent.parent / "docker" / "abrechnung.yaml").read_text() + + +def test_config_load_from_env(): + os.environ["ABRECHNUNG_SERVICE__NAME"] = "my abrechnung" + os.environ["ABRECHNUNG_API__SECRET_KEY"] = "secret" + os.environ["ABRECHNUNG_DATABASE__HOST"] = "localhost" + os.environ["ABRECHNUNG_DATABASE__DBNAME"] = "abrechnung" + os.environ["ABRECHNUNG_DATABASE__PASSWORD"] = "password" + os.environ["ABRECHNUNG_DATABASE__USER"] = "abrechnung" + os.environ["ABRECHNUNG_EMAIL__ADDRESS"] = "do-not-reply@test.com" + with tempfile.NamedTemporaryFile() as f: + filename = Path(f.name) + filename.write_text(docker_base_config, "utf-8") + + loaded_cfg = read_config(filename) + + assert loaded_cfg.service.name == "my abrechnung" + assert loaded_cfg.api.secret_key == "secret" + assert loaded_cfg.database.user == "abrechnung" + assert loaded_cfg.database.host == "localhost" + assert loaded_cfg.database.dbname == "abrechnung" + assert loaded_cfg.database.password == "password" + assert loaded_cfg.email.address == "do-not-reply@test.com" From 961eaba0194d218ea053de2dac0f910bb87fce22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Thu, 4 Jan 2024 18:52:49 +0100 Subject: [PATCH 2/2] feat: add config option to disable enamil confirmation --- abrechnung/application/users.py | 5 ++++- abrechnung/config.py | 1 + docs/usage/configuration.rst | 36 +++++++++++++++++++++++++++++---- tests/test_auth.py | 14 +++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/abrechnung/application/users.py b/abrechnung/application/users.py index 62482986..75ac0b6c 100644 --- a/abrechnung/application/users.py +++ b/abrechnung/application/users.py @@ -53,6 +53,7 @@ def __init__( super().__init__(db_pool=db_pool, config=config) self.enable_registration = self.cfg.registration.enabled + self.require_email_confirmation = self.cfg.registration.require_email_confirmation self.allow_guest_users = self.cfg.registration.allow_guest_users self.valid_email_domains = self.cfg.registration.valid_email_domains @@ -172,7 +173,7 @@ async def demo_register_user(self, *, conn: Connection, username: str, email: st def _validate_email_address(email: str) -> str: try: valid = validate_email(email) - email = valid.email + email = valid.normalized except EmailNotValidError as e: raise InvalidCommand(str(e)) @@ -203,6 +204,8 @@ async def register_user( if not self.enable_registration: raise PermissionError(f"User registrations are disabled on this server") + requires_email_confirmation = self.require_email_confirmation and requires_email_confirmation + await _check_user_exists(conn=conn, username=username, email=email) email = self._validate_email_address(email) diff --git a/abrechnung/config.py b/abrechnung/config.py index 679fe107..9834637c 100644 --- a/abrechnung/config.py +++ b/abrechnung/config.py @@ -37,6 +37,7 @@ class RegistrationConfig(BaseModel): enabled: bool = False allow_guest_users: bool = False valid_email_domains: Optional[List[str]] = None + require_email_confirmation: bool = True class EmailConfig(BaseModel): diff --git a/docs/usage/configuration.rst b/docs/usage/configuration.rst index 2bca155b..f1f1b91f 100644 --- a/docs/usage/configuration.rst +++ b/docs/usage/configuration.rst @@ -67,8 +67,7 @@ The config will then look like port: 8080 id: default -In most cases there is no need to adjust either the ``host``, ``port`` or ``id`` options. For an overview of all -possible options see :ref:`abrechnung-config-all-options`. +In most cases there is no need to adjust either the ``host``, ``port`` or ``id`` options. E-Mail Delivery --------------- @@ -96,10 +95,39 @@ Currently supported ``mode`` options are The ``auth`` section is optional, if omitted the mail delivery daemon will try to connect to the mail server without authentication. -.. _abrechnung-config-all-options: +User Registration +----------------- + +This section allows to configure how users can register at the abrechnung instance. +By default open registration is disabled. + +When enabling registration without any additional settings any user will be able to create an account and use it after +a successful email confirmation. + +E-mail confirmation can be turned of by setting the respective config variable to ``false``. + +.. code-block:: yaml + + registration: + enabled: true + require_email_confirmation: true + +Additionally open registration can be restricted adding domains to the ``valid_email_domains`` config variable. +This will restrict account creation to users who possess an email from one of the configured domains. +To still allow outside users to take part the ``allow_guest_users`` flag can be set which enables users to create a +"guest" account when in possession of a valid group invite link. +Guest users will not be able to create new groups themselves but can take part in groups they are invited to normally. + +.. code-block:: yaml + + registration: + enabled: true + require_email_confirmation: true + valid_email_domains: ["some-domain.com"] + allow_guest_users: true Configuration via Environment Variables ----------------------- +--------------------------------------- All of the configuration options set in the config yaml file can also be set via environment variables. The respective environment variable name for a config variable is in the pattern ``ABRECHNUNG___``. diff --git a/tests/test_auth.py b/tests/test_auth.py index 84160797..78382e90 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -67,3 +67,17 @@ async def test_register_guest_user(self): email="invalid-something@something.com", password="asdf1234", ) + + async def test_register_without_email_confirmation(self): + config = TEST_CONFIG.model_copy(deep=True) + config.registration.require_email_confirmation = False + user_service = UserService(self.db_pool, config=config) + + user_id = await user_service.register_user( + username="guest user 1", + email="foobar@something.com", + password="asdf1234", + ) + self.assertIsNotNone(user_id) + user = await user_service.get_user(user_id=user_id) + self.assertFalse(user.pending)