Postgres Full Text search.

.github/workflows/tests.yml

@@ -27,6 +27,9 @@ jobs:
           libpq-dev \
           make  \
           postgresql \
+          libpoppler-cpp-dev  \
+          pkg-config  \
+          python3-dev \
           --fix-missing
 
     - name: Checkout repo
@@ -75,6 +78,9 @@ jobs:
           libpq-dev \
           make  \
           postgresql \
+          libpoppler-cpp-dev  \
+          pkg-config  \
+          python3-dev \
           --fix-missing
 
         export LC_ALL="C.UTF-8"

README.md

@@ -5,6 +5,19 @@ Getting Started
 ---------------
 
 ```
+sudo apt install postgresql libpq-dev python3-dev build-essential weasyprint
+```
+
+Create the PostgreSQL database:
+```
+sudo -u postgres bash -c "psql <<EOF
+CREATE USER dev WITH PASSWORD 'postgres' LOGIN NOINHERIT;
+ALTER USER dev WITH SUPERUSER;
+CREATE DATABASE privatim;
+GRANT ALL PRIVILEGES ON DATABASE privatim TO dev;
+EOF"
+
+
 git clone [email protected]:seantis/privatim.git
 cd privatim
 python3 -m venv venv
@@ -35,7 +48,7 @@ make run
 ## Run the project's tests
 
 ```
-pytest -n 3
+pytest -n auto
 ```
 
 

development.ini.example

@@ -19,8 +19,8 @@ pyramid.available_languages =
 # pyramid.includes =
     # pyramid_debugtoolbar
 
-sqlalchemy.url = sqlite:///%(here)s/privatim.sqlite
-# sqlalchemy.url = postgresql://dev:postgres@localhost:5432/privatim
+# sqlalchemy.url = sqlite:///%(here)s/privatim.sqlite
+sqlalchemy.url = postgresql://dev:postgres@localhost:5432/privatim
 
 session.type = file
 session.data_dir = %(here)s/data/sessions/data

requirements.txt

@@ -1,7 +1,7 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile setup.cfg - -o requirements.txt --no-emit-package setuptools
 -e .
-alembic==1.13.1
+alembic==1.13.2
     # via
     #   privatim (setup.cfg)
     #   privatim
@@ -21,10 +21,14 @@ bcrypt==4.1.3
     #   privatim
 beaker==1.13.0
     # via pyramid-beaker
-certifi==2024.6.2
+brotli==1.1.0
+    # via fonttools
+certifi==2024.7.4
     # via
     #   requests
     #   sentry-sdk
+cffi==1.16.0
+    # via weasyprint
 chameleon==4.5.4
     # via pyramid-chameleon
 charset-normalizer==3.3.2
@@ -33,9 +37,11 @@ click==8.1.7
     # via
     #   privatim (setup.cfg)
     #   privatim
+cssselect2==0.7.0
+    # via weasyprint
 dnspython==2.6.1
     # via email-validator
-email-validator==2.1.2
+email-validator==2.2.0
     # via
     #   privatim (setup.cfg)
     #   privatim
@@ -47,9 +53,13 @@ fasteners==0.19
     # via
     #   privatim (setup.cfg)
     #   privatim
+fonttools==4.53.1
+    # via weasyprint
 greenlet==3.0.3
     # via sqlalchemy
-humanize==4.9.0
+html5lib==1.1
+    # via weasyprint
+humanize==4.10.0
     # via
     #   privatim (setup.cfg)
     #   privatim
@@ -74,18 +84,23 @@ markupsafe==2.1.5
     #   privatim
     #   werkzeug
     #   wtforms
-nh3==0.2.17
+nh3==0.2.18
     # via
     #   privatim (setup.cfg)
     #   privatim
 packaging==24.1
     # via zope-sqlalchemy
 pastedeploy==3.1.0
     # via plaster-pastedeploy
-pillow==10.3.0
+pdftotext==2.2.2
     # via
     #   privatim (setup.cfg)
     #   privatim
+pillow==10.4.0
+    # via
+    #   privatim (setup.cfg)
+    #   privatim
+    #   weasyprint
 plaster==1.1.2
     # via
     #   plaster-pastedeploy
@@ -99,12 +114,14 @@ psycopg2==2.9.9
     # via
     #   privatim (setup.cfg)
     #   privatim
+pycparser==2.22
+    # via cffi
+pydyf==0.10.0
+    # via weasyprint
 pygments==2.18.0
     # via pyramid-debugtoolbar
-pypdf==4.2.0
-    # via
-    #   privatim (setup.cfg)
-    #   privatim
+pyphen==0.15.0
+    # via weasyprint
 pyramid==2.0.2
     # via
     #   privatim (setup.cfg)
@@ -156,13 +173,15 @@ sedate==1.1.0
     # via
     #   privatim (setup.cfg)
     #   privatim
-sentry-sdk==2.5.1
+sentry-sdk==2.8.0
     # via
     #   privatim (setup.cfg)
     #   privatim
 six==1.16.0
-    # via python-dateutil
-sqlalchemy==2.0.30
+    # via
+    #   html5lib
+    #   python-dateutil
+sqlalchemy==2.0.31
     # via
     #   privatim (setup.cfg)
     #   alembic
@@ -178,6 +197,10 @@ sqlalchemy-utils==0.41.2
     # via
     #   privatim (setup.cfg)
     #   privatim
+tinycss2==1.3.0
+    # via
+    #   cssselect2
+    #   weasyprint
 transaction==4.0
     # via
     #   privatim (setup.cfg)
@@ -193,7 +216,6 @@ typing-extensions==4.12.2
     #   privatim (setup.cfg)
     #   alembic
     #   privatim
-    #   pypdf
     #   sqlalchemy
 urllib3==2.2.2
     # via
@@ -205,6 +227,15 @@ waitress==3.0.0
     # via
     #   privatim (setup.cfg)
     #   privatim
+weasyprint==62.3
+    # via
+    #   privatim (setup.cfg)
+    #   privatim
+webencodings==0.5.1
+    # via
+    #   cssselect2
+    #   html5lib
+    #   tinycss2
 webob==1.8.7
     # via
     #   privatim (setup.cfg)
@@ -246,6 +277,8 @@ zope-sqlalchemy==3.1
     # via
     #   privatim (setup.cfg)
     #   privatim
+zopfli==0.2.3
+    # via fonttools
 
 # The following packages were excluded from the output:
 # setuptools

requirements/compile.sh

@@ -19,8 +19,8 @@ echo "-e ." | uv pip compile setup.cfg - \
 
 echo "Building tests_requirements.txt"
 uv pip compile setup.cfg \
-    --extra testing \
-    --extra typing \
+    --extra test \
+    --extra mypy \
     -o tests_requirements.txt \
-    -c requirements.txt \
+    -c requirements.txt
     "$@"

setup.cfg

@@ -34,6 +34,7 @@ install_requires =
     Markdown
     markupsafe
     nh3
+    pdftotext
     pyramid
     pyramid_beaker
     pyramid_chameleon
@@ -53,7 +54,7 @@ install_requires =
     typing_extensions
     WebOb
     waitress
-    WeasyPrint
+    weasyprint==62.3
     WTForms
     werkzeug
     plaster_pastedeploy
@@ -84,24 +85,14 @@ console_scripts =
     add_meeting = privatim.cli.add_meeting:main
     delete_meetings = privatim.cli.delete_meetings:main
     upgrade = privatim.cli.upgrade:upgrade
+    print_tsvectors =  privatim.cli.helpers:print_tsvectors
+    print_text =  privatim.cli.helpers:print_text
+    reindex =  privatim.cli.helpers:reindex
     shell = privatim.cli.shell:shell
 
 [options.extras_require]
-dev =
-    bandit[toml]
-    flake8
-    flake8-bugbear
-    flake8-comprehensions
-    flake8-markupsafe@git+https://github.com/vmagamedov/flake8-markupsafe#egg=b391bd13df9330e01666d304b7f4403d67e5ceba
-    flake8-noqa
-    flake8-pyi
-    flake8-type-checking
-    pre-commit
-    pytest-profiling
-    scrambler
-
 mypy =
-    mypy
+    mypy==1.10.1
     mypy_zope>1.0.0
     lxml-stubs
     types-babel
@@ -124,6 +115,7 @@ test =
     pytest
     pytest-cov
     pytest-codecov
+    pytest-postgresql
     pytest-xdist
     pytest-sugar
     pyquery

src/privatim/__init__.py

@@ -1,5 +1,7 @@
 from functools import partial
 from fanstatic import Fanstatic
+from sqlalchemy.dialects.postgresql import TSVECTOR
+
 from privatim.layouts.action_menu import ActionMenuEntry
 from pyramid.config import Configurator
 from pyramid_beaker import session_factory_from_settings
@@ -69,7 +71,7 @@
         user = request.user
         if not user:
             return ''
-        return request.route_url('download_general_file', id=user.picture.id)
+        return request.route_url('download_file', id=user.picture.id)
 
     config.add_request_method(profile_pic, 'profile_pic', property=True)
     config.add_request_method(MessageQueue, 'messages', reify=True)
@@ -147,4 +149,9 @@
             ),
         )
 
+    context.operations.add_column(
+        'consultations',
+        Column('searchable_text_de_CH', TSVECTOR())
+    )
+
     context.commit()

src/privatim/cache.py

@@ -5,6 +5,7 @@
 
 from pyramid.threadlocal import get_current_request
 
+
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Callable