From 5c37fde16aec6810bed5deee45cd68b03e5a8952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 25 Jun 2024 11:39:26 +0200 Subject: [PATCH 01/52] Fix docstring. --- src/privatim/cli/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/privatim/cli/shell.py b/src/privatim/cli/shell.py index c940a94..0582737 100644 --- a/src/privatim/cli/shell.py +++ b/src/privatim/cli/shell.py @@ -70,7 +70,7 @@ def shell() -> None: Example: Query User: from privatim.models.user import User -query = session.query(User).filter_by(username='admin@example.org') +query = session.query(User).filter_by(email='admin@example.org') user = query.one() user.username = 'info@example.org' commit() From b8cf6caef65e0130a1c2ca3068929389dc42550f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 25 Jun 2024 13:25:28 +0200 Subject: [PATCH 02/52] Use postgres as default for the database --- development.ini.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/development.ini.example b/development.ini.example index 953b3a4..cac573e 100644 --- a/development.ini.example +++ b/development.ini.example @@ -19,8 +19,8 @@ pyramid.available_languages = # pyramid.includes = # pyramid_debugtoolbar -sqlalchemy.url = sqlite:///%(here)s/privatim.sqlite -# sqlalchemy.url = postgresql://dev:postgres@localhost:5432/privatim +# sqlalchemy.url = sqlite:///%(here)s/privatim.sqlite +sqlalchemy.url = postgresql://dev:postgres@localhost:5432/privatim session.type = file session.data_dir = %(here)s/data/sessions/data From 78642f3bc0e5cc9a6e5c62468bc0fe9f3a9c5628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Wed, 26 Jun 2024 15:37:28 +0200 Subject: [PATCH 03/52] wip --- requirements.txt | 45 +++++- setup.cfg | 5 +- src/privatim/__init__.py | 10 ++ src/privatim/cli/reindex.py | 22 +++ src/privatim/forms/fields/fields.py | 2 + src/privatim/forms/search_form.py | 30 ++++ src/privatim/i18n/__init__.py | 6 +- src/privatim/layouts/macros.pt | 12 ++ src/privatim/layouts/navbar.pt | 56 ++++---- src/privatim/layouts/navbar.py | 11 +- src/privatim/models/__init__.py | 37 ++++- src/privatim/models/associated_file.py | 44 ++++++ src/privatim/models/consultation.py | 18 ++- src/privatim/models/searchable.py | 93 ++++++++++++ src/privatim/models/utils.py | 68 +++++++++ src/privatim/static/css/custom.css | 14 +- src/privatim/views/__init__.py | 8 ++ src/privatim/views/comment.py | 2 +- src/privatim/views/search.py | 132 ++++++++++++++++++ src/privatim/views/templates/activities.pt | 7 +- .../views/templates/search_results.pt | 52 +++++++ tests/conftest.py | 57 +++++--- tests/models/test_searchable_mixin.py | 25 ++++ tests/views/client/test_views_homepage.py | 14 ++ 24 files changed, 695 insertions(+), 75 deletions(-) create mode 100644 src/privatim/cli/reindex.py create mode 100644 src/privatim/forms/search_form.py create mode 100644 src/privatim/models/searchable.py create mode 100644 src/privatim/models/utils.py create mode 100644 src/privatim/views/search.py create mode 100644 src/privatim/views/templates/search_results.pt create mode 100644 tests/models/test_searchable_mixin.py create mode 100644 tests/views/client/test_views_homepage.py diff --git a/requirements.txt b/requirements.txt index 2b03780..9af653e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,10 +21,14 @@ bcrypt==4.1.3 # privatim beaker==1.13.0 # via pyramid-beaker +brotli==1.1.0 + # via fonttools certifi==2024.6.2 # via # requests # sentry-sdk +cffi==1.16.0 + # via weasyprint chameleon==4.5.4 # via pyramid-chameleon charset-normalizer==3.3.2 @@ -33,6 +37,8 @@ click==8.1.7 # via # privatim (setup.cfg) # privatim +cssselect2==0.7.0 + # via weasyprint dnspython==2.6.1 # via email-validator email-validator==2.1.2 @@ -47,8 +53,12 @@ fasteners==0.19 # via # privatim (setup.cfg) # privatim +fonttools==4.53.0 + # via weasyprint greenlet==3.0.3 # via sqlalchemy +html5lib==1.1 + # via weasyprint humanize==4.9.0 # via # privatim (setup.cfg) @@ -82,10 +92,15 @@ packaging==24.1 # via zope-sqlalchemy pastedeploy==3.1.0 # via plaster-pastedeploy +pdftotext==2.2.2 + # via + # privatim (setup.cfg) + # privatim pillow==10.3.0 # via # privatim (setup.cfg) # privatim + # weasyprint plaster==1.1.2 # via # plaster-pastedeploy @@ -99,12 +114,14 @@ psycopg2==2.9.9 # via # privatim (setup.cfg) # privatim +pycparser==2.22 + # via cffi +pydyf==0.10.0 + # via weasyprint pygments==2.18.0 # via pyramid-debugtoolbar -pypdf==4.2.0 - # via - # privatim (setup.cfg) - # privatim +pyphen==0.15.0 + # via weasyprint pyramid==2.0.2 # via # privatim (setup.cfg) @@ -161,7 +178,9 @@ sentry-sdk==2.5.1 # privatim (setup.cfg) # privatim six==1.16.0 - # via python-dateutil + # via + # html5lib + # python-dateutil sqlalchemy==2.0.30 # via # privatim (setup.cfg) @@ -178,6 +197,10 @@ sqlalchemy-utils==0.41.2 # via # privatim (setup.cfg) # privatim +tinycss2==1.3.0 + # via + # cssselect2 + # weasyprint transaction==4.0 # via # privatim (setup.cfg) @@ -193,7 +216,6 @@ typing-extensions==4.12.2 # privatim (setup.cfg) # alembic # privatim - # pypdf # sqlalchemy urllib3==2.2.2 # via @@ -205,6 +227,15 @@ waitress==3.0.0 # via # privatim (setup.cfg) # privatim +weasyprint==62.3 + # via + # privatim (setup.cfg) + # privatim +webencodings==0.5.1 + # via + # cssselect2 + # html5lib + # tinycss2 webob==1.8.7 # via # privatim (setup.cfg) @@ -246,6 +277,8 @@ zope-sqlalchemy==3.1 # via # privatim (setup.cfg) # privatim +zopfli==0.2.3 + # via fonttools # The following packages were excluded from the output: # setuptools diff --git a/setup.cfg b/setup.cfg index e566d0a..6302486 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ install_requires = Markdown markupsafe nh3 + pdftotext pyramid pyramid_beaker pyramid_chameleon @@ -53,7 +54,7 @@ install_requires = typing_extensions WebOb waitress - WeasyPrint + weasyprint==62.3 WTForms werkzeug plaster_pastedeploy @@ -83,6 +84,7 @@ console_scripts = add_meeting = privatim.cli.add_meeting:main delete_meetings = privatim.cli.delete_meetings:main upgrade = privatim.cli.upgrade:upgrade + reindex = privatim.cli.reindex:reindex shell = privatim.cli.shell:shell [options.extras_require] @@ -123,6 +125,7 @@ test = pytest pytest-cov pytest-codecov + pytest-postgresql pytest-xdist pytest-sugar pyquery diff --git a/src/privatim/__init__.py b/src/privatim/__init__.py index d09dea4..68271bc 100644 --- a/src/privatim/__init__.py +++ b/src/privatim/__init__.py @@ -1,5 +1,7 @@ from functools import partial from fanstatic import Fanstatic +from sqlalchemy.dialects.postgresql import TSVECTOR + from privatim.layouts.action_menu import ActionMenuEntry from pyramid.config import Configurator from pyramid_beaker import session_factory_from_settings @@ -130,4 +132,12 @@ def upgrade(context: 'UpgradeContext'): # type: ignore[no-untyped-def] ), ) + if not context.has_column('consultations', + 'searchable_text_de_CH'): + for column in ('searchable_text_de_CH',): + if not context.has_column('consultations', column): + context.operations.add_column( + 'consultations', Column(column, TSVECTOR()) + ) + context.commit() diff --git a/src/privatim/cli/reindex.py b/src/privatim/cli/reindex.py new file mode 100644 index 0000000..a6c6ba5 --- /dev/null +++ b/src/privatim/cli/reindex.py @@ -0,0 +1,22 @@ +import click +import transaction +from pyramid.paster import bootstrap +from pyramid.paster import get_appsettings +from privatim.models.searchable import reindex_full_text_search +from privatim.orm import get_engine, Base, get_session_factory, get_tm_session + + +@click.command() +@click.argument('config_uri') +def reindex(config_uri: str) -> None: + + bootstrap(config_uri) # is this needed? + settings = get_appsettings(config_uri) + engine = get_engine(settings) + Base.metadata.create_all(engine) + + session_factory = get_session_factory(engine) + + with transaction.manager: + dbsession = get_tm_session(session_factory, transaction.manager) + reindex_full_text_search(dbsession, transaction.manager) diff --git a/src/privatim/forms/fields/fields.py b/src/privatim/forms/fields/fields.py index d98c1ea..a57d165 100644 --- a/src/privatim/forms/fields/fields.py +++ b/src/privatim/forms/fields/fields.py @@ -160,6 +160,8 @@ def process_formdata(self, valuelist: list['RawFormValue']) -> None: class SearchableSelectField(SelectField): """A multiple select field with tom-select.js support. + Note: This is unrelated to PostgreSQL full-text search, which also uses + the term 'searchable'. Note: you need to call form.raw_data() to actually get the choices as list """ diff --git a/src/privatim/forms/search_form.py b/src/privatim/forms/search_form.py new file mode 100644 index 0000000..a096f51 --- /dev/null +++ b/src/privatim/forms/search_form.py @@ -0,0 +1,30 @@ +from wtforms.fields.simple import SearchField +from wtforms.validators import DataRequired +from privatim.forms.core import Form +from privatim.i18n import _ + + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from pyramid.interfaces import IRequest + + +class SearchForm(Form): + + def __init__( + self, + request: 'IRequest', + ) -> None: + session = request.dbsession + super().__init__( + request.POST, + meta={ + 'dbsession': session + } + ) + search = SearchField( + _('Search'), + [DataRequired()], + render_kw={ + }, + ) diff --git a/src/privatim/i18n/__init__.py b/src/privatim/i18n/__init__.py index ac587a2..5c64633 100644 --- a/src/privatim/i18n/__init__.py +++ b/src/privatim/i18n/__init__.py @@ -4,11 +4,15 @@ from .translation_string import TranslationStringFactory +locales = {'de_CH': 'german'} + + _ = TranslationStringFactory('privatim') __all__ = ( '_', 'LocaleNegotiator', 'pluralize', - 'translate' + 'translate', + locales, ) diff --git a/src/privatim/layouts/macros.pt b/src/privatim/layouts/macros.pt index b12d4b7..96b2a99 100644 --- a/src/privatim/layouts/macros.pt +++ b/src/privatim/layouts/macros.pt @@ -155,4 +155,16 @@ + + + +
+ +
+ + diff --git a/src/privatim/layouts/navbar.pt b/src/privatim/layouts/navbar.pt index f1e4498..455d6ef 100644 --- a/src/privatim/layouts/navbar.pt +++ b/src/privatim/layouts/navbar.pt @@ -15,53 +15,51 @@ Activities + i18n:translate="">Activities
  • People + i18n:translate="">People
  • Consultations + i18n:translate="">Consultations
  • Working Groups + i18n:translate="">Working Groups
    • -
    - -
    - - -
    -
    - +
    + + + +
    + + +
    +
    + + diff --git a/src/privatim/layouts/navbar.py b/src/privatim/layouts/navbar.py index c5873f5..f1e786f 100644 --- a/src/privatim/layouts/navbar.py +++ b/src/privatim/layouts/navbar.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING -from privatim.i18n import _ + +from ..forms.search_form import SearchForm if TYPE_CHECKING: from pyramid.interfaces import IRequest @@ -39,10 +40,8 @@ def __html__(self) -> str: def navbar(context: object, request: 'IRequest') -> 'RenderData': + form = SearchForm(request) return { - 'entries': [ - NavbarEntry( - request, _('Activities'), request.route_url('activities') - ), - ] + 'form': form, + 'search': request.route_url('search'), } diff --git a/src/privatim/models/__init__.py b/src/privatim/models/__init__.py index 87871ad..4424bd0 100644 --- a/src/privatim/models/__init__.py +++ b/src/privatim/models/__init__.py @@ -1,13 +1,15 @@ -from typing import TYPE_CHECKING -from sqlalchemy.orm import configure_mappers +from sqlalchemy import event, func +from sqlalchemy.orm import configure_mappers, Mapper +from privatim.i18n import locales # XXX import or define all models here to ensure they are attached to the # Base.metadata prior to any initialization routines # https://docs.pylonsproject.org/projects/pyramid_cookbook/en/latest/database/sqlalchemy.html#importing-all-sqlalchemy-models from privatim.models.group import Group from privatim.models.group import WorkingGroup +from privatim.models.searchable import SearchableMixin from privatim.models.user import User from privatim.models.consultation import Consultation from privatim.models.meeting import Meeting, AgendaItem @@ -15,7 +17,7 @@ from privatim.models.consultation import Tag from privatim.models.statement import Statement from privatim.models.password_change_token import PasswordChangeToken -from privatim.orm import get_engine +from privatim.orm import get_engine, Base from privatim.orm import get_session_factory from privatim.orm import get_tm_session @@ -33,15 +35,44 @@ GeneralFile +from typing import TYPE_CHECKING, Any, TypeVar # noqa: E402 if TYPE_CHECKING: from pyramid.config import Configurator + from sqlalchemy.engine import Connection + T = TypeVar('T', bound='SearchableBase') + + class SearchableBase(Base, SearchableMixin): + pass # Run ``configure_mappers`` after defining all of the models to ensure # all relationships can be setup. configure_mappers() +def update_searchable_text_listener( + mapper: Mapper['SearchableBase'], connection: 'Connection', target: Any +) -> None: + if connection.engine.name != 'postgresql': + # todo: fallback if local development uses sqlite? + raise ValueError('Only PostgreSQL is supported') + for locale, language in locales.items(): + if hasattr(target, f'searchable_text_{locale}'): + setattr( + target, + f'searchable_text_{locale}', + func.to_tsvector(language, target.searchable_text), + ) + + +def register_search_listeners(model) -> None: + event.listen(model, 'after_insert', update_searchable_text_listener) + event.listen(model, 'after_update', update_searchable_text_listener) + + +register_search_listeners(Consultation) + + def includeme(config: 'Configurator') -> None: """ Initialize the model for a Pyramid app. diff --git a/src/privatim/models/associated_file.py b/src/privatim/models/associated_file.py index dea4bab..caaa795 100644 --- a/src/privatim/models/associated_file.py +++ b/src/privatim/models/associated_file.py @@ -1,9 +1,53 @@ +from sqlalchemy import select +from sqlalchemy.orm import object_session +from sqlalchemy_utils import observes from privatim.models.file import GeneralFile from privatim.orm.associable import associated +from typing import Sequence + + class AssociatedFiles: """ Use this mixin if uploaded files belong to a specific instance """ # one-to-many files = associated(GeneralFile, 'files') + + def reindex_files(self, searchable_files: Sequence[GeneralFile]) -> None: + """ Extract the text from the localized files and save it together with + the language. + + The language is determined by the locale, e.g. `de_CH` -> `german`. + + """ + + file: GeneralFile + # files: dict[str, list[tuple[GeneralFile, bool]]] + + for file in searchable_files: + print(file) + # if attribute.extension == 'pdf': + # file = SwissVote.__dict__[name].__get_by_locale__( + # self, locale + # ) + # if file: + # index = name in self.indexed_files + # files[locale].append((file, index)) + # + # setattr( + # self, + # f'searchable_text_{locale}', + # func.to_tsvector(locales[locale], text) + # ) + + def searchable_files(self) -> Sequence[GeneralFile]: + # For now we just consider PDF's + stmt = select(GeneralFile).where( + GeneralFile.content_type == 'application/pdf' + ) + return object_session(self).execute(stmt).scalars().all() + + @observes('files') + def files_observer(self) -> None: + self.reindex_files(self.searchable_files()) diff --git a/src/privatim/models/consultation.py b/src/privatim/models/consultation.py index f285fdc..0dc99e6 100644 --- a/src/privatim/models/consultation.py +++ b/src/privatim/models/consultation.py @@ -1,18 +1,21 @@ from datetime import datetime from sedate import utcnow from sqlalchemy import Text, ForeignKey -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import TSVECTOR +from sqlalchemy.orm import Mapped, mapped_column, relationship, deferred from pyramid.authorization import Allow from pyramid.authorization import Authenticated from privatim.models.associated_file import AssociatedFiles from privatim.models.commentable import Commentable +from privatim.models.searchable import SearchableMixin from privatim.orm import Base from privatim.orm.meta import UUIDStrPK from privatim.orm.meta import UUIDStr as UUIDStrType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Iterator + if TYPE_CHECKING: from privatim.types import ACL from privatim.models import User @@ -49,7 +52,7 @@ class Tag(Base): ) -class Consultation(Base, Commentable, AssociatedFiles): +class Consultation(Base, Commentable, AssociatedFiles, SearchableMixin): """Vernehmlassung (Verfahren der Stellungnahme zu einer öffentlichen Frage)""" @@ -83,6 +86,15 @@ class Consultation(Base, Commentable, AssociatedFiles): ForeignKey('users.id'), nullable=True ) + # searchable attachment texts + searchable_text_de_CH: Mapped[str] = deferred(mapped_column(TSVECTOR)) + + @classmethod + def searchable_fields(cls) -> Iterator[str]: + yield 'title' + yield 'description' + yield 'recommendation' + def __acl__(self) -> list['ACL']: return [ (Allow, Authenticated, ['view']), diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py new file mode 100644 index 0000000..d51ca37 --- /dev/null +++ b/src/privatim/models/searchable.py @@ -0,0 +1,93 @@ +from sqlalchemy import func, update, Text +import transaction +from sqlalchemy.ext.hybrid import hybrid_property +import inspect +from sqlalchemy import text +import sys +from functools import cache + +from sqlalchemy.orm import Session, class_mapper + +from privatim.i18n import locales +from privatim.orm import Base + + +from typing import Iterator + + +class SearchableMixin: + @classmethod + def searchable_fields(cls) -> Iterator[str]: + # Override this method in each model to specify searchable fields + raise NotImplementedError( + "Searchable fields must be defined for each model" + ) + + @hybrid_property + def searchable_text(self) -> str: + return ' '.join( + str(getattr(self, field)) for field in self.searchable_fields() + ) + + @searchable_text.expression + def searchable_text(cls): + # Called when the property is used in a SQL expression + return func.concat_ws( + ' ', + *[getattr(cls, field) for field in cls.searchable_fields()] + ) + + +def searchable_models() -> tuple[type[Base], ...]: + model_classes = set() + for _ in Base.metadata.tables.values(): + for mapper in Base.registry.mappers: + cls = mapper.class_ + if ( + inspect.isclass(cls) + and issubclass(cls, SearchableMixin) + and issubclass(cls, Base) + and cls != SearchableMixin + ): + model_classes.add(cls) + return tuple(model_classes) + + +def reindex_full_text_search(session: Session, manager) -> None: + """ + Updates the searchable_text_{} columns. + + 1. We use func.cast() to explicitly cast the searchable_text_{locale} + column to Text type. + This ensures that we're passing a text value to to_tsvector, + not a tsvector. + j + 2. We wrap this in a func.coalesce() call, which will + return an empty string if the column value is NULL. This prevents + potential errors if some rows have NULL values in the searchable_text + column. + + """ + models = searchable_models() + # todo: remove later + assert len(models) != 0, "No models with searchable fields found" + for model in models: + assert issubclass(model, SearchableMixin) + for locale, language in locales.items(): + assert language == 'german' # todo: remove later + if hasattr(model, f'searchable_text_{locale}'): + update_stmt = ( + update(model) + .values({ + f'searchable_text_{locale}': func.to_tsvector( + language, + func.cast(model.searchable_text, Text) + ) + }) + ) + updated = getattr(model, f'searchable_text_{locale}') + session.execute(update_stmt) + print('Reindex full text search for', + f'{model}.searchable_text_{locale}') + session.flush() + # manager.commit() diff --git a/src/privatim/models/utils.py b/src/privatim/models/utils.py new file mode 100644 index 0000000..2e0b978 --- /dev/null +++ b/src/privatim/models/utils.py @@ -0,0 +1,68 @@ +from mimetypes import guess_extension +from pdftotext import PDF # type: ignore + + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from _typeshed import SupportsRead + + +def word_count(text: str) -> int: + """The word-count of the given text. Goes through the string exactly + once and has constant memory usage. Not super sophisticated though. + + """ + if not text: + return 0 + + count = 0 + inside_word = False + + for char in text: + if char.isspace(): + inside_word = False + elif not inside_word: + count += 1 + inside_word = True + + return count + + +def extract_pdf_info( + content: 'SupportsRead[bytes]', remove: str = '\0' +) -> tuple[int, str]: + """Extracts the number of pages and text from a PDF. + + Requires poppler. + """ + try: + content.seek(0) # type:ignore[attr-defined] + except Exception: + pass + + pages = PDF(content) + + def clean(text: str) -> str: + for character in remove: + text = text.replace(character, '') + return ' '.join(text.split()) + + return len(pages), ' '.join(clean(page) for page in pages).strip() + + +def extension_for_content_type( + content_type: str, filename: str | None = None +) -> str: + """Gets the extension for the given content type. Note that this is + *meant for display only*. A file claiming to be a PDF might not be one, + but this function would not let you know that. + + """ + + if filename is not None: + _, sep, ext = filename.rpartition('.') + ext = ext.lower() if sep else '' + else: + ext = guess_extension(content_type, strict=False) or '' + + return ext.strip('. ') diff --git a/src/privatim/static/css/custom.css b/src/privatim/static/css/custom.css index c63fc36..afb5c0e 100644 --- a/src/privatim/static/css/custom.css +++ b/src/privatim/static/css/custom.css @@ -79,6 +79,10 @@ } +body { + font-family: "DM Sans", sans-serif; +} + .bg-light { background-color: #EDEDED !important; } @@ -137,10 +141,14 @@ a:hover { .bd-mode-toggle .dropdown-menu .active .bi { display: block !important; } - -body { - font-family: "DM Sans", sans-serif; +#search-button { + border: none; } +#search-button:hover { + background-color: transparent; + color: var(--primary-color); +} + /* Fix double border in form*/ .ts-control { diff --git a/src/privatim/views/__init__.py b/src/privatim/views/__init__.py index 0982137..4775247 100644 --- a/src/privatim/views/__init__.py +++ b/src/privatim/views/__init__.py @@ -38,6 +38,7 @@ from privatim.views.people import people_view, person_view from privatim.views.profile import profile_view, add_profile_image_view from privatim.views.comment import add_comment_view +from privatim.views.search import search from privatim.views.working_groups import ( add_or_edit_working_group, delete_working_group_view, @@ -504,3 +505,10 @@ def includeme(config: 'Configurator') -> None: xhr=True, request_param='target_url' ) + + config.add_route('search', '/search') + config.add_view( + search, + route_name='search', + renderer='templates/search_results.pt', + ) diff --git a/src/privatim/views/comment.py b/src/privatim/views/comment.py index ac209b0..8e2f264 100644 --- a/src/privatim/views/comment.py +++ b/src/privatim/views/comment.py @@ -49,8 +49,8 @@ def add_comment_view( user=request.user, parent=parent, ) - context.comments.append(comment) session.add(comment) + context.comments.append(comment) message = _('Successfully added comment') request.messages.add(message, 'success') data = { diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py new file mode 100644 index 0000000..0c58ab0 --- /dev/null +++ b/src/privatim/views/search.py @@ -0,0 +1,132 @@ +from sqlalchemy import func, select, ColumnElement +from privatim.forms.search_form import SearchForm +from privatim.models import Consultation +from privatim.i18n import locales +from sqlalchemy import or_ + +from privatim.models.searchable import searchable_models + + +from typing import TYPE_CHECKING, List, NamedTuple, Protocol +if TYPE_CHECKING: + from pyramid.interfaces import IRequest + from sqlalchemy.orm import Query + + +class SearchResult(NamedTuple): + id: int + title: str + model_type: str + description: str + rank: float + url: str + + +class SortingStrategy(Protocol): + def sort(self, results: List[SearchResult]) -> List[SearchResult]: ... + + +class RankSortStrategy: + def sort(self, results: List[SearchResult]) -> List[SearchResult]: + return sorted(results, key=lambda x: x.rank, reverse=True) + + +class AlphabeticalSortStrategy: + def sort(self, results: List[SearchResult]) -> List[SearchResult]: + return sorted(results, key=lambda x: x.title.lower()) + + +class ModelTypeSortStrategy: + def sort(self, results: List[SearchResult]) -> List[SearchResult]: + return sorted(results, key=lambda x: (x.model_type, -x.rank)) + + +class SearchResultCollection: + term: str + results: List[SearchResult] + sorting_strategy: SortingStrategy = AlphabeticalSortStrategy() + + def __init__(self, term: str, session: 'Session', language='de_CH'): + lang = locales[language] + self.session = session + self.term = func.websearch_to_tsquery(lang, term) + self.results = [] + self.sorting_strategy = AlphabeticalSortStrategy() + + def query(self) -> 'Query[SwissVote]': + + query = self.session.query(Consultation) + query = query.filter(or_(*self.term_filter)) + return query + + @property + def term_filter(self) -> list['ColumnElement[bool]']: + """ Returns a list of SqlAlchemy filter statements based on the search + term. + + """ + + return self.term_filter_text + + def __post_init__(self): + query = self.query() + # create : List[SearchResult] from the qeury + + + self.results = self.perform_() + + def sort_results(self): + self.results = self.sorting_strategy.sort(self.results) + + def set_sorting_strategy(self, strategy: SortingStrategy): + self.sorting_strategy = strategy + + @property + def term_filter_text(self) -> list['ColumnElement[bool]']: + """ Returns a list of SqlAlchemy filter statements matching possible + fulltext attributes based on the term. + + """ + + if not self.term: + return [] + + def match( + column: 'ColumnElement[str] | ColumnElement[str | None]', + language: str + ) -> 'ColumnElement[bool]': + return column.op('@@')(func.to_tsquery(language, self.term)) + + def match_convert( + column: 'ColumnElement[str] | ColumnElement[str | None]', + language: str + ) -> 'ColumnElement[bool]': + return match(func.to_tsvector(language, column), language) + + return [ + match(Consultation.searchable_text_de_CH, 'german'), + ] + + def __len__(self): + return len(self.results) + + def __iter__(self): + return iter(self.results) + + def __getitem__(self, index): + return self.results[index] + + +def search(request: 'IRequest'): + session = request.dbsession + form = SearchForm(request) + if request.method == 'POST' and form.validate(): + query = request.POST['search'] + + # todo:remove later + stmt = select(Consultation.searchable_text_de_CH) + assert None not in session.execute(stmt).scalars().all() + + result_collection = SearchResultCollection(query, session) + + return {'search_results': result_collection} diff --git a/src/privatim/views/templates/activities.pt b/src/privatim/views/templates/activities.pt index 4d5d657..a481291 100644 --- a/src/privatim/views/templates/activities.pt +++ b/src/privatim/views/templates/activities.pt @@ -13,10 +13,11 @@
    - Add Consultation + Add + Consultation
    -     + diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt new file mode 100644 index 0000000..73e89b2 --- /dev/null +++ b/src/privatim/views/templates/search_results.pt @@ -0,0 +1,52 @@ + + + +
    +
    +
    +

    Search Results

    +

    Showing + results for ""

    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    + +
    +

    +
    + Relevance Score: + View Details +
    +
    +
    +     + + + +
    +
    +
    +     +     diff --git a/tests/conftest.py b/tests/conftest.py index 7d102a8..23a8bc9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,12 +22,20 @@ @pytest.fixture -def base_config(): +def base_config(_postgresql): msg = '.*SQLAlchemy must convert from floating point.*' warnings.filterwarnings('ignore', message=msg) + # config = testing.setUp(settings={ + # 'sqlalchemy.url': 'sqlite:///:memory:', + # }) + config = testing.setUp(settings={ - 'sqlalchemy.url': 'sqlite:///:memory:', + 'sqlalchemy.url': ( + f'postgresql+psycopg://{_postgresql.info.user}:@' + f'{_postgresql.info.host}:{_postgresql.info.port}' + f'/{_postgresql.info.dbname}' + ), }) yield config testing.tearDown() @@ -45,13 +53,14 @@ def config(base_config, monkeypatch, tmpdir) -> 'Configurator': settings = base_config.get_settings() engine = get_engine(settings) + # enable foreign key constraints in sqlite, so we can rely on them # working during testing. - sqlalchemy.event.listen( - engine, - 'connect', - lambda c, r: c.execute('pragma foreign_keys=ON') - ) + # sqlalchemy.event.listen( + # engine, + # 'connect', + # lambda c, r: c.execute('pragma foreign_keys=ON') + # ) Base.metadata.create_all(engine) session_factory = get_session_factory(engine) @@ -78,13 +87,14 @@ def init_with_dbsession(self, *args, dbsession=dbsession, **kwargs): return base_config -@pytest.fixture -def pg_config(postgresql, monkeypatch): +# requires pytest-postgresql: +@pytest.fixture(scope='session') +def pg_config(_postgresql, monkeypatch): config = testing.setUp(settings={ 'sqlalchemy.url': ( - f'postgresql+psycopg://{postgresql.info.user}:@' - f'{postgresql.info.host}:{postgresql.info.port}' - f'/{postgresql.info.dbname}' + f'postgresql+psycopg://{_postgresql.info.user}:@' + f'{_postgresql.info.host}:{_postgresql.info.port}' + f'/{_postgresql.info.dbname}' ), }) config.include('privatim.models') @@ -166,7 +176,7 @@ def mailer(config): return mailer -@pytest.fixture() +@pytest.fixture(scope='session') def engine(app_settings): engine = engine_from_config(app_settings) Base.metadata.create_all(engine) @@ -177,16 +187,25 @@ def engine(app_settings): return engine -@pytest.fixture +@pytest.fixture(scope='session') def connection(engine): connection = engine.connect() yield connection connection.close() -@pytest.fixture(scope="session") -def app_settings(): - yield {"sqlalchemy.url": "sqlite://"} +@pytest.fixture(scope='session') +def _postgresql(postgresql): + return postgresql + + +@pytest.fixture(scope='session') +def app_settings(_postgresql): + yield {'sqlalchemy.url': ( + f'postgresql+psycopg://{_postgresql.info.user}:@' + f'{_postgresql.info.host}:{_postgresql.info.port}' + f'/{_postgresql.info.dbname}' + )} @pytest.fixture(scope="session") @@ -195,13 +214,13 @@ def app_inner(app_settings): yield app -@pytest.fixture +@pytest.fixture(scope='session') def app(app_inner, connection): app_inner.app.app.registry["dbsession_factory"].kw["bind"] = connection yield app_inner -@pytest.fixture(scope='function') +@pytest.fixture(scope='session') def client(app, engine): client = Client(app) diff --git a/tests/models/test_searchable_mixin.py b/tests/models/test_searchable_mixin.py new file mode 100644 index 0000000..366c075 --- /dev/null +++ b/tests/models/test_searchable_mixin.py @@ -0,0 +1,25 @@ +import transaction +from sqlalchemy import select +from sqlalchemy.orm import undefer +from privatim.models import Consultation +from privatim.models.searchable import reindex_full_text_search + + +def test_fulltext_indexing_on_searchable_fields(pg_config): + session = pg_config.dbsession + consultation = Consultation( + title='Datenschutz', + description='cat', + recommendation='bat', + ) + session.add(consultation) + session.flush() + + reindex_full_text_search(session, transaction.manager) + + updated = session.execute( + select(Consultation) + .options(undefer(Consultation.searchable_text_de_CH)) + .filter_by(title='Datenschutz') + ).scalar_one() + assert updated.searchable_text_de_CH == "'bat':3 'cat':2 'datenschutz':1" diff --git a/tests/views/client/test_views_homepage.py b/tests/views/client/test_views_homepage.py new file mode 100644 index 0000000..ce24b68 --- /dev/null +++ b/tests/views/client/test_views_homepage.py @@ -0,0 +1,14 @@ +def test_view_activities(client): + client.login_admin() + + page = client.get('/activities') + assert page.status_code == 200 + assert 'Aktivitäten' in page.text + + # fill out the search form + client.skip_n_forms = False + form = page.forms[0] + print(form.fields) + + form['search'] = 'My search' + page = form.submit() From b11eeeded86c9ee9aca1251656001c298331f524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Mon, 1 Jul 2024 16:37:23 +0200 Subject: [PATCH 04/52] update --- src/privatim/cli/reindex.py | 2 +- src/privatim/models/consultation.py | 6 + src/privatim/models/searchable.py | 5 +- src/privatim/views/search.py | 152 +++++++++--------- .../views/templates/search_results.pt | 51 ++---- tests/models/test_searchable_mixin.py | 2 +- 6 files changed, 92 insertions(+), 126 deletions(-) diff --git a/src/privatim/cli/reindex.py b/src/privatim/cli/reindex.py index a6c6ba5..25f71ea 100644 --- a/src/privatim/cli/reindex.py +++ b/src/privatim/cli/reindex.py @@ -19,4 +19,4 @@ def reindex(config_uri: str) -> None: with transaction.manager: dbsession = get_tm_session(session_factory, transaction.manager) - reindex_full_text_search(dbsession, transaction.manager) + reindex_full_text_search(dbsession) diff --git a/src/privatim/models/consultation.py b/src/privatim/models/consultation.py index 0dc99e6..ed97c8f 100644 --- a/src/privatim/models/consultation.py +++ b/src/privatim/models/consultation.py @@ -95,6 +95,12 @@ def searchable_fields(cls) -> Iterator[str]: yield 'description' yield 'recommendation' + def __repr__(self) -> str: + return ( + f'' + ) + def __acl__(self) -> list['ACL']: return [ (Allow, Authenticated, ['view']), diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index d51ca37..ceeb3d8 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -53,7 +53,7 @@ def searchable_models() -> tuple[type[Base], ...]: return tuple(model_classes) -def reindex_full_text_search(session: Session, manager) -> None: +def reindex_full_text_search(session: Session) -> None: """ Updates the searchable_text_{} columns. @@ -88,6 +88,5 @@ def reindex_full_text_search(session: Session, manager) -> None: updated = getattr(model, f'searchable_text_{locale}') session.execute(update_stmt) print('Reindex full text search for', - f'{model}.searchable_text_{locale}') + f'{model}.searchable_text_{locale} with val {updated}') session.flush() - # manager.commit() diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 0c58ab0..c30cbb5 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -1,101 +1,74 @@ -from sqlalchemy import func, select, ColumnElement +from sqlalchemy import func, select, ColumnElement, union_all, text, cast, \ + literal, String from privatim.forms.search_form import SearchForm -from privatim.models import Consultation +from privatim.models import Consultation, Meeting from privatim.i18n import locales from sqlalchemy import or_ +from privatim.models.comment import Comment from privatim.models.searchable import searchable_models from typing import TYPE_CHECKING, List, NamedTuple, Protocol if TYPE_CHECKING: from pyramid.interfaces import IRequest - from sqlalchemy.orm import Query - - -class SearchResult(NamedTuple): - id: int - title: str - model_type: str - description: str - rank: float - url: str - - -class SortingStrategy(Protocol): - def sort(self, results: List[SearchResult]) -> List[SearchResult]: ... - - -class RankSortStrategy: - def sort(self, results: List[SearchResult]) -> List[SearchResult]: - return sorted(results, key=lambda x: x.rank, reverse=True) - - -class AlphabeticalSortStrategy: - def sort(self, results: List[SearchResult]) -> List[SearchResult]: - return sorted(results, key=lambda x: x.title.lower()) - - -class ModelTypeSortStrategy: - def sort(self, results: List[SearchResult]) -> List[SearchResult]: - return sorted(results, key=lambda x: (x.model_type, -x.rank)) + from sqlalchemy.orm import Query, Session class SearchResultCollection: - term: str - results: List[SearchResult] - sorting_strategy: SortingStrategy = AlphabeticalSortStrategy() + web_search: str + results: list def __init__(self, term: str, session: 'Session', language='de_CH'): - lang = locales[language] + self.lang = locales[language] self.session = session - self.term = func.websearch_to_tsquery(lang, term) + self.web_search = term self.results = [] - self.sorting_strategy = AlphabeticalSortStrategy() - - def query(self) -> 'Query[SwissVote]': - - query = self.session.query(Consultation) - query = query.filter(or_(*self.term_filter)) - return query - - @property - def term_filter(self) -> list['ColumnElement[bool]']: - """ Returns a list of SqlAlchemy filter statements based on the search - term. - """ - - return self.term_filter_text - - def __post_init__(self): - query = self.query() - # create : List[SearchResult] from the qeury - - - self.results = self.perform_() - - def sort_results(self): - self.results = self.sorting_strategy.sort(self.results) - - def set_sorting_strategy(self, strategy: SortingStrategy): - self.sorting_strategy = strategy - - @property - def term_filter_text(self) -> list['ColumnElement[bool]']: + def do_search(self) -> None: + """ Main interface for getting the search results from templates""" + ts_query = func.websearch_to_tsquery(self.lang, self.web_search) + + consultation_query = select( + Consultation.id, + Consultation.title.label('title'), + Consultation.description.label('description'), + func.ts_headline(self.lang, Consultation.description, ts_query).label('headline'), + func.ts_rank_cd(func.to_tsvector(self.lang, Consultation.title + ' ' + Consultation.description), ts_query).label('rank'), + cast(literal("'consultation'"), String).label('type') + ).filter(or_(*self.term_filter_text_for_model(Consultation, self.lang))) + + meeting_query = select( + Meeting.id, + Meeting.name.label('title'), + Meeting.decisions.label('description'), + func.ts_headline(self.lang, Meeting.decisions, ts_query).label('headline'), + func.ts_rank_cd(func.to_tsvector(self.lang, Meeting.name + ' ' + Meeting.decisions), ts_query).label('rank'), + cast(literal("'meeting'"), String).label('type') + ).filter(or_(*self.term_filter_text_for_model(Meeting, self.lang))) + + # comment_query = select( + # Comment.id, + # Comment.content.label('title'), + # Comment.content.label('description'), + # func.ts_headline(self.lang, Comment.content, ts_query).label('headline'), + # func.ts_rank_cd(func.to_tsvector(self.lang, Comment.content), ts_query).label('rank'), + # func.literal('Comment').label('type') + # ).filter(or_(*self.term_filter_text_for_model(Comment, self.lang))) + + union_query = union_all(consultation_query, meeting_query).order_by( + text('rank DESC') + ) + self.results = list(self.session.execute(union_query).all()) + + def term_filter_text_for_model(self, model, language: str) -> list['ColumnElement[bool]']: """ Returns a list of SqlAlchemy filter statements matching possible - fulltext attributes based on the term. - - """ - - if not self.term: - return [] - + fulltext attributes based on the term for a given model.""" def match( column: 'ColumnElement[str] | ColumnElement[str | None]', language: str ) -> 'ColumnElement[bool]': - return column.op('@@')(func.to_tsquery(language, self.term)) + return column.op('@@')(func.websearch_to_tsquery(language, self.web_search)) def match_convert( column: 'ColumnElement[str] | ColumnElement[str | None]', @@ -103,10 +76,23 @@ def match_convert( ) -> 'ColumnElement[bool]': return match(func.to_tsvector(language, column), language) - return [ - match(Consultation.searchable_text_de_CH, 'german'), - ] - + if model == Consultation: + return [ + match_convert(Consultation.title, language), + match_convert(Consultation.description, language), + ] + elif model == Meeting: + return [ + match_convert(Meeting.name, language), + match_convert(Meeting.decisions, language), + ] + elif model == Comment: + return [ + match_convert(Comment.content, language), + ] + else: + return [] + def __len__(self): return len(self.results) @@ -116,6 +102,10 @@ def __iter__(self): def __getitem__(self, index): return self.results[index] + def __repr__(self) -> str: + # diplay the self.results, the first 4 + return f'' + def search(request: 'IRequest'): session = request.dbsession @@ -127,6 +117,8 @@ def search(request: 'IRequest'): stmt = select(Consultation.searchable_text_de_CH) assert None not in session.execute(stmt).scalars().all() - result_collection = SearchResultCollection(query, session) + result_collection = SearchResultCollection(term=query, session=session) + result_collection.do_search() + breakpoint() return {'search_results': result_collection} diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index 73e89b2..610b265 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -5,48 +5,17 @@ i18n:domain="privatim"> -
    -
    -
    -

    Search Results

    -

    Showing - results for ""

    -
    +
    +

    Search Results

    + +
    +
    +
    Title
    +

    Description

    + Read More +
    -
    - -
    -
    - -
    -
    -
    -
    - -
    - -
    -

    -
    - Relevance Score: - View Details -
    -
    -
    -     - - - -
    -
    +
    diff --git a/tests/models/test_searchable_mixin.py b/tests/models/test_searchable_mixin.py index 366c075..aebd76c 100644 --- a/tests/models/test_searchable_mixin.py +++ b/tests/models/test_searchable_mixin.py @@ -15,7 +15,7 @@ def test_fulltext_indexing_on_searchable_fields(pg_config): session.add(consultation) session.flush() - reindex_full_text_search(session, transaction.manager) + reindex_full_text_search(session) updated = session.execute( select(Consultation) From 21d7c3b5695f648feef8e596ccaf1e93868540f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 13:07:49 +0200 Subject: [PATCH 05/52] update --- src/privatim/models/comment.py | 8 +- src/privatim/models/consultation.py | 6 +- src/privatim/models/meeting.py | 12 +- src/privatim/models/searchable.py | 33 ++-- src/privatim/static/css/custom.css | 12 ++ src/privatim/views/search.py | 153 ++++++++++-------- .../views/templates/search_results.pt | 40 +++-- 7 files changed, 165 insertions(+), 99 deletions(-) diff --git a/src/privatim/models/comment.py b/src/privatim/models/comment.py index 97c2ff5..78dfa7e 100644 --- a/src/privatim/models/comment.py +++ b/src/privatim/models/comment.py @@ -2,6 +2,7 @@ from sedate import utcnow from privatim.orm import Base from privatim.orm.associable import Associable +from privatim.models import SearchableMixin from sqlalchemy.orm import relationship, Mapped, mapped_column, foreign, remote from privatim.orm.meta import UUIDStrPK, UUIDStr from sqlalchemy import Text, ForeignKey, Index, and_ @@ -12,7 +13,7 @@ from privatim.models import User -class Comment(Base, Associable): +class Comment(Base, Associable, SearchableMixin): """A generic comment that shall be attachable to any model. Meant to be used in conjunction with `Commentable`. @@ -72,6 +73,11 @@ class YourModel(Base, Commentable): def __repr__(self) -> str: return f'' + @classmethod + def searchable_fields(cls): + yield cls.content + yield cls.content + __table_args__ = ( Index('ix_comments_parent_id', 'parent_id'), Index('ix_comments_user_id', 'user_id'), diff --git a/src/privatim/models/consultation.py b/src/privatim/models/consultation.py index ed97c8f..adfa5d1 100644 --- a/src/privatim/models/consultation.py +++ b/src/privatim/models/consultation.py @@ -91,9 +91,9 @@ class Consultation(Base, Commentable, AssociatedFiles, SearchableMixin): @classmethod def searchable_fields(cls) -> Iterator[str]: - yield 'title' - yield 'description' - yield 'recommendation' + yield cls.title + yield cls.description + yield cls.recommendation def __repr__(self) -> str: return ( diff --git a/src/privatim/models/meeting.py b/src/privatim/models/meeting.py index b56a781..fa6bd33 100644 --- a/src/privatim/models/meeting.py +++ b/src/privatim/models/meeting.py @@ -7,12 +7,14 @@ from pyramid.authorization import Authenticated from privatim.models.commentable import Commentable +from privatim.models import SearchableMixin from privatim.orm.uuid_type import UUIDStr from privatim.orm import Base from privatim.orm.meta import UUIDStrPK, DateTimeWithTz -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Iterator + if TYPE_CHECKING: from privatim.models import User, WorkingGroup from datetime import datetime @@ -119,7 +121,7 @@ def __repr__(self) -> str: return f'' -class Meeting(Base, Commentable): +class Meeting(Base, Commentable, SearchableMixin): """Sitzung""" __tablename__ = 'meetings' @@ -170,6 +172,12 @@ def __init__( 'WorkingGroup', back_populates='meetings' ) + @classmethod + def searchable_fields(cls) -> Iterator[str]: + # todo: agenda item (seperately) + yield cls.name + yield cls.name + def __acl__(self) -> list['ACL']: return [ (Allow, Authenticated, ['view']), diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index ceeb3d8..87c1d0f 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -25,18 +25,11 @@ def searchable_fields(cls) -> Iterator[str]: @hybrid_property def searchable_text(self) -> str: + # todo: extract document text return ' '.join( str(getattr(self, field)) for field in self.searchable_fields() ) - @searchable_text.expression - def searchable_text(cls): - # Called when the property is used in a SQL expression - return func.concat_ws( - ' ', - *[getattr(cls, field) for field in cls.searchable_fields()] - ) - def searchable_models() -> tuple[type[Base], ...]: model_classes = set() @@ -44,10 +37,10 @@ def searchable_models() -> tuple[type[Base], ...]: for mapper in Base.registry.mappers: cls = mapper.class_ if ( - inspect.isclass(cls) - and issubclass(cls, SearchableMixin) - and issubclass(cls, Base) - and cls != SearchableMixin + inspect.isclass(cls) + and issubclass(cls, SearchableMixin) + and issubclass(cls, Base) + and cls != SearchableMixin ): model_classes.add(cls) return tuple(model_classes) @@ -76,17 +69,17 @@ def reindex_full_text_search(session: Session) -> None: for locale, language in locales.items(): assert language == 'german' # todo: remove later if hasattr(model, f'searchable_text_{locale}'): - update_stmt = ( - update(model) - .values({ + update_stmt = update(model).values( + { f'searchable_text_{locale}': func.to_tsvector( - language, - func.cast(model.searchable_text, Text) + language, func.cast(model.searchable_text, Text) ) - }) + } ) updated = getattr(model, f'searchable_text_{locale}') session.execute(update_stmt) - print('Reindex full text search for', - f'{model}.searchable_text_{locale} with val {updated}') + print( + 'Reindex full text search for', + f'{model}.searchable_text_{locale} with val {updated}', + ) session.flush() diff --git a/src/privatim/static/css/custom.css b/src/privatim/static/css/custom.css index afb5c0e..bd96274 100644 --- a/src/privatim/static/css/custom.css +++ b/src/privatim/static/css/custom.css @@ -141,9 +141,11 @@ a:hover { .bd-mode-toggle .dropdown-menu .active .bi { display: block !important; } + #search-button { border: none; } + #search-button:hover { background-color: transparent; color: var(--primary-color); @@ -286,3 +288,13 @@ span.accordion-text-9 { font-weight: bold; margin-right: 10px; } + + +/* Start search results */ +mark { + background-color: yellow; + color: black; + font-weight: bold; + padding: 0 5px; + border-radius: 3px; +} \ No newline at end of file diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index c30cbb5..ed3f35f 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -1,21 +1,50 @@ -from sqlalchemy import func, select, ColumnElement, union_all, text, cast, \ - literal, String +from sqlalchemy import ( + func, + select, + ColumnElement, + union_all, + cast, + literal, + String, +) from privatim.forms.search_form import SearchForm from privatim.models import Consultation, Meeting from privatim.i18n import locales from sqlalchemy import or_ from privatim.models.comment import Comment -from privatim.models.searchable import searchable_models -from typing import TYPE_CHECKING, List, NamedTuple, Protocol +from typing import TYPE_CHECKING, List, NamedTuple + if TYPE_CHECKING: from pyramid.interfaces import IRequest - from sqlalchemy.orm import Query, Session + from sqlalchemy.orm import Session, InstrumentedAttribute + from sqlalchemy.engine.row import Row + + +class SearchResult(NamedTuple): + id: int + title: str + headline: str + type: str + +class SearchCollection: + """ A class for searching the database for a given term. -class SearchResultCollection: + We use websearch_to_tsquery, which automatically converts the search term + into a tsquery. + + | term | tsquery | + |------|---------| + | the donkey | |'donkey' | + | "blue donkey" | 'blue' & 'donkey' | + + See also: + https://adamj.eu/tech/2024/01/03/postgresql-full-text-search-websearch/ + + """ web_search: str results: list @@ -26,73 +55,69 @@ def __init__(self, term: str, session: 'Session', language='de_CH'): self.results = [] def do_search(self) -> None: - """ Main interface for getting the search results from templates""" + """Main interface for getting the search results.""" ts_query = func.websearch_to_tsquery(self.lang, self.web_search) - consultation_query = select( - Consultation.id, - Consultation.title.label('title'), - Consultation.description.label('description'), - func.ts_headline(self.lang, Consultation.description, ts_query).label('headline'), - func.ts_rank_cd(func.to_tsvector(self.lang, Consultation.title + ' ' + Consultation.description), ts_query).label('rank'), - cast(literal("'consultation'"), String).label('type') - ).filter(or_(*self.term_filter_text_for_model(Consultation, self.lang))) - - meeting_query = select( - Meeting.id, - Meeting.name.label('title'), - Meeting.decisions.label('description'), - func.ts_headline(self.lang, Meeting.decisions, ts_query).label('headline'), - func.ts_rank_cd(func.to_tsvector(self.lang, Meeting.name + ' ' + Meeting.decisions), ts_query).label('rank'), - cast(literal("'meeting'"), String).label('type') - ).filter(or_(*self.term_filter_text_for_model(Meeting, self.lang))) - - # comment_query = select( - # Comment.id, - # Comment.content.label('title'), - # Comment.content.label('description'), - # func.ts_headline(self.lang, Comment.content, ts_query).label('headline'), - # func.ts_rank_cd(func.to_tsvector(self.lang, Comment.content), ts_query).label('rank'), - # func.literal('Comment').label('type') - # ).filter(or_(*self.term_filter_text_for_model(Comment, self.lang))) - - union_query = union_all(consultation_query, meeting_query).order_by( - text('rank DESC') + consultation_query = self.build_query( + Consultation, ts_query, 'Consultation' + ) + meeting_query = self.build_query(Meeting, ts_query, 'Meeting') + comment_query = self.build_query(Comment, ts_query, 'Comment') + + union_query = union_all( + consultation_query, meeting_query, comment_query + ) + + results = self.session.execute(union_query).all() + self.results = [SearchResult(*result) for result in results] + breakpoint() + + def build_query(self, model, ts_query, model_name: str): + search_fields: list[InstrumentedAttribute] = list( + model.searchable_fields() ) - self.results = list(self.session.execute(union_query).all()) - def term_filter_text_for_model(self, model, language: str) -> list['ColumnElement[bool]']: - """ Returns a list of SqlAlchemy filter statements matching possible + headline_expr = func.ts_headline( + self.lang, + search_fields[0], # using the first searchable field as the + # headline + ts_query, + 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ShortWord=3, HighlightAll=FALSE, MaxFragments=3, FragmentDelimiter=" ... "', + ).label('headline') + + return select( + model.id, # include the model itself in the search results + search_fields[0].label('title'), # Using the first searchable + # field for the title + headline_expr, + cast(literal(model_name), String).label('type'), + ).filter(or_(*self.term_filter_text_for_model(model, self.lang))) + + def term_filter_text_for_model( + self, model, language: str + ) -> list['ColumnElement[bool]']: + """Returns a list of SqlAlchemy filter statements matching possible fulltext attributes based on the term for a given model.""" + def match( - column: 'ColumnElement[str] | ColumnElement[str | None]', - language: str + column: 'ColumnElement[str] | ColumnElement[str | None]', + language: str, ) -> 'ColumnElement[bool]': - return column.op('@@')(func.websearch_to_tsquery(language, self.web_search)) + return column.op('@@')( + func.websearch_to_tsquery(language, self.web_search) + ) def match_convert( - column: 'ColumnElement[str] | ColumnElement[str | None]', - language: str + column: 'ColumnElement[str] | ColumnElement[str | None]', + language: str, ) -> 'ColumnElement[bool]': return match(func.to_tsvector(language, column), language) - if model == Consultation: - return [ - match_convert(Consultation.title, language), - match_convert(Consultation.description, language), - ] - elif model == Meeting: - return [ - match_convert(Meeting.name, language), - match_convert(Meeting.decisions, language), - ] - elif model == Comment: - return [ - match_convert(Comment.content, language), - ] - else: - return [] - + return [ + match_convert(field, language) + for field in model.searchable_fields() + ] + def __len__(self): return len(self.results) @@ -117,8 +142,8 @@ def search(request: 'IRequest'): stmt = select(Consultation.searchable_text_de_CH) assert None not in session.execute(stmt).scalars().all() - result_collection = SearchResultCollection(term=query, session=session) + result_collection = SearchCollection(term=query, session=session) result_collection.do_search() - breakpoint() - return {'search_results': result_collection} + activities = result_collection.results + return {'activities': activities} diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index 610b265..d3baf49 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -5,17 +5,39 @@ i18n:domain="privatim"> -
    -

    Search Results

    - -
    -
    -
    Title
    -

    Description

    - Read More +
    +
    +
    +

    Search Results

    +
    +
    + +
    + +
    + +
    + +
    +
    +
    ${result.title}
    +

    +
    + Show Details +
    + +
    - + + + + + +
    From f52f3f617e697a15cc7a4fa2b8ca825468d2d1e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 13:35:45 +0200 Subject: [PATCH 06/52] works sort of --- src/privatim/views/search.py | 103 +++++++++++------- .../views/templates/search_results.pt | 5 +- 2 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index ed3f35f..ece2b54 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -15,7 +15,7 @@ from privatim.models.comment import Comment -from typing import TYPE_CHECKING, List, NamedTuple +from typing import TYPE_CHECKING, List, NamedTuple, TypeVar if TYPE_CHECKING: from pyramid.interfaces import IRequest @@ -30,6 +30,8 @@ class SearchResult(NamedTuple): type: str +Model = TypeVar('Model') + class SearchCollection: """ A class for searching the database for a given term. @@ -57,65 +59,84 @@ def __init__(self, term: str, session: 'Session', language='de_CH'): def do_search(self) -> None: """Main interface for getting the search results.""" ts_query = func.websearch_to_tsquery(self.lang, self.web_search) - - consultation_query = self.build_query( - Consultation, ts_query, 'Consultation' - ) - meeting_query = self.build_query(Meeting, ts_query, 'Meeting') - comment_query = self.build_query(Comment, ts_query, 'Comment') - + consultation_query = self.build_query(Consultation, ts_query) + meeting_query = self.build_query(Meeting, ts_query) + comment_query = self.build_query(Comment, ts_query) union_query = union_all( - consultation_query, meeting_query, comment_query + consultation_query, + meeting_query, + comment_query, ) - - results = self.session.execute(union_query).all() - self.results = [SearchResult(*result) for result in results] - breakpoint() - - def build_query(self, model, ts_query, model_name: str): - search_fields: list[InstrumentedAttribute] = list( + raw_results = self.session.execute(union_query).all() + self.results = self.process_results(raw_results) + + def process_results(self, raw_results) -> List[SearchResult]: + processed_results = [] + for result in raw_results: + try: + processed_results.append( + SearchResult( + id=result.id, + title=result.title, + headline=result.headline, + type=result.type, + ) + ) + except AttributeError as e: + print(f"Error processing result: {e}") + return processed_results + + def build_query(self, model: Model, ts_query): + search_fields: List[InstrumentedAttribute] = list( model.searchable_fields() ) - - headline_expr = func.ts_headline( - self.lang, - search_fields[0], # using the first searchable field as the - # headline - ts_query, - 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ShortWord=3, HighlightAll=FALSE, MaxFragments=3, FragmentDelimiter=" ... "', - ).label('headline') + headline_expr = self.generate_headline(model, ts_query) return select( - model.id, # include the model itself in the search results - search_fields[0].label('title'), # Using the first searchable - # field for the title + model.id, + search_fields[0].label( + 'title' + ), # Using the first searchable field for the title headline_expr, - cast(literal(model_name), String).label('type'), + cast(literal(model.__name__), String).label('type'), ).filter(or_(*self.term_filter_text_for_model(model, self.lang))) - def term_filter_text_for_model( - self, model, language: str - ) -> list['ColumnElement[bool]']: - """Returns a list of SqlAlchemy filter statements matching possible - fulltext attributes based on the term for a given model.""" + def generate_headline(self, model: Model, ts_query): + """The generate_headline method concatenates all searchable + fields into a single string, separated by ' | '. This ensures that all + searchable fields are included in the headline.""" + + search_fields = model.searchable_fields() + headline_fields = [func.coalesce(field, '') for field in search_fields] + concatenated_fields = func.concat_ws(' | ', *headline_fields) + + # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE + return func.ts_headline( + self.lang, + concatenated_fields, + ts_query, + 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ' + 'ShortWord=3, HighlightAll=TRUE, MaxFragments=3, ' + 'FragmentDelimiter=" ... "', + ).label('headline') + def term_filter_text_for_model( + self, model: Model, lang: str + ) -> List[ColumnElement[bool]]: def match( - column: 'ColumnElement[str] | ColumnElement[str | None]', - language: str, - ) -> 'ColumnElement[bool]': + column: ColumnElement[str], language: str + ) -> ColumnElement[bool]: return column.op('@@')( func.websearch_to_tsquery(language, self.web_search) ) def match_convert( - column: 'ColumnElement[str] | ColumnElement[str | None]', - language: str, - ) -> 'ColumnElement[bool]': + column: ColumnElement[str], language: str + ) -> ColumnElement[bool]: return match(func.to_tsvector(language, column), language) return [ - match_convert(field, language) - for field in model.searchable_fields() + match_convert(field, lang) for field in model.searchable_fields() ] def __len__(self): diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index d3baf49..0c96e0b 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -16,13 +16,12 @@
    +
    -
    ${result.title}
    -

    -
    +
    Show Details
    From f459eed4d61bc2102c745b9656d054a683be0b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 13:56:51 +0200 Subject: [PATCH 07/52] Use all the fields in the headline --- src/privatim/models/__init__.py | 2 + src/privatim/views/search.py | 126 +++++++++--------- .../views/templates/search_results.pt | 11 +- 3 files changed, 69 insertions(+), 70 deletions(-) diff --git a/src/privatim/models/__init__.py b/src/privatim/models/__init__.py index 4424bd0..44ebc02 100644 --- a/src/privatim/models/__init__.py +++ b/src/privatim/models/__init__.py @@ -53,6 +53,8 @@ class SearchableBase(Base, SearchableMixin): def update_searchable_text_listener( mapper: Mapper['SearchableBase'], connection: 'Connection', target: Any ) -> None: + return + # we will implement this for updates of document text if connection.engine.name != 'postgresql': # todo: fallback if local development uses sqlite? raise ValueError('Only PostgreSQL is supported') diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index ece2b54..bf967fa 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -20,20 +20,20 @@ if TYPE_CHECKING: from pyramid.interfaces import IRequest from sqlalchemy.orm import Session, InstrumentedAttribute - from sqlalchemy.engine.row import Row class SearchResult(NamedTuple): id: int title: str - headline: str + headlines: dict[str, str] type: str Model = TypeVar('Model') + class SearchCollection: - """ A class for searching the database for a given term. + """A class for searching the database for a given term. We use websearch_to_tsquery, which automatically converts the search term into a tsquery. @@ -47,81 +47,57 @@ class SearchCollection: https://adamj.eu/tech/2024/01/03/postgresql-full-text-search-websearch/ """ - web_search: str - results: list def __init__(self, term: str, session: 'Session', language='de_CH'): self.lang = locales[language] self.session = session self.web_search = term - self.results = [] + self.results: List[SearchResult] = [] + self.models = [Consultation, Meeting, Comment] def do_search(self) -> None: - """Main interface for getting the search results.""" ts_query = func.websearch_to_tsquery(self.lang, self.web_search) - consultation_query = self.build_query(Consultation, ts_query) - meeting_query = self.build_query(Meeting, ts_query) - comment_query = self.build_query(Comment, ts_query) - union_query = union_all( - consultation_query, - meeting_query, - comment_query, - ) - raw_results = self.session.execute(union_query).all() - self.results = self.process_results(raw_results) + for model in self.models: + self.results.extend(self.search_model(model, ts_query)) - def process_results(self, raw_results) -> List[SearchResult]: - processed_results = [] - for result in raw_results: - try: - processed_results.append( - SearchResult( - id=result.id, - title=result.title, - headline=result.headline, - type=result.type, - ) - ) - except AttributeError as e: - print(f"Error processing result: {e}") - return processed_results + def search_model(self, model: type[Model], ts_query) -> List[SearchResult]: + query = self.build_query(model, ts_query) + raw_results = self.session.execute(query).all() + return self.process_results(raw_results, model) - def build_query(self, model: Model, ts_query): + def build_query(self, model: type[Model], ts_query): search_fields: List[InstrumentedAttribute] = list( model.searchable_fields() ) - headline_expr = self.generate_headline(model, ts_query) + headline_exprs = self.generate_headlines(model, ts_query) - return select( + select_fields = [ model.id, - search_fields[0].label( - 'title' - ), # Using the first searchable field for the title - headline_expr, + search_fields[0].label('title'), + *headline_exprs, cast(literal(model.__name__), String).label('type'), - ).filter(or_(*self.term_filter_text_for_model(model, self.lang))) - - def generate_headline(self, model: Model, ts_query): - """The generate_headline method concatenates all searchable - fields into a single string, separated by ' | '. This ensures that all - searchable fields are included in the headline.""" - - search_fields = model.searchable_fields() - headline_fields = [func.coalesce(field, '') for field in search_fields] - concatenated_fields = func.concat_ws(' | ', *headline_fields) - - # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-HEADLINE - return func.ts_headline( - self.lang, - concatenated_fields, - ts_query, - 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ' - 'ShortWord=3, HighlightAll=TRUE, MaxFragments=3, ' - 'FragmentDelimiter=" ... "', - ).label('headline') + ] + + return select(*select_fields).filter( + or_(*self.term_filter_text_for_model(model, self.lang)) + ) + + def generate_headlines(self, model: type[Model], ts_query): + search_fields = list(model.searchable_fields()) + return [ + func.ts_headline( + self.lang, + field, + ts_query, + 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ShortWord=3, HighlightAll=FALSE, MaxFragments=3, FragmentDelimiter=" ... "', + ).label(field.name) + for field in search_fields[ + 1: + ] # Skip the first field as it's used for the title + ] def term_filter_text_for_model( - self, model: Model, lang: str + self, model: type[Model], language: str ) -> List[ColumnElement[bool]]: def match( column: ColumnElement[str], language: str @@ -136,9 +112,32 @@ def match_convert( return match(func.to_tsvector(language, column), language) return [ - match_convert(field, lang) for field in model.searchable_fields() + match_convert(field, language) + for field in model.searchable_fields() ] + def process_results( + self, raw_results, model: type[Model] + ) -> List[SearchResult]: + searchable = list(model.searchable_fields()) + processed_results = [] + field_names = [field.name for field in searchable[1:]] + for result in raw_results: + headlines = { + field_name: getattr(result, field_name) + for field_name in field_names + if getattr(result, field_name) + } + processed_results.append( + SearchResult( + id=result.id, + title=result.title, + headlines=headlines, + type=result.type, + ) + ) + return processed_results + def __len__(self): return len(self.results) @@ -149,7 +148,6 @@ def __getitem__(self, index): return self.results[index] def __repr__(self) -> str: - # diplay the self.results, the first 4 return f'' @@ -159,10 +157,6 @@ def search(request: 'IRequest'): if request.method == 'POST' and form.validate(): query = request.POST['search'] - # todo:remove later - stmt = select(Consultation.searchable_text_de_CH) - assert None not in session.execute(stmt).scalars().all() - result_collection = SearchCollection(term=query, session=session) result_collection.do_search() diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index 0c96e0b..b3a87ff 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -21,13 +21,16 @@
    ${result.title}
    -
    - Show Details +
    + ${item[0]}: + +
    + Show Details
    -
    + +
    From 3a13bb170b09e78f668927ff9fa776e9e03b6581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 14:18:44 +0200 Subject: [PATCH 08/52] update --- src/privatim/forms/search_form.py | 2 +- src/privatim/layouts/macros.pt | 2 +- .../locale/de/LC_MESSAGES/privatim.mo | Bin 9015 -> 8979 bytes .../locale/de/LC_MESSAGES/privatim.po | 33 +++++-- .../locale/fr/LC_MESSAGES/privatim.mo | Bin 9255 -> 9271 bytes .../locale/fr/LC_MESSAGES/privatim.po | 33 +++++-- src/privatim/locale/privatim.pot | 28 ++++-- src/privatim/models/comment.py | 1 - src/privatim/models/meeting.py | 1 - src/privatim/views/search.py | 82 +++++++++++++----- .../views/templates/search_results.pt | 24 ++--- 11 files changed, 144 insertions(+), 62 deletions(-) diff --git a/src/privatim/forms/search_form.py b/src/privatim/forms/search_form.py index a096f51..03b1f54 100644 --- a/src/privatim/forms/search_form.py +++ b/src/privatim/forms/search_form.py @@ -22,7 +22,7 @@ def __init__( 'dbsession': session } ) - search = SearchField( + term = SearchField( _('Search'), [DataRequired()], render_kw={ diff --git a/src/privatim/layouts/macros.pt b/src/privatim/layouts/macros.pt index 96b2a99..1b61992 100644 --- a/src/privatim/layouts/macros.pt +++ b/src/privatim/layouts/macros.pt @@ -161,7 +161,7 @@
    diff --git a/src/privatim/locale/de/LC_MESSAGES/privatim.mo b/src/privatim/locale/de/LC_MESSAGES/privatim.mo index 3a2074dd28579cf18d71ecfb451077178bbe45d6..f8ea01e7f0dd61f0eefff1325dd6b3d359f44562 100644 GIT binary patch delta 2637 zcmXZc4NR3)9LMoL!bR={R9>u%00R*SEwE87!`0>}OM`5&)*>Lwl$Ur(QVh6Rfp@JV-h)c;0MeHYq3#=T z=f_a(zej#{hCj8th)Qf06Byrotcy-0qdLq$byR>#=yp^GC8(LLN3B#P>i$Md#&%cl zLrtI`ug3wr1COE}%@tH)a~M=1ie8mK99CnhtGA+-ZkMZfqY~POYS)kSVeg|74mn3r z1AXJ_-#JgC`kO{Qstb!)f6d?zPN?HqXEf8(QYE4~%0e|PK;2)6nsEuz#LDq?tU)FA zH>&*%DzSKOR-!4WO`eTf(R-6we_bf!ggUN54OH*y&8QALQ3-ar^L@^K)Bs1^_0Lc< z8%4GI7V~fd!*2`bQ%_1UTZXHH6mFqV?R?E$_yU)6ehOFMJf@@jZo`}L3CzKER7V52 z0Z*cy`C?v;8*nA6UW!WeDO6&cQ4Id5-Ssil zQcgP0p;q!wRJ%D;yMM6&eSAcYM41&KdoE~GjHV^~9hKo#)Kbr3A;vHbbx@4D?;g~i zxF0o8IqJhyiL9<|Lz>ziWby2vGvuzHMD_Cv9@6`NiGl{~=Ch`b-awMI!>9p{qgLV+ zYP0=}dRs1|mhc*e6Gm;q7#_4HvJZ9tyQuaOjJj?s1+ziU0>rYL$#|$-PeekK!>aMqQ-d}mGB|d$IL#Wpc#)KldwtDQeJlc z<6OwQqeqd2+AH~3iK|fwy@Ohj0n~kmQ7iQ^-ib$1<6TAF7nSbb|Jd}%g%k`AfO>ZM zu3m!5d@U;T^{Az+aP=Bghjp&rgc@k4tM{N1@lbEiK~%zH>8!sBr`!!cq4vOe)C$aC zANn)g3{j700F~%*R3hWZ&n8{{FVujus6>33k;GF`k0b-tz92|J9jrn<%M$0qsKlyJ zGv0!_uLaex18>FMn1vxM!n3GE;+96f7r9tKeIqW%PRzv*oxuqTD>!itbwd{GlZWe& zQ`Ulypocs*yM!8W9)G+zmVP*B4` z)HD19)!{H|1&+D;3DmBhKz&%IQ0>m6Ch!|Z;X2|OLdoV3yNJgLO|FV)A$0T*+ceTr z3Y$EizbvPhl8(C~ST(j2nuoTdjvZc`KP_sjx8I-Zdz$(y-kAR(-*c4qcu9fW^iE23 zgpL}5@3Fm1=-BN&7RXL)j8ww^-CkE9Tgzj`L=*8U@sc+ZNK4;KX%*2y=n1?)=y;OY z>0Jq=r+${Xgw``t1M! delta 2671 zcmXZce@vBC9LMouVDd|Y@B(`I8FUj{WwSqOB#nrXa~d^Eexw100)ZN87g8b> zb@e1=|M(+q(uggo)M9H++d!@5kkMvi;BbwN{-8;#)}r_4-t+8!pVv9h^PF?O=RD`R zPVt4};CRAeDB%rK`VIEpDagAZV2tTDN`3?uP5RMWK> zg)gHQt8o!Fp!(a3amEDA0ej&SjOWH@wtf-$nX8=6;Z4*4Z41MRbl_s@AEO8RQ3G8> zIyP5O1KmRAWF}DU@8VXR!6?Qz759ZRt3-9wh!NO^YS50FNteBU6qBeQMjDN)9j!(sRD|lF3^lWjsFi9!Js-pb z?6&n2s0sAp{dgW9!)vHlE5^X9#lf7QSJJWKFt6s z;VahbsDVao{a5Q0s=pc3tD22t{WXJsxuA|cOiL!Bmdc0fXeFxQYSi;jp=Ml$G&PlY z7@JUu&7;~!@{kfsMJ4J-ZSqG@EBZnL>#rM|xS)==qXybz>m8^LkDwAeW<6o=_gcTf z&K67)x;$llZW|13YiqvsCWM`pGY!3i>gwE%m=6jXHdI)5Z!nU)$k|O z^P~3uZPb!ZTmM3>XcT`MYL|p+myG$Cj=e74e<=lRzBxwIlDSA%8OEZPJPB7|8mfbG z)N>n9d!q_9P(9L>*@>*OIf9zNDP-}?CF_s&{v>+!{okW-k_YBd1D@pXO&xuXOu}47 z4KRjUi7C{k`vaA@n-{7TT!f*7QJXOhHIY8l^IxIb|A3mni0b`DWx2YaZ%VJc-&XMfeJ?MowjP}x9xxG=NIEL<0P2;j3Q|x9YwV5Z zF|@l;o2JUzj2d_!s)KISb3Lf`r!au$a5;`)A^w9(s30T!zfp?$)OTPu22WGSp)hQn z#RBRntdAZj#$0SbZkZljix-gBXgrLj0kb*r0htoiE2>5%R*(9JZ9@HDbfET37qY#A z<~Ri{@zn|Vg6i-W)Cx@4`ZQ{H&!XDRquROs;R$%rMZJXBOeonb;vJ%b z(BwAhAL304It~)8LIVRbfme?jO;|gLYq3y13Zad*7bhJCZ7=KC|rDq6z@p;5n zLVJ3f^HNM|;TB4IZyShpL>;k<(D81V&^eQH1F>V^Y|Nd=sH*0g+CU&Mus@+c*0Zj) Orgrl-RSVPaxc>*qAOP?H diff --git a/src/privatim/locale/de/LC_MESSAGES/privatim.po b/src/privatim/locale/de/LC_MESSAGES/privatim.po index e942b00..f7dd1a5 100644 --- a/src/privatim/locale/de/LC_MESSAGES/privatim.po +++ b/src/privatim/locale/de/LC_MESSAGES/privatim.po @@ -5,7 +5,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2024-06-23 05:02+0200\n" +"POT-Creation-Date: 2024-07-02 14:17+0200\n" "PO-Revision-Date: 2024-05-21 21:20+0200\n" "Last-Translator: cyrill \n" "Language-Team: German \n" @@ -53,8 +53,7 @@ msgstr "Nachricht" msgid "Answer" msgstr "Antworten" -#: src/privatim/layouts/navbar.pt src/privatim/layouts/navbar.py -#: src/privatim/views/activities.py +#: src/privatim/layouts/navbar.pt src/privatim/views/activities.py msgid "Activities" msgstr "Aktivitäten" @@ -73,10 +72,6 @@ msgstr "Vernehmlassungen" msgid "Working Groups" msgstr "Gremien" -#: src/privatim/layouts/navbar.pt -msgid "Search..." -msgstr "Suchen..." - #: src/privatim/layouts/navbar.pt msgid " Profile" msgstr " Profil" @@ -339,13 +334,26 @@ msgstr "Status" msgid "Your Comment" msgstr "Ihr Kommentar" +#: src/privatim/views/templates/search_results.pt +msgid "Search Results" +msgstr "Suchergebnisse" + +#: src/privatim/views/templates/search_results.pt +#: src/privatim/views/templates/activities.pt +msgid "Show Details" +msgstr "Details anzeigen" + #: src/privatim/views/templates/password_change.pt msgid "Change Password" msgstr "Passwort ändern" #: src/privatim/views/templates/activities.pt -msgid "Show Details" -msgstr "Details anzeigen" +msgid "Unbekannter Ersteller" +msgstr "Unbekannter Ersteller" + +#: src/privatim/views/templates/activities.pt +msgid "Kein Leiter" +msgstr "Kein Leiter" #: src/privatim/views/templates/activities.pt msgid "Show Meeting" @@ -600,6 +608,10 @@ msgstr "Kein Leiter" msgid "Members" msgstr "Mitglieder" +#: src/privatim/forms/search_form.py +msgid "Search" +msgstr "" + #: src/privatim/forms/widgets/widgets.py msgid "Uploaded file" msgstr "Hochgeladene Datei" @@ -629,5 +641,8 @@ msgstr "Auswählen" msgid "Protocol of meeting ${title}" msgstr "Protokoll der Sitzung ${title}" +#~ msgid "Search..." +#~ msgstr "Suchen..." + #~ msgid "Do you really wish to delete \"${item_title}\"?" #~ msgstr "Möchten Sie \"${item_title}\" wirklich löschen?" diff --git a/src/privatim/locale/fr/LC_MESSAGES/privatim.mo b/src/privatim/locale/fr/LC_MESSAGES/privatim.mo index bd2b26cd70f8fe5290a7e6d988390ea2f5641b0a..229aadaa51ba602d9e2f5d9c12d6e3a59b4a81d3 100644 GIT binary patch delta 1809 zcmXZceQ4EH9LMqR_2=g9ooU*Z?*2ur zUZ8x=Q><=FO1+U9^7#LimJ?CsE!X&9j7pkd2`J+gv^#>A=kak zro{T41E`6=!YVxL?%%>ExxSlrHzrY~et>$>W4VlJan#CR=ZosuyGYfo6ZO0YmGA&w z!|zaA*HvmZ9}hZ@qZaZbDxs^We(7-<%KQ&h=_XMtxQ|D0%02I4!^zE#p%VQDRf!=i zz#mZKjG+3Dp|*A$HSzzbiKmz6o);t6X)B|lj29zSwdav-w|eJh)QVG>EwQs7HSh^k z|C6W{oyQoCBE_(KsPSx`SrmNK^-9cVep^dJ6R$@WYwf5N^`KVLhgxaiuD`%Kt_M+D za1V9(rcf*NsJkX8LM2#<+PW7|B2lXE-ju%%F??nVuG*ggLoRf#WA z|86Hy3648|K}|e?&)_6#fwQ^z2P;sC`U|MPRve?D0cvqJ)?p=XM3r>E^C&8j0o40v zP>EbXt!xB!s((gJcpLTpBr2i2sM~wg!pfr5UjtQe<7KQt{;Z9!Rrm?&ci}oJp?~pJ zEahZXV}r8?^}}=uYjGSGVadYW{g+VhH)0d^p%(u4!gQ`g1)Lb&s6rj4*HDSAMP?P(M(AZls|Xr%)X|(yqY>hVTud zh2U&uD{&Yb2vuV%ks|c@l;|R&M04<8L37aoT1r_*PoKR(GF%=?2EE~`P*ZR^ye`xh zJPODCgFM@jeS`X-HtbMP8;OTHgJfi>va}V%ZsG{BKgdL4{>QZ5B0eDWd5`!g_$^XW z-I~2Ndmon(-GmOwi^ML%BlJlJ^9y68`)Jh>Dx!91JD~#a4Bjp*U!I_)1G$OVK(rHk z2z`2UWWOnvbr2s8e_i-{{&0WsJa6Xij?TuGL`$?a*^wCj-Y<&`-&io=`Ky!7?a4$c V+BKT#7|k@dH>TP~Gr>@-`!N;C#N7Y@ delta 1792 zcmXZcYix{J9LMqhWNeukx&~9NVn&N@O|@1cLZXqVu#t6-OCmP5ooZFPU36ipC5J4*b?fSDa0W;Pg)Iooj{?G)<%H4Ne{%)$rw89u^MXv58lAs@%%T%3d(oNccE z22P~^C6?ge5td93oH4>|9353S0b5ZMoyRY+3rAu!VwQ{3QSHU3L~BrqHR1ql#!qlN zhVduV_#^1Y^QiG1kz{7zRXT>y(TUl3&-nyZnOCR>`%n+|V;p@sX3KnLlQBelCvQUOKz(IH! zHBTq%`8%jB?Lxiy9qPrFpShonw3C*@g)$zGRMkq6?Y0W%YSfAwFkNEjLDa;@P|vrc zR@8xocpWK*y+F6mRs1>!KRG~t?&ctg+r*n5-dP%T^VY=IjH#;V1W1Aa(82`vl?~S8c?O)gqm=NyZ;NS630>h zX6>j1yPS_vFa8t9Vh?J8AsTOR94gUpl=^GMQ7$yW4Ez*l<3wDHDrvK`6_rREYW!JL zA{S9B>qKqEE!640k9zTQ)N_5P#6q#m_}m!v*9v2FXu>$o#Hq-i)$=t6521b+9-pLRI*R*i7iNpJ*atM3wjNpvv$Lu9T{do<84uwSj!!25(QG$XDrI2rTl| zc>f3Dk#D)TF+IX;9crV@n-z@v8ok-MQr!Z1PdcATrDBC5c;eke(?SZ z4liDxZgXmJ3eimHkW44OA^e0sKYC+Ag?Zb!svvYsv@>;t3fthV4&{Gd&6N)13St>i zOKc_d*^?pttv#+MzDu17{hc*%-KNT#($dn@?qN}X>S81}7+9L9tVz(DNZpS9\n" "Language-Team: French \n" @@ -50,8 +50,7 @@ msgstr "Message" msgid "Answer" msgstr "Répondre" -#: src/privatim/layouts/navbar.pt src/privatim/layouts/navbar.py -#: src/privatim/views/activities.py +#: src/privatim/layouts/navbar.pt src/privatim/views/activities.py msgid "Activities" msgstr "Activités" @@ -70,10 +69,6 @@ msgstr "Consultations" msgid "Working Groups" msgstr "Comités" -#: src/privatim/layouts/navbar.pt -msgid "Search..." -msgstr "Recherche" - #: src/privatim/layouts/navbar.pt msgid " Profile" msgstr " Profil" @@ -338,13 +333,26 @@ msgstr "Statut" msgid "Your Comment" msgstr "Votre commentaire" +#: src/privatim/views/templates/search_results.pt +msgid "Search Results" +msgstr "" + +#: src/privatim/views/templates/search_results.pt +#: src/privatim/views/templates/activities.pt +msgid "Show Details" +msgstr "Afficher les détails" + #: src/privatim/views/templates/password_change.pt msgid "Change Password" msgstr "Changer le mot de passe" #: src/privatim/views/templates/activities.pt -msgid "Show Details" -msgstr "Afficher les détails" +msgid "Unbekannter Ersteller" +msgstr "" + +#: src/privatim/views/templates/activities.pt +msgid "Kein Leiter" +msgstr "" #: src/privatim/views/templates/activities.pt msgid "Show Meeting" @@ -597,6 +605,10 @@ msgstr "Pas de chef" msgid "Members" msgstr "Membres" +#: src/privatim/forms/search_form.py +msgid "Search" +msgstr "" + #: src/privatim/forms/widgets/widgets.py msgid "Uploaded file" msgstr "Fichier téléchargé" @@ -626,5 +638,8 @@ msgstr "Sélectionner..." msgid "Protocol of meeting ${title}" msgstr "Procès-verbal de la réunion ${title}" +#~ msgid "Search..." +#~ msgstr "Recherche" + #~ msgid "Do you really wish to delete \"${item_title}\"?" #~ msgstr "Souhaitez-vous vraiment supprimer \"${item_title}\" ?" diff --git a/src/privatim/locale/privatim.pot b/src/privatim/locale/privatim.pot index ce85409..a7acb9c 100644 --- a/src/privatim/locale/privatim.pot +++ b/src/privatim/locale/privatim.pot @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2024-06-23 05:02+0200\n" +"POT-Creation-Date: 2024-07-02 14:17+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -50,8 +50,7 @@ msgstr "" msgid "Answer" msgstr "" -#: ./src/privatim/layouts/navbar.pt ./src/privatim/layouts/navbar.py -#: ./src/privatim/views/activities.py +#: ./src/privatim/layouts/navbar.pt ./src/privatim/views/activities.py msgid "Activities" msgstr "" @@ -70,10 +69,6 @@ msgstr "" msgid "Working Groups" msgstr "" -#: ./src/privatim/layouts/navbar.pt -msgid "Search..." -msgstr "" - #: ./src/privatim/layouts/navbar.pt msgid " Profile" msgstr "" @@ -329,12 +324,25 @@ msgstr "" msgid "Your Comment" msgstr "" +#: ./src/privatim/views/templates/search_results.pt +msgid "Search Results" +msgstr "" + +#: ./src/privatim/views/templates/search_results.pt +#: ./src/privatim/views/templates/activities.pt +msgid "Show Details" +msgstr "" + #: ./src/privatim/views/templates/password_change.pt msgid "Change Password" msgstr "" #: ./src/privatim/views/templates/activities.pt -msgid "Show Details" +msgid "Unbekannter Ersteller" +msgstr "" + +#: ./src/privatim/views/templates/activities.pt +msgid "Kein Leiter" msgstr "" #: ./src/privatim/views/templates/activities.pt @@ -588,6 +596,10 @@ msgstr "" msgid "Members" msgstr "" +#: ./src/privatim/forms/search_form.py +msgid "Search" +msgstr "" + #: ./src/privatim/forms/widgets/widgets.py msgid "Uploaded file" msgstr "" diff --git a/src/privatim/models/comment.py b/src/privatim/models/comment.py index 78dfa7e..3bbbcde 100644 --- a/src/privatim/models/comment.py +++ b/src/privatim/models/comment.py @@ -76,7 +76,6 @@ def __repr__(self) -> str: @classmethod def searchable_fields(cls): yield cls.content - yield cls.content __table_args__ = ( Index('ix_comments_parent_id', 'parent_id'), diff --git a/src/privatim/models/meeting.py b/src/privatim/models/meeting.py index fa6bd33..8b33be2 100644 --- a/src/privatim/models/meeting.py +++ b/src/privatim/models/meeting.py @@ -176,7 +176,6 @@ def __init__( def searchable_fields(cls) -> Iterator[str]: # todo: agenda item (seperately) yield cls.name - yield cls.name def __acl__(self) -> list['ACL']: return [ diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index bf967fa..ec29ce5 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -1,3 +1,4 @@ +from pyramid.httpexceptions import HTTPFound from sqlalchemy import ( func, select, @@ -9,7 +10,8 @@ ) from privatim.forms.search_form import SearchForm from privatim.models import Consultation, Meeting -from privatim.i18n import locales +from privatim.i18n import locales, translate +from privatim.i18n import _ from sqlalchemy import or_ from privatim.models.comment import Comment @@ -24,7 +26,8 @@ class SearchResult(NamedTuple): id: int - title: str + """ headlines are key value pairs of fields on various models that matched + the search query.""" headlines: dict[str, str] type: str @@ -66,14 +69,11 @@ def search_model(self, model: type[Model], ts_query) -> List[SearchResult]: return self.process_results(raw_results, model) def build_query(self, model: type[Model], ts_query): - search_fields: List[InstrumentedAttribute] = list( - model.searchable_fields() - ) + headline_exprs = self.generate_headlines(model, ts_query) select_fields = [ model.id, - search_fields[0].label('title'), *headline_exprs, cast(literal(model.__name__), String).label('type'), ] @@ -83,17 +83,33 @@ def build_query(self, model: type[Model], ts_query): ) def generate_headlines(self, model: type[Model], ts_query): - search_fields = list(model.searchable_fields()) + """ + Generate headline expressions for all searchable fields of the model. + + Headlines in this context are snippets of text from the searchable + fields, with the matching search terms highlighted. + They provide context around where the search term appears in each field. + + Args: + model (type[Model]): The model class to generate headlines for. + ts_query: The text search query to use for highlighting. + + Returns: + List[ColumnElement]: A list of SQLAlchemy column expressions, each + representing a headline for a searchable field. These expressions + use the ts_headline function to generate highlighted snippets of + text. + """ return [ func.ts_headline( self.lang, field, ts_query, - 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ShortWord=3, HighlightAll=FALSE, MaxFragments=3, FragmentDelimiter=" ... "', + 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ' + 'ShortWord=3, HighlightAll=FALSE, MaxFragments=3, ' + 'FragmentDelimiter=" ... "', ).label(field.name) - for field in search_fields[ - 1: - ] # Skip the first field as it's used for the title + for field in model.searchable_fields() ] def term_filter_text_for_model( @@ -117,21 +133,19 @@ def match_convert( ] def process_results( - self, raw_results, model: type[Model] + self, raw_results, model: type[Model] ) -> List[SearchResult]: searchable = list(model.searchable_fields()) processed_results = [] - field_names = [field.name for field in searchable[1:]] for result in raw_results: headlines = { - field_name: getattr(result, field_name) - for field_name in field_names - if getattr(result, field_name) + field.name: getattr(result, field.name) + for field in searchable + if getattr(result, field.name) } processed_results.append( SearchResult( id=result.id, - title=result.title, headlines=headlines, type=result.type, ) @@ -152,13 +166,41 @@ def __repr__(self) -> str: def search(request: 'IRequest'): + """ + Handle search form submission using POST/Redirect/GET pattern. + + This view processes the search form submitted via POST method, then + redirects to avoid browser warnings on page refresh. The pattern + prevents accidental form resubmission when users refresh the results page. + + + """ session = request.dbsession form = SearchForm(request) + if request.method == 'POST' and form.validate(): - query = request.POST['search'] + query = form.term.data + return HTTPFound(location=request.route_url('search', _query={'q': query})) + query = request.GET.get('q') + if query: result_collection = SearchCollection(term=query, session=session) result_collection.do_search() - activities = result_collection.results - return {'activities': activities} + translated_results = [] + for result in result_collection.results: + headlines_with_translated_keys = { + translate(key.capitalize()): value + for key, value in result.headlines.items() + } + translated_results.append(SearchResult( + id=result.id, + headlines=headlines_with_translated_keys, + type=result.type + )) + + return {'search_results': translated_results, 'query': query} + + return {'search_results': [], 'query': None} + + diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index b3a87ff..98e10a6 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -2,8 +2,8 @@ xmlns="http://www.w3.org/1999/xhtml" xmlns:tal="http://xml.zope.org/namespaces/tal" xmlns:metal="http://xml.zope.org/namespaces/metal" + xmlns:i18n="http://xml.zope.org/namespaces/i18n" i18n:domain="privatim"> -
    @@ -11,34 +11,34 @@

    Search Results

    -
    - -
    +
    -
    ${result.title}
    -
    - ${item[0]}: + +
    + +
    +     +
    + :
    - Show Details + Show Details
    - -
    - -
    From d8c80bc0681523da4313de58ddbedbfd0302daf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 15:09:13 +0200 Subject: [PATCH 09/52] Increase robustness for unsafe getattr call --- src/privatim/views/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index ec29ce5..1769caa 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -139,9 +139,9 @@ def process_results( processed_results = [] for result in raw_results: headlines = { - field.name: getattr(result, field.name) + field.name: value for field in searchable - if getattr(result, field.name) + if (value := getattr(result, field.name, None)) is not None } processed_results.append( SearchResult( From 13cc7aeb29284524c57b46c888a426b182443ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 15:16:56 +0200 Subject: [PATCH 10/52] Using the precomputed ts_query instead of recomputing it each time. In `SearchCollection`, the ts_query is currently created every time the do_search method is called. This is unnecessary if ts_query only depends on the search term, which is not expected to change throughout the lifetime of the object. --- src/privatim/views/search.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 1769caa..b9cc2f6 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -26,7 +26,7 @@ class SearchResult(NamedTuple): id: int - """ headlines are key value pairs of fields on various models that matched + """ headlines are key value pairs of fields on various models that matched the search query.""" headlines: dict[str, str] type: str @@ -55,13 +55,13 @@ def __init__(self, term: str, session: 'Session', language='de_CH'): self.lang = locales[language] self.session = session self.web_search = term + self.ts_query = func.websearch_to_tsquery(self.lang, self.web_search) self.results: List[SearchResult] = [] self.models = [Consultation, Meeting, Comment] def do_search(self) -> None: - ts_query = func.websearch_to_tsquery(self.lang, self.web_search) for model in self.models: - self.results.extend(self.search_model(model, ts_query)) + self.results.extend(self.search_model(model, self.ts_query)) def search_model(self, model: type[Model], ts_query) -> List[SearchResult]: query = self.build_query(model, ts_query) @@ -113,19 +113,17 @@ def generate_headlines(self, model: type[Model], ts_query): ] def term_filter_text_for_model( - self, model: type[Model], language: str + self, model: type[Model], language: str ) -> List[ColumnElement[bool]]: def match( - column: ColumnElement[str], language: str + column: ColumnElement[str], ) -> ColumnElement[bool]: - return column.op('@@')( - func.websearch_to_tsquery(language, self.web_search) - ) + return column.op('@@')(self.ts_query) def match_convert( - column: ColumnElement[str], language: str + column: ColumnElement[str], language: str ) -> ColumnElement[bool]: - return match(func.to_tsvector(language, column), language) + return match(func.to_tsvector(language, column)) return [ match_convert(field, language) @@ -202,5 +200,3 @@ def search(request: 'IRequest'): return {'search_results': translated_results, 'query': query} return {'search_results': [], 'query': None} - - From da82be35e14b31932f593738dd103ea50ac9c11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 15:57:06 +0200 Subject: [PATCH 11/52] Display comments --- src/privatim/models/searchable.py | 1 + src/privatim/views/search.py | 66 ++++++++++++------- .../views/templates/search_results.pt | 57 +++++++++++----- 3 files changed, 86 insertions(+), 38 deletions(-) diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index 87c1d0f..5d98b41 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -32,6 +32,7 @@ def searchable_text(self) -> str: def searchable_models() -> tuple[type[Base], ...]: + """Retrieve all models inheriting from SearchableMixin.""" model_classes = set() for _ in Base.metadata.tables.values(): for mapper in Base.registry.mappers: diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index b9cc2f6..81f9474 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -9,6 +9,7 @@ String, ) from privatim.forms.search_form import SearchForm +from privatim.layouts import Layout from privatim.models import Consultation, Meeting from privatim.i18n import locales, translate from privatim.i18n import _ @@ -19,6 +20,8 @@ from typing import TYPE_CHECKING, List, NamedTuple, TypeVar +from privatim.models.searchable import searchable_models + if TYPE_CHECKING: from pyramid.interfaces import IRequest from sqlalchemy.orm import Session, InstrumentedAttribute @@ -30,6 +33,8 @@ class SearchResult(NamedTuple): the search query.""" headlines: dict[str, str] type: str + model: 'Model | None' # Note that this is not loaded by default for + # performance reasons. Model = TypeVar('Model') @@ -60,9 +65,28 @@ def __init__(self, term: str, session: 'Session', language='de_CH'): self.models = [Consultation, Meeting, Comment] def do_search(self) -> None: - for model in self.models: + for model in searchable_models(): self.results.extend(self.search_model(model, self.ts_query)) + # Fetch Comment objects after the search + comment_ids = [ + result.id for result in self.results if result.type == 'Comment' + ] + if comment_ids: + stmt = select(Comment).filter(Comment.id.in_(comment_ids)) + comments = self.session.scalars(stmt).all() + comment_dict = {comment.id: comment for comment in comments} + + # Update results with fetched Comment objects + self.results = [ + ( + result._replace(model=comment_dict.get(result.id)) + if result.type == 'Comment' + else result + ) + for result in self.results + ] + def search_model(self, model: type[Model], ts_query) -> List[SearchResult]: query = self.build_query(model, ts_query) raw_results = self.session.execute(query).all() @@ -70,11 +94,11 @@ def search_model(self, model: type[Model], ts_query) -> List[SearchResult]: def build_query(self, model: type[Model], ts_query): - headline_exprs = self.generate_headlines(model, ts_query) + headline_expression = self.generate_headlines(model, ts_query) select_fields = [ model.id, - *headline_exprs, + *headline_expression, cast(literal(model.__name__), String).label('type'), ] @@ -99,6 +123,9 @@ def generate_headlines(self, model: type[Model], ts_query): representing a headline for a searchable field. These expressions use the ts_headline function to generate highlighted snippets of text. + + See also https://www.postgresql.org/docs/current/textsearch + -controls.html#TEXTSEARCH-HEADLINE """ return [ func.ts_headline( @@ -133,12 +160,11 @@ def match_convert( def process_results( self, raw_results, model: type[Model] ) -> List[SearchResult]: - searchable = list(model.searchable_fields()) processed_results = [] for result in raw_results: headlines = { - field.name: value - for field in searchable + translate(field.name.capitalize()): value + for field in model.searchable_fields() if (value := getattr(result, field.name, None)) is not None } processed_results.append( @@ -146,6 +172,7 @@ def process_results( id=result.id, headlines=headlines, type=result.type, + model=None ) ) return processed_results @@ -175,7 +202,6 @@ def search(request: 'IRequest'): """ session = request.dbsession form = SearchForm(request) - if request.method == 'POST' and form.validate(): query = form.term.data return HTTPFound(location=request.route_url('search', _query={'q': query})) @@ -184,19 +210,15 @@ def search(request: 'IRequest'): if query: result_collection = SearchCollection(term=query, session=session) result_collection.do_search() + return { + 'search_results': result_collection.results, + 'query': query, + 'layout': Layout(None, request), + } + + return { + 'search_results': [], + 'query': None, + 'layout': Layout(None, request), + } - translated_results = [] - for result in result_collection.results: - headlines_with_translated_keys = { - translate(key.capitalize()): value - for key, value in result.headlines.items() - } - translated_results.append(SearchResult( - id=result.id, - headlines=headlines_with_translated_keys, - type=result.type - )) - - return {'search_results': translated_results, 'query': query} - - return {'search_results': [], 'query': None} diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index 98e10a6..95f0a96 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -2,7 +2,6 @@ xmlns="http://www.w3.org/1999/xhtml" xmlns:tal="http://xml.zope.org/namespaces/tal" xmlns:metal="http://xml.zope.org/namespaces/metal" - xmlns:i18n="http://xml.zope.org/namespaces/i18n" i18n:domain="privatim">
    @@ -16,23 +15,49 @@
    -
    -
    - -
    - -
    -     -
    - : - + +
    +
    + +
    + +
    +     +
    + : + +
    + Show Details
    - Show Details
    -
    -
    + + + + + +
    +
    +
    + avatar +
    +
    + ${result.model.user.fullname} +
    +

    ${layout.format_date(comment.created, 'relative')}

    +
    +
    +

    +
    +
    +     +
    From 6b76406cf06d99f84b687388375fdf414229d80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 16:04:00 +0200 Subject: [PATCH 12/52] fix flake8 --- src/privatim/__init__.py | 5 +++-- src/privatim/models/searchable.py | 12 ++++-------- src/privatim/views/search.py | 17 ++++++++++------- tests/conftest.py | 1 - tests/models/test_searchable_mixin.py | 1 - 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/privatim/__init__.py b/src/privatim/__init__.py index 68271bc..031c08a 100644 --- a/src/privatim/__init__.py +++ b/src/privatim/__init__.py @@ -132,8 +132,9 @@ def upgrade(context: 'UpgradeContext'): # type: ignore[no-untyped-def] ), ) - if not context.has_column('consultations', - 'searchable_text_de_CH'): + if not context.has_column( + 'consultations', 'searchable_text_de_CH' + ): for column in ('searchable_text_de_CH',): if not context.has_column('consultations', column): context.operations.add_column( diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index 5d98b41..8f117b0 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -1,18 +1,14 @@ from sqlalchemy import func, update, Text -import transaction from sqlalchemy.ext.hybrid import hybrid_property import inspect -from sqlalchemy import text -import sys -from functools import cache - -from sqlalchemy.orm import Session, class_mapper from privatim.i18n import locales from privatim.orm import Base -from typing import Iterator +from typing import Iterator, TYPE_CHECKING +if TYPE_CHECKING: + from sqlalchemy.orm import Session class SearchableMixin: @@ -47,7 +43,7 @@ def searchable_models() -> tuple[type[Base], ...]: return tuple(model_classes) -def reindex_full_text_search(session: Session) -> None: +def reindex_full_text_search(session: 'Session') -> None: """ Updates the searchable_text_{} columns. diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 81f9474..0c00707 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -3,7 +3,6 @@ func, select, ColumnElement, - union_all, cast, literal, String, @@ -12,7 +11,6 @@ from privatim.layouts import Layout from privatim.models import Consultation, Meeting from privatim.i18n import locales, translate -from privatim.i18n import _ from sqlalchemy import or_ from privatim.models.comment import Comment @@ -24,7 +22,7 @@ if TYPE_CHECKING: from pyramid.interfaces import IRequest - from sqlalchemy.orm import Session, InstrumentedAttribute + from sqlalchemy.orm import Session class SearchResult(NamedTuple): @@ -99,7 +97,7 @@ def build_query(self, model: type[Model], ts_query): select_fields = [ model.id, *headline_expression, - cast(literal(model.__name__), String).label('type'), + cast(literal(model.__name__), String).label('type'), # noqa: MS001 ] return select(*select_fields).filter( @@ -112,7 +110,8 @@ def generate_headlines(self, model: type[Model], ts_query): Headlines in this context are snippets of text from the searchable fields, with the matching search terms highlighted. - They provide context around where the search term appears in each field. + They provide context around where the search term appears in each + field. Args: model (type[Model]): The model class to generate headlines for. @@ -204,7 +203,12 @@ def search(request: 'IRequest'): form = SearchForm(request) if request.method == 'POST' and form.validate(): query = form.term.data - return HTTPFound(location=request.route_url('search', _query={'q': query})) + return HTTPFound( + location=request.route_url( + 'search', + _query={'q': query}, + ) + ) query = request.GET.get('q') if query: @@ -221,4 +225,3 @@ def search(request: 'IRequest'): 'query': None, 'layout': Layout(None, request), } - diff --git a/tests/conftest.py b/tests/conftest.py index 23a8bc9..a37b22a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ import warnings import pytest -import sqlalchemy import transaction from libcloud.storage.drivers.local import LocalStorageDriver from pyramid import testing diff --git a/tests/models/test_searchable_mixin.py b/tests/models/test_searchable_mixin.py index aebd76c..24ef113 100644 --- a/tests/models/test_searchable_mixin.py +++ b/tests/models/test_searchable_mixin.py @@ -1,4 +1,3 @@ -import transaction from sqlalchemy import select from sqlalchemy.orm import undefer from privatim.models import Consultation From efa9ba970ae0f6714e987b20ae7f67452cf0d907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 16:46:31 +0200 Subject: [PATCH 13/52] Fixes a couple of mypy issues --- src/privatim/i18n/__init__.py | 2 +- src/privatim/models/searchable.py | 13 ++--- src/privatim/types.py | 18 ++++++- src/privatim/views/search.py | 88 +++++++++++++++++++------------ 4 files changed, 76 insertions(+), 45 deletions(-) diff --git a/src/privatim/i18n/__init__.py b/src/privatim/i18n/__init__.py index 5c64633..c45bf5d 100644 --- a/src/privatim/i18n/__init__.py +++ b/src/privatim/i18n/__init__.py @@ -14,5 +14,5 @@ 'LocaleNegotiator', 'pluralize', 'translate', - locales, + 'locales', ) diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index 8f117b0..fbd3262 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -7,6 +7,9 @@ from typing import Iterator, TYPE_CHECKING + +from privatim.types import HasSearchableFields + if TYPE_CHECKING: from sqlalchemy.orm import Session @@ -27,18 +30,13 @@ def searchable_text(self) -> str: ) -def searchable_models() -> tuple[type[Base], ...]: +def searchable_models() -> tuple[type[HasSearchableFields], ...]: """Retrieve all models inheriting from SearchableMixin.""" model_classes = set() for _ in Base.metadata.tables.values(): for mapper in Base.registry.mappers: cls = mapper.class_ - if ( - inspect.isclass(cls) - and issubclass(cls, SearchableMixin) - and issubclass(cls, Base) - and cls != SearchableMixin - ): + if issubclass(cls, SearchableMixin): model_classes.add(cls) return tuple(model_classes) @@ -62,7 +60,6 @@ def reindex_full_text_search(session: 'Session') -> None: # todo: remove later assert len(models) != 0, "No models with searchable fields found" for model in models: - assert issubclass(model, SearchableMixin) for locale, language in locales.items(): assert language == 'german' # todo: remove later if hasattr(model, f'searchable_text_{locale}'): diff --git a/src/privatim/types.py b/src/privatim/types.py index c1019a9..cf0bb62 100644 --- a/src/privatim/types.py +++ b/src/privatim/types.py @@ -1,4 +1,11 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Type, Union, Iterable + +from sqlalchemy import ColumnElement + +from privatim.models import SearchableMixin +from privatim.orm import Base +from privatim.orm.meta import UUIDStrPK + if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -77,3 +84,12 @@ class LaxFileDict(TypedDict): class Callback(Protocol[_Tco]): def __call__(self, context: Any, request: IRequest) -> _Tco: ... + + + class HasSearchableFields(Protocol): + id: UUIDStrPK + + @classmethod + def searchable_fields(cls) -> Iterable[ColumnElement[Any]]: + ... + diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 0c00707..0f2f25f 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -13,31 +13,43 @@ from privatim.i18n import locales, translate from sqlalchemy import or_ +from privatim.models.searchable import searchable_models from privatim.models.comment import Comment -from typing import TYPE_CHECKING, List, NamedTuple, TypeVar +from typing import ( + TYPE_CHECKING, + List, + NamedTuple, + Dict, + Any, + Optional, + Iterator, +) -from privatim.models.searchable import searchable_models if TYPE_CHECKING: from pyramid.interfaces import IRequest from sqlalchemy.orm import Session + from privatim.orm.meta import UUIDStrPK + from privatim.types import HasSearchableFields + from builtins import type as type_t + from privatim.orm import Base + from privatim.models.searchable import SearchableMixin + class SearchResult(NamedTuple): id: int """ headlines are key value pairs of fields on various models that matched the search query.""" - headlines: dict[str, str] + headlines: Dict[str, str] type: str - model: 'Model | None' # Note that this is not loaded by default for + model: 'Optional[type_t[HasSearchableFields]]' # Note that this is not loaded by + # default for # performance reasons. -Model = TypeVar('Model') - - class SearchCollection: """A class for searching the database for a given term. @@ -54,26 +66,27 @@ class SearchCollection: """ - def __init__(self, term: str, session: 'Session', language='de_CH'): - self.lang = locales[language] - self.session = session - self.web_search = term + def __init__(self, term: str, session: 'Session', language: str = 'de_CH'): + self.lang: str = locales[language] + self.session: 'Session' = session + self.web_search: str = term self.ts_query = func.websearch_to_tsquery(self.lang, self.web_search) self.results: List[SearchResult] = [] - self.models = [Consultation, Meeting, Comment] def do_search(self) -> None: for model in searchable_models(): self.results.extend(self.search_model(model, self.ts_query)) # Fetch Comment objects after the search - comment_ids = [ + comment_ids: List[int] = [ result.id for result in self.results if result.type == 'Comment' ] if comment_ids: stmt = select(Comment).filter(Comment.id.in_(comment_ids)) comments = self.session.scalars(stmt).all() - comment_dict = {comment.id: comment for comment in comments} + comment_dict: dict[UUIDStrPK, Comment] = { + comment.id: comment for comment in comments + } # Update results with fetched Comment objects self.results = [ @@ -85,16 +98,18 @@ def do_search(self) -> None: for result in self.results ] - def search_model(self, model: type[Model], ts_query) -> List[SearchResult]: + def search_model( + self, model: 'type[HasSearchableFields]', ts_query: Any + ) -> List[SearchResult]: query = self.build_query(model, ts_query) raw_results = self.session.execute(query).all() return self.process_results(raw_results, model) - def build_query(self, model: type[Model], ts_query): + def build_query(self, model: 'type[HasSearchableFields]', ts_query: Any) -> Any: headline_expression = self.generate_headlines(model, ts_query) - select_fields = [ + select_fields: List[Any] = [ model.id, *headline_expression, cast(literal(model.__name__), String).label('type'), # noqa: MS001 @@ -104,7 +119,9 @@ def build_query(self, model: type[Model], ts_query): or_(*self.term_filter_text_for_model(model, self.lang)) ) - def generate_headlines(self, model: type[Model], ts_query): + def generate_headlines( + self, model:'type[HasSearchableFields]', ts_query: Any + ) -> List[ColumnElement]: """ Generate headline expressions for all searchable fields of the model. @@ -114,7 +131,7 @@ def generate_headlines(self, model: type[Model], ts_query): field. Args: - model (type[Model]): The model class to generate headlines for. + model (type['type[HasSearchableFields]']): The model class to generate headlines for. ts_query: The text search query to use for highlighting. Returns: @@ -139,15 +156,15 @@ def generate_headlines(self, model: type[Model], ts_query): ] def term_filter_text_for_model( - self, model: type[Model], language: str + self, model:'type[HasSearchableFields]', language: str ) -> List[ColumnElement[bool]]: def match( - column: ColumnElement[str], + column: ColumnElement[str], ) -> ColumnElement[bool]: return column.op('@@')(self.ts_query) def match_convert( - column: ColumnElement[str], language: str + column: ColumnElement[str], language: str ) -> ColumnElement[bool]: return match(func.to_tsvector(language, column)) @@ -157,11 +174,11 @@ def match_convert( ] def process_results( - self, raw_results, model: type[Model] + self, raw_results: List[Any], model:'type[HasSearchableFields]' ) -> List[SearchResult]: - processed_results = [] + processed_results: List[SearchResult] = [] for result in raw_results: - headlines = { + headlines: Dict[str, str] = { translate(field.name.capitalize()): value for field in model.searchable_fields() if (value := getattr(result, field.name, None)) is not None @@ -171,25 +188,25 @@ def process_results( id=result.id, headlines=headlines, type=result.type, - model=None + model=None, ) ) return processed_results - def __len__(self): + def __len__(self) -> int: return len(self.results) - def __iter__(self): + def __iter__(self) -> Iterator[SearchResult]: return iter(self.results) - def __getitem__(self, index): + def __getitem__(self, index: int) -> SearchResult: return self.results[index] def __repr__(self) -> str: return f'' -def search(request: 'IRequest'): +def search(request: 'IRequest') -> Dict[str, Any]: """ Handle search form submission using POST/Redirect/GET pattern. @@ -199,20 +216,21 @@ def search(request: 'IRequest'): """ - session = request.dbsession - form = SearchForm(request) + session: 'Session' = request.dbsession + form: SearchForm = SearchForm(request) if request.method == 'POST' and form.validate(): - query = form.term.data return HTTPFound( location=request.route_url( 'search', - _query={'q': query}, + _query={'q': form.term.data}, ) ) query = request.GET.get('q') if query: - result_collection = SearchCollection(term=query, session=session) + result_collection: SearchCollection = SearchCollection( + term=query, session=session + ) result_collection.do_search() return { 'search_results': result_collection.results, From 4e39dd28316e95573a069164e1aaa5293b3a224b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 17:41:57 +0200 Subject: [PATCH 14/52] Attempts to resolve more mypy errors. --- src/privatim/forms/fields/fields.py | 2 +- src/privatim/models/__init__.py | 9 +--- src/privatim/models/associated_file.py | 2 +- src/privatim/models/comment.py | 13 +++-- src/privatim/models/consultation.py | 10 +++- src/privatim/models/meeting.py | 3 +- src/privatim/models/searchable.py | 16 +++--- src/privatim/types.py | 19 ++++--- src/privatim/views/search.py | 69 +++++++++++++------------- 9 files changed, 75 insertions(+), 68 deletions(-) diff --git a/src/privatim/forms/fields/fields.py b/src/privatim/forms/fields/fields.py index a57d165..02c72de 100644 --- a/src/privatim/forms/fields/fields.py +++ b/src/privatim/forms/fields/fields.py @@ -384,7 +384,7 @@ def append_entry_from_field_storage( # we fake the formdata for the new field # we use a werkzeug MultiDict because the WebOb version # needs to get wrapped to be usable in WTForms - formdata: MultiDict[str, 'RawFormValue'] = MultiDict() + formdata: MultiDict[str, RawFormValue] = MultiDict() name = f'{self.short_name}{self._separator}{len(self)}' formdata.add(name, fs) return self._add_entry(formdata) diff --git a/src/privatim/models/__init__.py b/src/privatim/models/__init__.py index 44ebc02..d71d0f3 100644 --- a/src/privatim/models/__init__.py +++ b/src/privatim/models/__init__.py @@ -40,18 +40,13 @@ from pyramid.config import Configurator from sqlalchemy.engine import Connection - T = TypeVar('T', bound='SearchableBase') - - class SearchableBase(Base, SearchableMixin): - pass - # Run ``configure_mappers`` after defining all of the models to ensure # all relationships can be setup. configure_mappers() def update_searchable_text_listener( - mapper: Mapper['SearchableBase'], connection: 'Connection', target: Any + mapper: Mapper['Any'], connection: 'Connection', target: Any ) -> None: return # we will implement this for updates of document text @@ -67,7 +62,7 @@ def update_searchable_text_listener( ) -def register_search_listeners(model) -> None: +def register_search_listeners(model: 'type[Consultation | Meeting]') -> None: event.listen(model, 'after_insert', update_searchable_text_listener) event.listen(model, 'after_update', update_searchable_text_listener) diff --git a/src/privatim/models/associated_file.py b/src/privatim/models/associated_file.py index caaa795..9a68aa8 100644 --- a/src/privatim/models/associated_file.py +++ b/src/privatim/models/associated_file.py @@ -44,7 +44,7 @@ def reindex_files(self, searchable_files: Sequence[GeneralFile]) -> None: def searchable_files(self) -> Sequence[GeneralFile]: # For now we just consider PDF's stmt = select(GeneralFile).where( - GeneralFile.content_type == 'application/pdf' + GeneralFile.content_type.is_('application/pdf') ) return object_session(self).execute(stmt).scalars().all() diff --git a/src/privatim/models/comment.py b/src/privatim/models/comment.py index 3bbbcde..33a0cb9 100644 --- a/src/privatim/models/comment.py +++ b/src/privatim/models/comment.py @@ -3,14 +3,21 @@ from privatim.orm import Base from privatim.orm.associable import Associable from privatim.models import SearchableMixin -from sqlalchemy.orm import relationship, Mapped, mapped_column, foreign, remote +from sqlalchemy.orm import ( + relationship, + Mapped, + mapped_column, + foreign, + remote, +) from privatim.orm.meta import UUIDStrPK, UUIDStr from sqlalchemy import Text, ForeignKey, Index, and_ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Iterator if TYPE_CHECKING: from privatim.models import User + from sqlalchemy.orm import InstrumentedAttribute class Comment(Base, Associable, SearchableMixin): @@ -74,7 +81,7 @@ def __repr__(self) -> str: return f'' @classmethod - def searchable_fields(cls): + def searchable_fields(cls) -> Iterator['InstrumentedAttribute[str]']: yield cls.content __table_args__ = ( diff --git a/src/privatim/models/consultation.py b/src/privatim/models/consultation.py index adfa5d1..18a8ca6 100644 --- a/src/privatim/models/consultation.py +++ b/src/privatim/models/consultation.py @@ -2,7 +2,12 @@ from sedate import utcnow from sqlalchemy import Text, ForeignKey from sqlalchemy.dialects.postgresql import TSVECTOR -from sqlalchemy.orm import Mapped, mapped_column, relationship, deferred +from sqlalchemy.orm import ( + Mapped, + mapped_column, + relationship, + deferred +) from pyramid.authorization import Allow from pyramid.authorization import Authenticated @@ -19,6 +24,7 @@ if TYPE_CHECKING: from privatim.types import ACL from privatim.models import User + from sqlalchemy.orm import InstrumentedAttribute class Status(Base): @@ -90,7 +96,7 @@ class Consultation(Base, Commentable, AssociatedFiles, SearchableMixin): searchable_text_de_CH: Mapped[str] = deferred(mapped_column(TSVECTOR)) @classmethod - def searchable_fields(cls) -> Iterator[str]: + def searchable_fields(cls) -> Iterator['InstrumentedAttribute[str]']: yield cls.title yield cls.description yield cls.recommendation diff --git a/src/privatim/models/meeting.py b/src/privatim/models/meeting.py index 8b33be2..a62f001 100644 --- a/src/privatim/models/meeting.py +++ b/src/privatim/models/meeting.py @@ -20,6 +20,7 @@ from datetime import datetime from privatim.types import ACL from sqlalchemy.orm import Session + from sqlalchemy.orm import InstrumentedAttribute class AgendaItemCreationError(Exception): @@ -173,7 +174,7 @@ def __init__( ) @classmethod - def searchable_fields(cls) -> Iterator[str]: + def searchable_fields(cls) -> Iterator['InstrumentedAttribute[str]']: # todo: agenda item (seperately) yield cls.name diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index fbd3262..6589410 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -1,22 +1,19 @@ from sqlalchemy import func, update, Text from sqlalchemy.ext.hybrid import hybrid_property -import inspect from privatim.i18n import locales from privatim.orm import Base from typing import Iterator, TYPE_CHECKING - -from privatim.types import HasSearchableFields - if TYPE_CHECKING: - from sqlalchemy.orm import Session + from privatim.types import HasSearchableFields + from sqlalchemy.orm import Session, InstrumentedAttribute -class SearchableMixin: +class SearchableMixin(HasSearchableFields): @classmethod - def searchable_fields(cls) -> Iterator[str]: + def searchable_fields(cls) -> Iterator['InstrumentedAttribute[str]']: # Override this method in each model to specify searchable fields raise NotImplementedError( "Searchable fields must be defined for each model" @@ -26,11 +23,12 @@ def searchable_fields(cls) -> Iterator[str]: def searchable_text(self) -> str: # todo: extract document text return ' '.join( - str(getattr(self, field)) for field in self.searchable_fields() + str(getattr(self, field.key)) + for field in self.__class__.searchable_fields() ) -def searchable_models() -> tuple[type[HasSearchableFields], ...]: +def searchable_models() -> tuple[type['HasSearchableFields'], ...]: """Retrieve all models inheriting from SearchableMixin.""" model_classes = set() for _ in Base.metadata.tables.values(): diff --git a/src/privatim/types.py b/src/privatim/types.py index cf0bb62..7ab3051 100644 --- a/src/privatim/types.py +++ b/src/privatim/types.py @@ -1,14 +1,11 @@ -from typing import TYPE_CHECKING, Type, Union, Iterable - -from sqlalchemy import ColumnElement - -from privatim.models import SearchableMixin -from privatim.orm import Base -from privatim.orm.meta import UUIDStrPK - +from typing import TYPE_CHECKING, Iterable, Iterator if TYPE_CHECKING: from collections.abc import Mapping, Sequence + from sqlalchemy.orm import InstrumentedAttribute + from sqlalchemy import ColumnElement + from privatim.orm.meta import UUIDStrPK + from sqlalchemy.ext.hybrid import hybrid_property from decimal import Decimal from fractions import Fraction from pyramid.httpexceptions import ( @@ -85,11 +82,13 @@ class LaxFileDict(TypedDict): class Callback(Protocol[_Tco]): def __call__(self, context: Any, request: IRequest) -> _Tco: ... - class HasSearchableFields(Protocol): id: UUIDStrPK @classmethod - def searchable_fields(cls) -> Iterable[ColumnElement[Any]]: + def searchable_fields(cls) -> Iterator['InstrumentedAttribute[str]']: ... + @hybrid_property + def searchable_text(self) -> str: + ... diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 0f2f25f..e134c40 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -9,7 +9,6 @@ ) from privatim.forms.search_form import SearchForm from privatim.layouts import Layout -from privatim.models import Consultation, Meeting from privatim.i18n import locales, translate from sqlalchemy import or_ @@ -17,37 +16,28 @@ from privatim.models.comment import Comment -from typing import ( - TYPE_CHECKING, - List, - NamedTuple, - Dict, - Any, - Optional, - Iterator, -) +from typing import (TYPE_CHECKING, List, NamedTuple, Dict, Any, Optional, + Iterator, Sequence, ) if TYPE_CHECKING: + from sqlalchemy.sql.selectable import Select from pyramid.interfaces import IRequest from sqlalchemy.orm import Session - - from privatim.orm.meta import UUIDStrPK - from privatim.types import HasSearchableFields + from sqlalchemy.dialects.postgresql.ext import websearch_to_tsquery + from privatim.types import HasSearchableFields, RenderDataOrRedirect from builtins import type as type_t - from privatim.orm import Base - from privatim.models.searchable import SearchableMixin class SearchResult(NamedTuple): - id: int + id: str """ headlines are key value pairs of fields on various models that matched the search query.""" headlines: Dict[str, str] type: str - model: 'Optional[type_t[HasSearchableFields]]' # Note that this is not loaded by - # default for - # performance reasons. + model: 'Optional[type_t[HasSearchableFields]]' # We only load this if it + # makes sense int he UI to display additional attributes from the model, + # otherwise we can save ourselves a query. class SearchCollection: @@ -68,9 +58,12 @@ class SearchCollection: def __init__(self, term: str, session: 'Session', language: str = 'de_CH'): self.lang: str = locales[language] - self.session: 'Session' = session + self.session = session self.web_search: str = term - self.ts_query = func.websearch_to_tsquery(self.lang, self.web_search) + + self.ts_query: 'websearch_to_tsquery' = func.websearch_to_tsquery( + self.lang, self.web_search + ) self.results: List[SearchResult] = [] def do_search(self) -> None: @@ -78,13 +71,13 @@ def do_search(self) -> None: self.results.extend(self.search_model(model, self.ts_query)) # Fetch Comment objects after the search - comment_ids: List[int] = [ + comment_ids = [ result.id for result in self.results if result.type == 'Comment' ] if comment_ids: stmt = select(Comment).filter(Comment.id.in_(comment_ids)) comments = self.session.scalars(stmt).all() - comment_dict: dict[UUIDStrPK, Comment] = { + comment_dict: dict[str, Comment] = { comment.id: comment for comment in comments } @@ -99,13 +92,19 @@ def do_search(self) -> None: ] def search_model( - self, model: 'type[HasSearchableFields]', ts_query: Any + self, + model: type['HasSearchableFields'], + ts_query: 'websearch_to_tsquery', ) -> List[SearchResult]: query = self.build_query(model, ts_query) raw_results = self.session.execute(query).all() return self.process_results(raw_results, model) - def build_query(self, model: 'type[HasSearchableFields]', ts_query: Any) -> Any: + def build_query( + self, + model: type['HasSearchableFields'], + ts_query: 'websearch_to_tsquery', + ) -> 'Select': headline_expression = self.generate_headlines(model, ts_query) @@ -120,8 +119,10 @@ def build_query(self, model: 'type[HasSearchableFields]', ts_query: Any) -> Any: ) def generate_headlines( - self, model:'type[HasSearchableFields]', ts_query: Any - ) -> List[ColumnElement]: + self, + model: type['HasSearchableFields'], + ts_query: 'websearch_to_tsquery', + ) -> List[ColumnElement[bool]]: """ Generate headline expressions for all searchable fields of the model. @@ -130,9 +131,9 @@ def generate_headlines( They provide context around where the search term appears in each field. - Args: - model (type['type[HasSearchableFields]']): The model class to generate headlines for. - ts_query: The text search query to use for highlighting. + Args: model (type['type[HasSearchableFields]']): The model class to + generate headlines for. ts_query: The text search query to use for + highlighting. Returns: List[ColumnElement]: A list of SQLAlchemy column expressions, each @@ -156,7 +157,7 @@ def generate_headlines( ] def term_filter_text_for_model( - self, model:'type[HasSearchableFields]', language: str + self, model: 'type[HasSearchableFields]', language: str ) -> List[ColumnElement[bool]]: def match( column: ColumnElement[str], @@ -174,7 +175,7 @@ def match_convert( ] def process_results( - self, raw_results: List[Any], model:'type[HasSearchableFields]' + self, raw_results: Sequence[Any], model: 'type[HasSearchableFields]' ) -> List[SearchResult]: processed_results: List[SearchResult] = [] for result in raw_results: @@ -206,7 +207,7 @@ def __repr__(self) -> str: return f'' -def search(request: 'IRequest') -> Dict[str, Any]: +def search(request: 'IRequest') -> 'RenderDataOrRedirect': """ Handle search form submission using POST/Redirect/GET pattern. @@ -216,7 +217,7 @@ def search(request: 'IRequest') -> Dict[str, Any]: """ - session: 'Session' = request.dbsession + session: Session = request.dbsession form: SearchForm = SearchForm(request) if request.method == 'POST' and form.validate(): return HTTPFound( From ff3b42e2ae8e514bb55b94357052f5ed2322df2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 22:28:34 +0200 Subject: [PATCH 15/52] clean up --- src/privatim/views/search.py | 43 +++++++++++++----------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index e134c40..63b2070 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -68,9 +68,15 @@ def __init__(self, term: str, session: 'Session', language: str = 'de_CH'): def do_search(self) -> None: for model in searchable_models(): - self.results.extend(self.search_model(model, self.ts_query)) + self.results.extend(self.search_model(model)) - # Fetch Comment objects after the search + self._add_comments_to_results() + + def _add_comments_to_results(self): + """ Updates SearchResult.model with the complete query for Comment. + + This is for displaying more information in the search results. + """ comment_ids = [ result.id for result in self.results if result.type == 'Comment' ] @@ -80,33 +86,24 @@ def do_search(self) -> None: comment_dict: dict[str, Comment] = { comment.id: comment for comment in comments } - - # Update results with fetched Comment objects - self.results = [ - ( - result._replace(model=comment_dict.get(result.id)) - if result.type == 'Comment' - else result - ) - for result in self.results - ] + self.results = [(result._replace(model=comment_dict.get( + result.id)) if result.type == 'Comment' else result) for result + in self.results] def search_model( self, model: type['HasSearchableFields'], - ts_query: 'websearch_to_tsquery', ) -> List[SearchResult]: - query = self.build_query(model, ts_query) + query = self.build_query(model, self.ts_query) raw_results = self.session.execute(query).all() return self.process_results(raw_results, model) def build_query( self, model: type['HasSearchableFields'], - ts_query: 'websearch_to_tsquery', ) -> 'Select': - headline_expression = self.generate_headlines(model, ts_query) + headline_expression = self.generate_headlines(model,) select_fields: List[Any] = [ model.id, @@ -121,7 +118,6 @@ def build_query( def generate_headlines( self, model: type['HasSearchableFields'], - ts_query: 'websearch_to_tsquery', ) -> List[ColumnElement[bool]]: """ Generate headline expressions for all searchable fields of the model. @@ -148,7 +144,7 @@ def generate_headlines( func.ts_headline( self.lang, field, - ts_query, + self.ts_query, 'StartSel=, StopSel=, MaxWords=35, MinWords=15, ' 'ShortWord=3, HighlightAll=FALSE, MaxFragments=3, ' 'FragmentDelimiter=" ... "', @@ -179,7 +175,7 @@ def process_results( ) -> List[SearchResult]: processed_results: List[SearchResult] = [] for result in raw_results: - headlines: Dict[str, str] = { + headlines: dict[str, str] = { translate(field.name.capitalize()): value for field in model.searchable_fields() if (value := getattr(result, field.name, None)) is not None @@ -194,15 +190,6 @@ def process_results( ) return processed_results - def __len__(self) -> int: - return len(self.results) - - def __iter__(self) -> Iterator[SearchResult]: - return iter(self.results) - - def __getitem__(self, index: int) -> SearchResult: - return self.results[index] - def __repr__(self) -> str: return f'' From dcc90c78cd96da25f8cf4b766dd8a98d7a28dbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 22:59:40 +0200 Subject: [PATCH 16/52] Clean up, add some docstring. --- src/privatim/views/search.py | 77 ++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 63b2070..63fbbcc 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -60,7 +60,6 @@ def __init__(self, term: str, session: 'Session', language: str = 'de_CH'): self.lang: str = locales[language] self.session = session self.web_search: str = term - self.ts_query: 'websearch_to_tsquery' = func.websearch_to_tsquery( self.lang, self.web_search ) @@ -73,7 +72,7 @@ def do_search(self) -> None: self._add_comments_to_results() def _add_comments_to_results(self): - """ Updates SearchResult.model with the complete query for Comment. + """ Extends self.results with the complete query for Comment. This is for displaying more information in the search results. """ @@ -103,44 +102,26 @@ def build_query( model: type['HasSearchableFields'], ) -> 'Select': - headline_expression = self.generate_headlines(model,) - - select_fields: List[Any] = [ - model.id, - *headline_expression, - cast(literal(model.__name__), String).label('type'), # noqa: MS001 - ] - - return select(*select_fields).filter( - or_(*self.term_filter_text_for_model(model, self.lang)) - ) - - def generate_headlines( - self, - model: type['HasSearchableFields'], - ) -> List[ColumnElement[bool]]: """ - Generate headline expressions for all searchable fields of the model. + Builds the actual query for full text search. - Headlines in this context are snippets of text from the searchable + 1. Generate headline expressions for all searchable fields of the + model. Headlines in this context are snippets of text from the searchable fields, with the matching search terms highlighted. They provide context around where the search term appears in each field. - Args: model (type['type[HasSearchableFields]']): The model class to - generate headlines for. ts_query: The text search query to use for - highlighting. + 2. Perform the actual search in all + searchable fields using `create_fulltext_search_conditions` - Returns: - List[ColumnElement]: A list of SQLAlchemy column expressions, each - representing a headline for a searchable field. These expressions - use the ts_headline function to generate highlighted snippets of - text. + Returns A list of SQLAlchemy column expressions, each + representing a headline for a searchable field. See also https://www.postgresql.org/docs/current/textsearch -controls.html#TEXTSEARCH-HEADLINE + """ - return [ + headline_expression = [ func.ts_headline( self.lang, field, @@ -151,10 +132,31 @@ def generate_headlines( ).label(field.name) for field in model.searchable_fields() ] + select_fields: List[Any] = [ + model.id, + *headline_expression, + cast(literal(model.__name__), String).label('type'), # noqa: MS001 + ] - def term_filter_text_for_model( - self, model: 'type[HasSearchableFields]', language: str + return select(*select_fields).filter( + or_(*self.create_fulltext_search_conditions(model.searchable_fields())) + ) + + def create_fulltext_search_conditions( + self, + searchable_fields: Iterator['InstrumentedAttribute[str]'], ) -> List[ColumnElement[bool]]: + """ + The column.op@@ expression is SQLAlchemy's custom operator + functionality to create a full-text search operation. + + Note that we convert to tsvector at runtime, this could be done at + indexing time for performance reasons. But for now we keep it simple. + For relatively small datasets, as we expect them to be this will + probably not be a bottleneck. + + """ + def match( column: ColumnElement[str], ) -> ColumnElement[bool]: @@ -165,14 +167,13 @@ def match_convert( ) -> ColumnElement[bool]: return match(func.to_tsvector(language, column)) - return [ - match_convert(field, language) - for field in model.searchable_fields() - ] + return [match_convert(field, self.lang) for field in searchable_fields] def process_results( self, raw_results: Sequence[Any], model: 'type[HasSearchableFields]' ) -> List[SearchResult]: + """ Helper function to produce a safe typed output of + list[SearchResult] """ processed_results: List[SearchResult] = [] for result in raw_results: headlines: dict[str, str] = { @@ -196,11 +197,11 @@ def __repr__(self) -> str: def search(request: 'IRequest') -> 'RenderDataOrRedirect': """ - Handle search form submission using POST/Redirect/GET pattern. + Handle search form submission using POST/Redirect/GET design pattern. This view processes the search form submitted via POST method, then - redirects to avoid browser warnings on page refresh. The pattern - prevents accidental form resubmission when users refresh the results page. + redirects to avoid browser warnings on page refresh. This prevents + accidental form resubmission if users refresh the results page. """ From 61bb501657aef173892ed9414b30bce8eb0c704d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Tue, 2 Jul 2024 23:38:12 +0200 Subject: [PATCH 17/52] Add SearchableFile --- src/privatim/models/__init__.py | 1 + src/privatim/models/file.py | 29 ++++++++++++++++++++++++++++- src/privatim/models/searchable.py | 7 ++++--- src/privatim/types.py | 3 +-- src/privatim/views/search.py | 6 ++---- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/privatim/models/__init__.py b/src/privatim/models/__init__.py index d71d0f3..47acf97 100644 --- a/src/privatim/models/__init__.py +++ b/src/privatim/models/__init__.py @@ -33,6 +33,7 @@ Statement PasswordChangeToken GeneralFile +SearchableMixin from typing import TYPE_CHECKING, Any, TypeVar # noqa: E402 diff --git a/src/privatim/models/file.py b/src/privatim/models/file.py index 9327fec..0230c34 100644 --- a/src/privatim/models/file.py +++ b/src/privatim/models/file.py @@ -2,11 +2,12 @@ from pyramid.authorization import Allow from pyramid.authorization import Authenticated -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.orm import Mapped, mapped_column, deferred from sqlalchemy_file import File from privatim.orm.associable import Associable from privatim.orm.meta import UUIDStrPK, AttachedFile +from sqlalchemy import Text, ForeignKey from privatim.orm import Base from typing import TYPE_CHECKING @@ -46,3 +47,29 @@ def __acl__(self) -> list['ACL']: return [ (Allow, Authenticated, ['view']), ] + + +class SearchableFile(GeneralFile): + """ A file with the intention of being searchable.$ + + """ + __tablename__ = 'searchable_files' + + id: Mapped[UUIDStrPK] = mapped_column( + ForeignKey('general_files.id'), primary_key=True + ) + + # the content of the given file as text. + # (it is important that this column be loaded deferred by default, lest + # we load massive amounts of text on simple queries) + extract: Mapped[str | None] = deferred(mapped_column(Text, nullable=True)) + + __mapper_args__ = { + 'polymorphic_identity': 'searchable_file', + } + + def __init__( + self, filename: str, content: bytes, extract: str | None = None + ) -> None: + super().__init__(filename, content) + self.extract = extract diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index 6589410..7ca485b 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -7,11 +7,12 @@ from typing import Iterator, TYPE_CHECKING if TYPE_CHECKING: - from privatim.types import HasSearchableFields from sqlalchemy.orm import Session, InstrumentedAttribute + from privatim.types import HasSearchableFields -class SearchableMixin(HasSearchableFields): +# todo: should this implement the protocol? +class SearchableMixin: @classmethod def searchable_fields(cls) -> Iterator['InstrumentedAttribute[str]']: # Override this method in each model to specify searchable fields @@ -47,7 +48,7 @@ def reindex_full_text_search(session: 'Session') -> None: column to Text type. This ensures that we're passing a text value to to_tsvector, not a tsvector. - j + 2. We wrap this in a func.coalesce() call, which will return an empty string if the column value is NULL. This prevents potential errors if some rows have NULL values in the searchable_text diff --git a/src/privatim/types.py b/src/privatim/types.py index 7ab3051..eeb93ed 100644 --- a/src/privatim/types.py +++ b/src/privatim/types.py @@ -1,9 +1,8 @@ -from typing import TYPE_CHECKING, Iterable, Iterator +from typing import TYPE_CHECKING, Iterator if TYPE_CHECKING: from collections.abc import Mapping, Sequence from sqlalchemy.orm import InstrumentedAttribute - from sqlalchemy import ColumnElement from privatim.orm.meta import UUIDStrPK from sqlalchemy.ext.hybrid import hybrid_property from decimal import Decimal diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index 63fbbcc..de227ac 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -60,9 +60,7 @@ def __init__(self, term: str, session: 'Session', language: str = 'de_CH'): self.lang: str = locales[language] self.session = session self.web_search: str = term - self.ts_query: 'websearch_to_tsquery' = func.websearch_to_tsquery( - self.lang, self.web_search - ) + self.ts_query = func.websearch_to_tsquery(self.lang, self.web_search) self.results: List[SearchResult] = [] def do_search(self) -> None: @@ -93,7 +91,7 @@ def search_model( self, model: type['HasSearchableFields'], ) -> List[SearchResult]: - query = self.build_query(model, self.ts_query) + query = self.build_query(model) raw_results = self.session.execute(query).all() return self.process_results(raw_results, model) From df1507c86ebac2aa4868a18aa1ce94c1ba273a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Wed, 3 Jul 2024 00:33:51 +0200 Subject: [PATCH 18/52] tests: migrate away from SQlite In-Memory db. --- i18n.sh | 27 ++++ .../locale/de/LC_MESSAGES/privatim.mo | Bin 8979 -> 9125 bytes .../locale/de/LC_MESSAGES/privatim.po | 6 +- .../locale/fr/LC_MESSAGES/privatim.mo | Bin 9271 -> 9425 bytes .../locale/fr/LC_MESSAGES/privatim.po | 14 ++- src/privatim/locale/privatim.pot | 6 +- src/privatim/models/searchable.py | 2 - .../views/templates/search_results.pt | 7 ++ tests/cli/test_upgrade.py | 16 +-- tests/conftest.py | 119 +++++------------- tests/forms/test_forms_meeting.py | 2 +- tests/i18n/test_core.py | 18 +-- tests/models/test_searchable_mixin.py | 2 + tests/models/test_user.py | 24 ++-- tests/reporting/test_report.py | 2 +- tests/views/client/test_logout.py | 2 +- tests/views/client/test_views_homepage.py | 2 +- .../views/without_client/test_meeting_view.py | 4 +- .../without_client/test_password_change.py | 10 +- .../without_client/test_password_retrieval.py | 4 +- 20 files changed, 131 insertions(+), 136 deletions(-) diff --git a/i18n.sh b/i18n.sh index 6432bd2..99ebf22 100755 --- a/i18n.sh +++ b/i18n.sh @@ -34,6 +34,21 @@ if [ ! -f "$LOCALES_PATH"/$DOMAIN.pot ]; then touch "$LOCALES_PATH"/$DOMAIN.pot fi +# Function to insert a space before "msgid" +insert_space_before_msgid() { + local input="$1" + echo "$input" | sed 's/msgid/ msgid/' +} + +# Function to remove the fourth space in a string +remove_fourth_space() { + local input="$1" + local part1=$(echo "$input" | cut -d' ' -f1-4) + local part2=$(echo "$input" | cut -d' ' -f5-) + echo "$part1$part2" +} + + # no arguments, extract and update if [ $# -eq 0 ]; then echo "Extract messages" @@ -47,6 +62,18 @@ if [ $# -eq 0 ]; then echo "Compile message catalogs" for po in "$LOCALES_PATH"/*/LC_MESSAGES/*.po; do msgfmt --statistics -o "${po%.*}.mo" "$po" + + untranslated_messages=$(msggrep -v -T -e "." "$po" | grep -n "msgid" | grep -v '""') + if [ -n "$untranslated_messages" ]; then + echo -n "${po}:" + while IFS= read -r line; do + formatted_line=$(insert_space_before_msgid "$line") + formatted_line=$(remove_fourth_space "$formatted_line") + echo "$formatted_line" + done <<< "$untranslated_messages" + echo # Add a newline after all untranslated messages + fi + echo # Add a newline after all untranslated messages done # first argument represents language identifier, create catalog diff --git a/src/privatim/locale/de/LC_MESSAGES/privatim.mo b/src/privatim/locale/de/LC_MESSAGES/privatim.mo index f8ea01e7f0dd61f0eefff1325dd6b3d359f44562..e3c3e83d1c32825e9d35104efacbecb652831f58 100644 GIT binary patch delta 2987 zcmZwIdrZ}39LMnoLAf1;!zB$w!3%OR14^{A873$Qn37r{a_fi)qTCeCJSb!)GOM## z(@0%&VRmDz61H;bs?{8GZ8glLe`G_`Y%Hm1t$KgXdHSQR-}s)_^E|)j{(F8uJUPdg zxGzNnykfXIiOEEJfH9}>Mi4(-S3`~A&s^i@E*!sdGG>CW{aCCa>%c(9H!UPIvsP3`FXC7{gc|TDY9(*l`|n{C`F_*{2az$%MO3|O zwtNKD{x9UujAanDi$YB-86Avo=GuZqDj;8o>Zlksp-NN-n@}s;iQ1`7RQ)3ufp6RV zY19JF;Y9ozGw~O)N$GuS7Zt&1gQB;bNOVj@r6joBt3sp))pr4%L1L>DydJ zP5c+@@2CO)w)r4tt^47qekVEEe?7A}3bc|W)Btm>3sGB^i|S|{s$mtXel2R{+mSKM z0qn$X)WlM`srDJDftR2rx*T=bOC#8SZS78bqXX6PVbnl9HvbN)!vR$NPf-&+Z|{F+ z9Yzgw%ijMJwZL(azV=Q`B0mZB_GGz9B$HT+GjK1a;7RMZw%m_pBvPJ;v#|hGZwDse zah!=?p!)e43owFLL}#E3Ct-`tzl55o`veKCtQWQ7Q>er9G4f~5^22VL3#g8+qfWa& z9|M1MqS{BJ+D%5?pN870H0wgt&K9ED6(jB3rj$e~1?%yIpD{0^4&wr*tF2p#nrRVg z%Zo7$U8sS(Q1!Y|XQT%;P%qMz`2=|i<{Q)kt|FW2{fb5E{g2{vrV2@@jvm6Jn1dQ{ z7}e1TYDa>2a2j9=YNe^D!}kDc;(4f@T#eodV<&kRY9YU(>i>b`_5M3zeFMj!I-H4W zcps|4!$@6|joQ*OR0o^w{Z>@FeW)GiviFZz-$(DEMb$fxTEHcAtKd2b4RjkdupjS+ z8ib-&9D^+0q@i{y&st*Lf_fBfsI#&kOYk++gzlhr#6QkgF9@|$VR7vLJQA@KXuu*= z#R}96t894_szDoSpo6Gq*k$u?p%&1KT0lSQQJ%H=K~%p(Hvc_pyx}}3cc40W)!sjX>gZk6v+uWlg&O!Gs{IvIy_=}^w{Z@dcwR}k7t?ViYC`UI5-B9! zz*HQ>M7)EuFoCa&%*EMQkGlUlCgEook2i4zhA|3{+mxdQ+{F)XoH>SiL<6Xaok2du zZu2DxeK;zPD5@g$K*lo3sYt^>qYBA#%0MuTP*tRTDJF7ggZRd!LjKzq;xIu;oV^+a|msQt|H=T;yI$&^FeTGLO!V{ zh)0R#gs#VYm}=bb@e7%iyqnY}TUvqqhcIn6|GZU4sMNC}B+{v8S#L|l3!dhXRA&RJ zy~MLbCb7}e8xmR4K`MiA6Dx^kVlh!d>?8^ay*0YD^&7k?V;V7yXd(1A=u@zT@F#Tb zA)fLigidr9km5X;XNYV<2PK)PCG=_BOspdGm0CvV-Te2eBEd;BYlz2*N@6>qYo8DA z?+kwHh;76=ViA!`j5nt5qtG7%LYKL!>*DiW)lII3zQf@+{2b*Cl^fz$x*D5nn;QGt zBSyzW)m6A^%IoT=5ue@A*yO6MrS7)qlY#$foDk<6Goh%ta)Ya(%2iQU-Pq{rOP`wI OAHC}T+IuRd_5K4i?c)4BamgrOS~lc5^$pwh1MwV zYT1}NiV3-y?O`oM*I1aFi)fb33Ddbr&h@s~V1u>l`{Oxn-97*3bI!f@obx~Do_jlU z-5lpiZ0P%jqm{UWs0cOYD?A;+AIFzb#_%(}{M~|Q@irX6+i?QZF=nPQGw=aa)72P` z>oF3Ga1Q#ACZ-ajjd9Fwd!iL%xbT6kcOgI1!(SKnqXsCO6-cB86R00R4<1Gh)P;0x z{HTEjkU5#}QSC2cIgVgBs0#F>n5!dMrbh(&dngzCtPO6VR`2id5Z<)K!}hq}KCW3k@W z+fWl|$GO;nEASNR(fo-@YziF}g6LHVL|_TV+j<>p>6&c48I{mMRJ(Sh5A!i9VZXHx zHPF|#{;l;#RDUC=M|CZl_16r3<%Bw(w1zM}E!8YkM=7X=Uex`Ws2OJ?O-v!agQcj% z{y?>#KqVH%%}O*5waM>9t>{CstiLW4a6%myqXw$5^%_)%ji?0o+4F7IcGLjf_WI|j znf0OCeS_&Zh|_NiW>AlbGiD*KbSNyQP-1=CUibnRaef$=U@+6seRtym+>EJMkLsuc z^YI+&nJ4gS+=@$4^&C{9TTqE@Lrv7Fq@c}Gi>#K}g=)}>+SMHxf+tZ8dr^a8_Gn$s{1}eib)KX7jCWbN%b+8XW2gbnpjP5M zYO`HJy)8FUOE`|x38OY)C=Xf_If%OdBUJn2sDXQJeE>ar|1VHbgAwFsuJflQ{Rh>- z?0JFf$*7LfP%E$;b$yk!0M)Jpbzc=~0u8p_iW=ubRKiD5A2ahQ1caE#*z? zU)DLiJ9-o;sJ)VbK3s`P=n!f}I#Bl=L#@;aybn*I#v4Q37nEq<{~3va3voC-0P5Lg z*m^c9^EIf<^H58<$<|9z9hTdAHEN(ew%&qD#6`V5M^Fj(C$j!3oVPdpfZ79BQ7bTk zZRkm|GekY24pgFNP>Bp6KQmB_z zX1pDBUoEO(1Kx#uF$MjYg+HMZiI^YwUZkOydJ!(dMohy_t20Pp2`9!;H>9vW>9`I# zWomIFy2xWQqo@Ic`Qybgsi;S@3YAz6>Wi9>`cC*zdu1!KjZ7VCW#2=-503eSf*N+A zp5bSx4tr25aN5?-qIUHl>ccXOYIhYifuAu5*AXueN;Z{fBAzBRxniQ0(9uHd&`9$s zY;{vTY5C7mT1?~;|3B`f@+7g8SWUDNI_?i(O0b^Le6?+L>~arz=4HJcsF)n9R#wNO zgjPdG5%E0nCb8Ka^>`ONM(HWy5n=LL5M_kkuS((xLSLixgm%n- z$BPuS!!{9*6WfWMgpPdyrvDf6UqRFnKH_0wEx{)8--sFw^_3s572=MkoctFIe+kL} diff --git a/src/privatim/locale/de/LC_MESSAGES/privatim.po b/src/privatim/locale/de/LC_MESSAGES/privatim.po index f7dd1a5..9f8228b 100644 --- a/src/privatim/locale/de/LC_MESSAGES/privatim.po +++ b/src/privatim/locale/de/LC_MESSAGES/privatim.po @@ -5,7 +5,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2024-07-02 14:17+0200\n" +"POT-Creation-Date: 2024-07-02 23:44+0200\n" "PO-Revision-Date: 2024-05-21 21:20+0200\n" "Last-Translator: cyrill \n" "Language-Team: German \n" @@ -338,6 +338,10 @@ msgstr "Ihr Kommentar" msgid "Search Results" msgstr "Suchergebnisse" +#: src/privatim/views/templates/search_results.pt +msgid "No results containing all your search terms were found." +msgstr "Es wurden keine Ergebnisse gefunden, die alle Ihre Suchbegriffe enthalten." + #: src/privatim/views/templates/search_results.pt #: src/privatim/views/templates/activities.pt msgid "Show Details" diff --git a/src/privatim/locale/fr/LC_MESSAGES/privatim.mo b/src/privatim/locale/fr/LC_MESSAGES/privatim.mo index 229aadaa51ba602d9e2f5d9c12d6e3a59b4a81d3..542a64e3dfab0f40914499912944eba195c8c27b 100644 GIT binary patch delta 2995 zcmXxk4NO&K9LMnoL6Lj0xEIM6Brm=oh&gpiuC2^>NI_9cO%bkeRP=hy%S92fYrba6 zX5O`$j-{nbO6M4s<}5SUi_4kiDzh!ysFc!}PUmK&eSh~pot^*rJkL4jd4JA1R`^X( zZk z4frrNqUvkHG-D#B+1@yT9v*yd^Oum*{L1AZ-b6L9yIZ^?`!Iw2aZJJUsD>^f6`M9x zL${GJ8S_xQd}mxnHWia;-_(&%&(@$zCsFA#5pC7^Q2#0ak@di1Xa;IREH{26)Z=MtN}Gsji~qEL@yq) z`BSI?oWmaYEf(M(sHMrK5_N2_=3j~7B-EqPSb-C4{s3y~T5SFUREIvb`E#i9myo*6 z6;#JNtanijKCt<&^jgo;Q1$loF#lS!EN*Bdxu^z4S;wHJtO!-n0#w1psP_Y?k*`78 zFgvgjn@}Cg=b_4vKs7uW)zN9F&0g+h{x!7?_Q7^k#d}Z<9kltws0vS@-an1%=mq=y zvh^yeq1*QPKd1qANspKJU@rNdsIO;ighU>R`8WW#;9z{u`n|oMz%X*SpMyiN81-H~ zX5#@Ih+m-Uxq-##$qMtmH#c|Jl;bA}7EX?{di^e1Y! zC-O5$L=UQb2C7^i)boC*nHpvtgPPeARJnObxriwzkW8XeC~A!hPz}#PHSE~uH8#H%)!}WZrHdllz%*MwM)sq*h_M;Qh!VF*sN(ym3fz1f z8c_!J#{tNsm_k&Ae)Qogn|~d(2M(bcZbc@^oJ9?&4K)i+m z^`wh0ULX@=Yme&KP-LH&0@N-qMK!z-Rjv-zu}0MUyHO*23)O+c_yitBPSe3lXei`Zqv&Y{51oeJ9F2wZy@flc!Op4iQ^G8ri za~9RHi%7>K<}wL=E>}@Mnj5GQKfrwKO=}wQBuvEls0u1D2^XVA9zgAZuyq}(zAc!9 zZ=lLG;Sk)93FdKQo*}b{U{l7XWE0LMa)=5dOz7A_tRj4b6TQ=QQ1N_H+AW2Ij`gvt zF(dIg;xS?@(MaeR9mn|bMMA^YHr7!etxm}tzATN~se5ptv&s3zIEz{bX6^6Pd&Ze&74(N= o_XB=k$l)P1I;~&!ZA$cv51ncDhn?DxuR2({A{eZVR`tL5AKcb7r2qf` delta 2844 zcmYk;e@vBC9LMnk7XmJl+>2zQ7|QQ1F40ELMhhiHk@%q%X$V;U2qA?0nZy+gLs&Vx z%Z)~C;c8^|!(A$swnmn&CH-LC)ri@gI{wI#{DaazsP~6^+Iq(Kd7bk-&pGFN&U2nS zepvNsf%lR->~+J@LChg)!;JYHKa1qVaXQ)<{>&Lpci~q!3r8^lColtD(~P+Tm!g{H zVFa#5Czj!KtU#KWI*c{OYxda-9q8i5FoXHQ0fg$sT*(i!-PnMkUyT^kv>d zJ$J@lA4IkP7Wp#)PHJ}*mDnwu#`q?jb8Xcjy$M@7Na`IMa^stYNaYr&)1_H zn{53EY69Ij6MHcS&!S$bMd$P_3;upgL?uCD>-KAF+0$26)%r zKZTmv0IJ=2%)lX>`dToPx+~6@6nxN2;a&<=)=qol6HMd!FPMd)Oh?b%hx2efreYJS zqh2h+3#fM<&!=%WEiqTV_R+ANL8YMH&L1|6tf-HQ(V2-UD3_56Un zKZsh&@2rGjLAm!oY%B5nld|T?LrOw7N+Be_WDn#cm1oa zkE53QZ`5-RrmG()4{GMiII(zU1G4I-9(BJJm2ek+qwl|ug5K>;e%2DP+1ibo$yroF z7f=lYsITQJY9+=|6PUn*IBD;<^1{i@yoyTn9n?ykLT&2P7_IMrkb*iGLcQAnYT&<6 z1BcBH-gl$w38;iKkX1E{kk@VstQDvkH=+`4vmQnD`zEUW3Dl37>8Fs4=a9uPW2g@G ztt3G_s-A{KnFp$Xlt1!w14SWUX z;y7xC)2ZCRR8*qz_XKC0jB1yQ`u)hqG+c*T(f!sAR3crd=RZIt@-b>+gI)^S)nB6q z96>c0MglLgl7~vH5S4ft>g%XP_OaQF9z1}W=t*?w z`~QN18lFdOiXqg}45Qxpi1iBU2kJWNxk*$z2U%yK6GLzn@eIKpn_7t$EGM)Yl|&<< z<1n#PBPCII+Ls!YUbKOdw#Q21|Hu7Q9wQbIc|-@HV@VKGg-wL!t8J@eudgd=cJ{Nu zidkXRlIwVc;3Jzl$~bFI*Nmst@yGp+nJu(OliBlT8-K^ z2Wj)iRTF&PZ2K= zMTEX%9a{P=Qzc^>a3Qgq&@O*~c#d!oI=sY_zS8KK-ZhliNM;|APiRME5;cT&OC7O} z&<1PPbB_~iiEYFlLPuMWsdEbFT4EPbK`bK*h#3A+8H>Ia=KneNw-EoZ a`(~(rB>w#fzc2B4sJ}F++u{2sdH7${*$N~8 diff --git a/src/privatim/locale/fr/LC_MESSAGES/privatim.po b/src/privatim/locale/fr/LC_MESSAGES/privatim.po index 7a1aaf7..eaec1a2 100644 --- a/src/privatim/locale/fr/LC_MESSAGES/privatim.po +++ b/src/privatim/locale/fr/LC_MESSAGES/privatim.po @@ -5,7 +5,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2024-07-02 14:17+0200\n" +"POT-Creation-Date: 2024-07-02 23:44+0200\n" "PO-Revision-Date: 2024-04-11 15:53+0200\n" "Last-Translator: cyrill \n" "Language-Team: French \n" @@ -335,7 +335,11 @@ msgstr "Votre commentaire" #: src/privatim/views/templates/search_results.pt msgid "Search Results" -msgstr "" +msgstr "Résultats de la recherche" + +#: src/privatim/views/templates/search_results.pt +msgid "No results containing all your search terms were found.." +msgstr "Aucun résultat ne contient tous les termes de votre recherche." #: src/privatim/views/templates/search_results.pt #: src/privatim/views/templates/activities.pt @@ -348,11 +352,11 @@ msgstr "Changer le mot de passe" #: src/privatim/views/templates/activities.pt msgid "Unbekannter Ersteller" -msgstr "" +msgstr "Créateur inconnu" #: src/privatim/views/templates/activities.pt msgid "Kein Leiter" -msgstr "" +msgstr "Pas de chef" #: src/privatim/views/templates/activities.pt msgid "Show Meeting" @@ -607,7 +611,7 @@ msgstr "Membres" #: src/privatim/forms/search_form.py msgid "Search" -msgstr "" +msgstr "Suchen" #: src/privatim/forms/widgets/widgets.py msgid "Uploaded file" diff --git a/src/privatim/locale/privatim.pot b/src/privatim/locale/privatim.pot index a7acb9c..9b1d167 100644 --- a/src/privatim/locale/privatim.pot +++ b/src/privatim/locale/privatim.pot @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE 1.0\n" -"POT-Creation-Date: 2024-07-02 14:17+0200\n" +"POT-Creation-Date: 2024-07-02 23:44+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -328,6 +328,10 @@ msgstr "" msgid "Search Results" msgstr "" +#: ./src/privatim/views/templates/search_results.pt +msgid "No results containing all your search terms were found.." +msgstr "" + #: ./src/privatim/views/templates/search_results.pt #: ./src/privatim/views/templates/activities.pt msgid "Show Details" diff --git a/src/privatim/models/searchable.py b/src/privatim/models/searchable.py index 7ca485b..8612126 100644 --- a/src/privatim/models/searchable.py +++ b/src/privatim/models/searchable.py @@ -56,8 +56,6 @@ def reindex_full_text_search(session: 'Session') -> None: """ models = searchable_models() - # todo: remove later - assert len(models) != 0, "No models with searchable fields found" for model in models: for locale, language in locales.items(): assert language == 'german' # todo: remove later diff --git a/src/privatim/views/templates/search_results.pt b/src/privatim/views/templates/search_results.pt index 95f0a96..12f5b8d 100644 --- a/src/privatim/views/templates/search_results.pt +++ b/src/privatim/views/templates/search_results.pt @@ -14,6 +14,13 @@
    + +
    + +
    diff --git a/tests/cli/test_upgrade.py b/tests/cli/test_upgrade.py index e16c571..86a4d6d 100644 --- a/tests/cli/test_upgrade.py +++ b/tests/cli/test_upgrade.py @@ -1,21 +1,21 @@ from privatim.cli.upgrade import UpgradeContext -def test_has_table(config): - upgrade = UpgradeContext(config.dbsession) +def test_has_table(pg_config): + upgrade = UpgradeContext(pg_config.dbsession) assert upgrade.has_table('consultations') assert not upgrade.has_table('bogus') -def test_drop_table(config): - upgrade = UpgradeContext(config.dbsession) - assert upgrade.has_table('meetings') - assert upgrade.drop_table('meetings') +def test_drop_table(pg_config): + upgrade = UpgradeContext(pg_config.dbsession) + assert upgrade.has_table('agenda_items') + assert upgrade.drop_table('agenda_items') assert not upgrade.has_table('meetings') assert not upgrade.drop_table('bogus') -def test_has_column(config): - upgrade = UpgradeContext(config.dbsession) +def test_has_column(pg_config): + upgrade = UpgradeContext(pg_config.dbsession) assert upgrade.has_column('meetings', 'id') assert not upgrade.has_column('meetings', 'bogus') diff --git a/tests/conftest.py b/tests/conftest.py index a37b22a..28b4b81 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,6 @@ import warnings import pytest import transaction -from libcloud.storage.drivers.local import LocalStorageDriver from pyramid import testing from sqlalchemy import engine_from_config from privatim import main @@ -10,30 +9,19 @@ from privatim.models.consultation import Status, Consultation from privatim.orm import Base, get_engine, get_session_factory, get_tm_session from privatim.testing import DummyRequest, DummyMailer, MockRequests -from sqlalchemy_file.storage import StorageManager - from tests.shared.client import Client -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from pyramid.config import Configurator - - -@pytest.fixture -def base_config(_postgresql): +@pytest.fixture(scope='function') +def base_config(postgresql): msg = '.*SQLAlchemy must convert from floating point.*' warnings.filterwarnings('ignore', message=msg) - # config = testing.setUp(settings={ - # 'sqlalchemy.url': 'sqlite:///:memory:', - # }) - config = testing.setUp(settings={ 'sqlalchemy.url': ( - f'postgresql+psycopg://{_postgresql.info.user}:@' - f'{_postgresql.info.host}:{_postgresql.info.port}' - f'/{_postgresql.info.dbname}' + f'postgresql+psycopg://{postgresql.info.user}:@' + f'{postgresql.info.host}:{postgresql.info.port}' + f'/{postgresql.info.dbname}' ), }) yield config @@ -41,59 +29,25 @@ def base_config(_postgresql): transaction.abort() -@pytest.fixture -def config(base_config, monkeypatch, tmpdir) -> 'Configurator': - """ Returns the config used in tests. Note that this has side effects - on `DummyRequest` in that the session becomes available. """ - - base_config.include('privatim.models') - base_config.include('pyramid_chameleon') - base_config.include('pyramid_layout') - settings = base_config.get_settings() - - engine = get_engine(settings) - - # enable foreign key constraints in sqlite, so we can rely on them - # working during testing. - # sqlalchemy.event.listen( - # engine, - # 'connect', - # lambda c, r: c.execute('pragma foreign_keys=ON') - # ) - - Base.metadata.create_all(engine) - session_factory = get_session_factory(engine) - - dbsession = get_tm_session(session_factory, transaction.manager) - base_config.dbsession = dbsession - - orig_init = DummyRequest.__init__ - - def init_with_dbsession(self, *args, dbsession=dbsession, **kwargs): - orig_init(self, *args, dbsession=dbsession, **kwargs) - - monkeypatch.setattr(DummyRequest, '__init__', init_with_dbsession) - - # Store static files in a temporary directory - if not StorageManager._storages: - # NOTE: StorageManager does not expose any method to check if some - # storage has already been added. However, if you attempt to add a - # storage that already exists, StorageManager raises a RuntimeError. - tmpdir.mkdir('assets') - container = LocalStorageDriver(tmpdir).get_container('assets') - StorageManager.add_storage("default", container) +@pytest.fixture(scope='function', autouse=True) +def run_around_tests(engine): + # todo: check if this is actually needed? - return base_config + # This fixture will run before and after each test + # Thanks to the autouse=True parameter + yield + # After the test, we ensure all tables are dropped + Base.metadata.drop_all(bind=engine) # requires pytest-postgresql: -@pytest.fixture(scope='session') -def pg_config(_postgresql, monkeypatch): +@pytest.fixture(scope='function') +def pg_config(postgresql, monkeypatch): config = testing.setUp(settings={ 'sqlalchemy.url': ( - f'postgresql+psycopg://{_postgresql.info.user}:@' - f'{_postgresql.info.host}:{_postgresql.info.port}' - f'/{_postgresql.info.dbname}' + f'postgresql+psycopg://{postgresql.info.user}:@' + f'{postgresql.info.host}:{postgresql.info.port}' + f'/{postgresql.info.dbname}' ), }) config.include('privatim.models') @@ -125,13 +79,13 @@ def init_with_dbsession(self, *args, dbsession=dbsession, **kwargs): transaction.abort() -@pytest.fixture -def session(config): +@pytest.fixture(scope='function') +def session(pg_config): # convenience fixture - return config.dbsession + return pg_config.dbsession -@pytest.fixture +@pytest.fixture(scope='function') def user(session): user = User( email='admin@example.org', @@ -169,13 +123,13 @@ def user_with_working_group(session): @pytest.fixture -def mailer(config): +def mailer(pg_config): mailer = DummyMailer() - config.registry.registerUtility(mailer) + pg_config.registry.registerUtility(mailer) return mailer -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def engine(app_settings): engine = engine_from_config(app_settings) Base.metadata.create_all(engine) @@ -186,40 +140,35 @@ def engine(app_settings): return engine -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def connection(engine): connection = engine.connect() yield connection connection.close() -@pytest.fixture(scope='session') -def _postgresql(postgresql): - return postgresql - - -@pytest.fixture(scope='session') -def app_settings(_postgresql): +@pytest.fixture(scope='function') +def app_settings(postgresql): yield {'sqlalchemy.url': ( - f'postgresql+psycopg://{_postgresql.info.user}:@' - f'{_postgresql.info.host}:{_postgresql.info.port}' - f'/{_postgresql.info.dbname}' + f'postgresql+psycopg://{postgresql.info.user}:@' + f'{postgresql.info.host}:{postgresql.info.port}' + f'/{postgresql.info.dbname}' )} -@pytest.fixture(scope="session") +@pytest.fixture(scope='function') def app_inner(app_settings): app = main({}, **app_settings) yield app -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def app(app_inner, connection): app_inner.app.app.registry["dbsession_factory"].kw["bind"] = connection yield app_inner -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def client(app, engine): client = Client(app) diff --git a/tests/forms/test_forms_meeting.py b/tests/forms/test_forms_meeting.py index 8863e09..9dfe90d 100644 --- a/tests/forms/test_forms_meeting.py +++ b/tests/forms/test_forms_meeting.py @@ -17,7 +17,7 @@ def getlist(self, key): @pytest.mark.skip() -def test_meeting_form_time_not_optional(config): +def test_meeting_form_time_not_optional(pg_config): meeting = create_meeting() session = config.dbsession session.add(meeting) diff --git a/tests/i18n/test_core.py b/tests/i18n/test_core.py index 25c0b98..cc03be2 100644 --- a/tests/i18n/test_core.py +++ b/tests/i18n/test_core.py @@ -40,28 +40,28 @@ def test_translate_markup(): assert result == Markup('bold') -def test_translate_translation_dirs(config): - config.add_translation_dirs('privatim:locale/') +def test_translate_translation_dirs(pg_config): + pg_config.add_translation_dirs('privatim:locale/') # Testing localizer doesn't seem to work. - config.registry.localizer_de = None + pg_config.registry.localizer_de = None msg = _('Just a test') assert translate(msg, 'en') == 'Just a test' assert translate(msg, 'de') == 'Nur ein Test' -def test_translate_translation_dirs_markup(config): - config.add_translation_dirs('privatim:locale/') +def test_translate_translation_dirs_markup(pg_config): + pg_config.add_translation_dirs('privatim:locale/') # Testing localizer doesn't seem to work. - config.registry.localizer_de = None + pg_config.registry.localizer_de = None msg = _('bold', markup=True) assert escape(translate(msg, 'en')) == Markup('bold') assert escape(translate(msg, 'de')) == Markup('fett') -def test_translate_translation_dirs_markup_omitted(config): - config.add_translation_dirs('privatim:locale/') +def test_translate_translation_dirs_markup_omitted(pg_config): + pg_config.add_translation_dirs('privatim:locale/') # Testing localizer doesn't seem to work. - config.registry.localizer_de = None + pg_config.registry.localizer_de = None msg = _('bold') assert escape(translate(msg, 'en')) == Markup('<b>bold</b>') assert escape(translate(msg, 'de')) == Markup('<b>fett</b>') diff --git a/tests/models/test_searchable_mixin.py b/tests/models/test_searchable_mixin.py index 24ef113..dbe16be 100644 --- a/tests/models/test_searchable_mixin.py +++ b/tests/models/test_searchable_mixin.py @@ -1,9 +1,11 @@ +import pytest from sqlalchemy import select from sqlalchemy.orm import undefer from privatim.models import Consultation from privatim.models.searchable import reindex_full_text_search +@pytest.mark.skip('fulltext indexing is disabled for now') def test_fulltext_indexing_on_searchable_fields(pg_config): session = pg_config.dbsession consultation = Consultation( diff --git a/tests/models/test_user.py b/tests/models/test_user.py index a46ae35..c6d3366 100644 --- a/tests/models/test_user.py +++ b/tests/models/test_user.py @@ -2,8 +2,8 @@ from sqlalchemy import select -def test_set_password(config): - session = config.dbsession +def test_set_password(pg_config): + session = pg_config.dbsession user = User(email='admin@example.org') session.add(user) @@ -13,8 +13,8 @@ def test_set_password(config): assert user.check_password('Test123!') is True -def test_user_password_failure(config): - session = config.dbsession +def test_user_password_failure(pg_config): + session = pg_config.dbsession user = User(email='admin@example.org') session.add(user) session.flush() @@ -24,16 +24,16 @@ def test_user_password_failure(config): assert user.check_password('Test123!') is True -def test_user_groups_empty(config): - session = config.dbsession +def test_user_groups_empty(pg_config): + session = pg_config.dbsession user = User(email='admin@example.org') session.add(user) session.flush() assert user.groups == [] # No groups associated initially -def test_user_groups_association(config): - session = config.dbsession +def test_user_groups_association(pg_config): + session = pg_config.dbsession user = User(email='admin@example.org') group = Group(name='Test Group') @@ -45,8 +45,8 @@ def test_user_groups_association(config): assert user.groups == [group] -def test_user_leading_group_relationship(config): - session = config.dbsession +def test_user_leading_group_relationship(pg_config): + session = pg_config.dbsession user = User(email='admin@example.org') group = WorkingGroup(name='Leadership Group') @@ -62,8 +62,8 @@ def test_user_leading_group_relationship(config): assert group.leader.email == 'admin@example.org' -def test_group_type_polymorphism(config): - session = config.dbsession +def test_group_type_polymorphism(pg_config): + session = pg_config.dbsession group = Group(name='General Group') working_group = WorkingGroup(name='Specific Working Group') session.add(group) diff --git a/tests/reporting/test_report.py b/tests/reporting/test_report.py index 287d372..96f1990 100644 --- a/tests/reporting/test_report.py +++ b/tests/reporting/test_report.py @@ -5,7 +5,7 @@ from tests.shared.utils import create_meeting, CustomDummyRequest -def test_generate_meeting_report(config): +def test_generate_meeting_report(pg_config): meeting = create_meeting() renderer = HTMLReportRenderer() diff --git a/tests/views/client/test_logout.py b/tests/views/client/test_logout.py index eced534..8ffe9e8 100644 --- a/tests/views/client/test_logout.py +++ b/tests/views/client/test_logout.py @@ -2,7 +2,7 @@ from privatim.views import logout_view -def test_logout_view(config): +def test_logout_view(pg_config): config.add_route('login', '/login') request = DummyRequest() diff --git a/tests/views/client/test_views_homepage.py b/tests/views/client/test_views_homepage.py index ce24b68..c1cf2d7 100644 --- a/tests/views/client/test_views_homepage.py +++ b/tests/views/client/test_views_homepage.py @@ -10,5 +10,5 @@ def test_view_activities(client): form = page.forms[0] print(form.fields) - form['search'] = 'My search' + form['term'] = 'My search' page = form.submit() diff --git a/tests/views/without_client/test_meeting_view.py b/tests/views/without_client/test_meeting_view.py index b3954f6..211e920 100644 --- a/tests/views/without_client/test_meeting_view.py +++ b/tests/views/without_client/test_meeting_view.py @@ -8,7 +8,7 @@ create_meeting_with_agenda_items -def test_export_meeting_without_agenda_items(config): +def test_export_meeting_without_agenda_items(pg_config): config.add_route('export_meeting_as_pdf_view', '/meetings/{id}/export') @@ -30,7 +30,7 @@ def test_export_meeting_without_agenda_items(config): assert 'Logo' in all_text -def test_sortable_agenda_items_view(config): +def test_sortable_agenda_items_view(pg_config): # Add route config.add_route( diff --git a/tests/views/without_client/test_password_change.py b/tests/views/without_client/test_password_change.py index c2c679a..43a4e54 100644 --- a/tests/views/without_client/test_password_change.py +++ b/tests/views/without_client/test_password_change.py @@ -7,7 +7,7 @@ from privatim.views.password_change import password_change_view -def test_password_change_view(config): +def test_password_change_view(pg_config): request = DummyRequest() result = password_change_view(request) assert 'form' in result @@ -35,7 +35,7 @@ def test_password_change_view_password_strength(config, user): assert 'Password must have minimal length' in messages[0]['message'] -def test_password_change_view_invalid(config): +def test_password_change_view_invalid(pg_config): request = DummyRequest() request.params['token'] = '1234567' request.POST['email'] = 'username' @@ -48,7 +48,7 @@ def test_password_change_view_invalid(config): assert 'There was a problem with your submission.' in message['message'] -def test_password_change_view_invalid_confirmation(config): +def test_password_change_view_invalid_confirmation(pg_config): request = DummyRequest() request.params['token'] = '1234567' request.POST['email'] = 'username' @@ -61,7 +61,7 @@ def test_password_change_view_invalid_confirmation(config): assert 'There was a problem with your submission.' in message['message'] -def test_password_change_view_invalid_min_length(config): +def test_password_change_view_invalid_min_length(pg_config): request = DummyRequest() request.params['token'] = '1234567' request.POST['email'] = 'username' @@ -96,7 +96,7 @@ def test_password_change_view_invalid_username(config, user): assert 'form' in result -def test_password_change_view_invalid_token(config): +def test_password_change_view_invalid_token(pg_config): request = DummyRequest() request.params['token'] = '123456' request.POST['email'] = 'username' diff --git a/tests/views/without_client/test_password_retrieval.py b/tests/views/without_client/test_password_retrieval.py index 07c822c..a386062 100644 --- a/tests/views/without_client/test_password_retrieval.py +++ b/tests/views/without_client/test_password_retrieval.py @@ -4,7 +4,7 @@ from privatim.views.password_retrieval import password_retrieval_view -def test_view(config): +def test_view(pg_config): request = DummyRequest() result = password_retrieval_view(request) assert 'form' in result.keys() @@ -64,7 +64,7 @@ def test_view_submit_username(config, user, mailer): assert message['receivers'].addr_spec == 'gregory@house.com' -def test_view_submit_invalid(config): +def test_view_submit_invalid(pg_config): request = DummyRequest() request.POST['email'] = '' # Empty username request.POST['submit'] = '1' From f9a64e8dc22a058e04ace8b99c83b2bd6ae623ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Wed, 3 Jul 2024 10:15:15 +0200 Subject: [PATCH 19/52] Cosmetics --- src/privatim/views/search.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/privatim/views/search.py b/src/privatim/views/search.py index de227ac..91df8a2 100644 --- a/src/privatim/views/search.py +++ b/src/privatim/views/search.py @@ -69,7 +69,7 @@ def do_search(self) -> None: self._add_comments_to_results() - def _add_comments_to_results(self): + def _add_comments_to_results(self) -> None: """ Extends self.results with the complete query for Comment. This is for displaying more information in the search results. @@ -91,8 +91,8 @@ def search_model( self, model: type['HasSearchableFields'], ) -> List[SearchResult]: - query = self.build_query(model) - raw_results = self.session.execute(query).all() + model_fulltext_query = self.build_query(model) + raw_results = self.session.execute(model_fulltext_query).all() return self.process_results(raw_results, model) def build_query( @@ -104,13 +104,12 @@ def build_query( Builds the actual query for full text search. 1. Generate headline expressions for all searchable fields of the - model. Headlines in this context are snippets of text from the searchable - fields, with the matching search terms highlighted. - They provide context around where the search term appears in each - field. + model. Headlines in this context are snippets of text from the + searchable fields, with the matching search terms highlighted. They + provide context around where the search term appears in each field. - 2. Perform the actual search in all - searchable fields using `create_fulltext_search_conditions` + 2. Perform the actual search in all searchable fields using + the filters of `create_fulltext_search_conditions` Returns A list of SQLAlchemy column expressions, each representing a headline for a searchable field. @@ -145,13 +144,17 @@ def create_fulltext_search_conditions( searchable_fields: Iterator['InstrumentedAttribute[str]'], ) -> List[ColumnElement[bool]]: """ - The column.op@@ expression is SQLAlchemy's custom operator - functionality to create a full-text search operation. + Returns a list of SqlAlchemy filter statements matching possible + fulltext attributes based on the term. - Note that we convert to tsvector at runtime, this could be done at - indexing time for performance reasons. But for now we keep it simple. - For relatively small datasets, as we expect them to be this will - probably not be a bottleneck. + This is somewhat inspired from + `onegov.swissvotes.collections.votes.SwissVoteCollection`. + + **Note:** We convert the searchable fields to `tsvector` at runtime. + This could be precomputed for example using SQLAlchemy's `observes` for + better performance. For simplicity, we're doing it during the search. + Given our relatively small dataset, this approach should not be a + problem. """ @@ -201,7 +204,6 @@ def search(request: 'IRequest') -> 'RenderDataOrRedirect': redirects to avoid browser warnings on page refresh. This prevents accidental form resubmission if users refresh the results page. - """ session: Session = request.dbsession form: SearchForm = SearchForm(request) From 926ac25d5cb1f76c42540fdc2dcf20716db81182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cyrill=20K=C3=BCttel?= Date: Wed, 3 Jul 2024 10:54:18 +0200 Subject: [PATCH 20/52] Update readme with instructions for setup --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d6d1f06..a2565b8 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,19 @@ Getting Started --------------- ``` +sudo apt install postgresql libpq-dev python3-dev build-essential weasyprint +``` + +Create the PostgreSQL database: +``` +sudo -u postgres bash -c "psql < Date: Wed, 3 Jul 2024 16:47:26 +0200 Subject: [PATCH 21/52] finish migration, fix mypy, add SearchableFile. --- README.md | 2 +- setup.cfg | 1 - src/privatim/cli/initialize_db.py | 5 +- src/privatim/cli/reindex.py | 22 ---- src/privatim/forms/fields/fields.py | 22 ++-- src/privatim/layouts/action_menu.pt | 2 +- src/privatim/layouts/footer.pt | 1 + src/privatim/layouts/layout.pt | 19 +-- src/privatim/layouts/navbar.pt | 2 +- .../locale/de/LC_MESSAGES/privatim.mo | Bin 9125 -> 9272 bytes .../locale/de/LC_MESSAGES/privatim.po | 7 +- .../locale/fr/LC_MESSAGES/privatim.mo | Bin 9425 -> 9591 bytes .../locale/fr/LC_MESSAGES/privatim.po | 4 +- src/privatim/locale/privatim.pot | 4 +- src/privatim/models/__init__.py | 34 +----- src/privatim/models/associated_file.py | 111 ++++++++++++------ src/privatim/models/consultation.py | 11 +- src/privatim/models/file.py | 36 +++--- src/privatim/models/meeting.py | 10 +- src/privatim/models/searchable.py | 53 +-------- src/privatim/models/utils.py | 2 +- src/privatim/static/css/custom.css | 11 +- src/privatim/types.py | 5 - src/privatim/views/consultations.py | 14 ++- src/privatim/views/search.py | 26 ++-- .../views/templates/search_results.pt | 2 +- .../views/templates/working_groups.pt | 2 +- tests/forms/test_forms_meeting.py | 4 +- tests/models/test_searchable_mixin.py | 52 ++++---- tests/shared/utils.py | 15 ++- tests/views/client/test_logout.py | 2 +- .../views/without_client/test_meeting_view.py | 9 +- .../without_client/test_password_change.py | 34 +++--- .../without_client/test_password_retrieval.py | 20 ++-- 34 files changed, 258 insertions(+), 286 deletions(-) delete mode 100644 src/privatim/cli/reindex.py diff --git a/README.md b/README.md index a2565b8..0bb42eb 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ make run ## Run the project's tests ``` -pytest -n auto +pytest -n auto ``` diff --git a/setup.cfg b/setup.cfg index 6302486..9f2b6b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,7 +84,6 @@ console_scripts = add_meeting = privatim.cli.add_meeting:main delete_meetings = privatim.cli.delete_meetings:main upgrade = privatim.cli.upgrade:upgrade - reindex = privatim.cli.reindex:reindex shell = privatim.cli.shell:shell [options.extras_require] diff --git a/src/privatim/cli/initialize_db.py b/src/privatim/cli/initialize_db.py index 190deb7..b1ce022 100644 --- a/src/privatim/cli/initialize_db.py +++ b/src/privatim/cli/initialize_db.py @@ -5,9 +5,10 @@ from privatim.layouts.layout import DEFAULT_TIMEZONE from privatim.models.consultation import Status, Tag +from privatim.models.file import SearchableFile from privatim.orm import get_engine from privatim.models import (Consultation, User, Meeting, WorkingGroup, - AgendaItem, GeneralFile) + AgendaItem) from privatim.orm import Base import click @@ -103,7 +104,7 @@ def add_example_content( pdf = here / 'sample-pdf-for-initialize-db/' / pdfname content = pdf.read_bytes() consultation = Consultation( - files=[GeneralFile(filename=pdfname, content=content)], + files=[SearchableFile(filename=pdfname, content=content)], title='Verordnung über den Einsatz elektronischer Mittel zur Ton- ' 'und Bildübertragung in Zivilverfahren (VEMZ)', description='Mit der Revision der Schweizerischen ' diff --git a/src/privatim/cli/reindex.py b/src/privatim/cli/reindex.py deleted file mode 100644 index 25f71ea..0000000 --- a/src/privatim/cli/reindex.py +++ /dev/null @@ -1,22 +0,0 @@ -import click -import transaction -from pyramid.paster import bootstrap -from pyramid.paster import get_appsettings -from privatim.models.searchable import reindex_full_text_search -from privatim.orm import get_engine, Base, get_session_factory, get_tm_session - - -@click.command() -@click.argument('config_uri') -def reindex(config_uri: str) -> None: - - bootstrap(config_uri) # is this needed? - settings = get_appsettings(config_uri) - engine = get_engine(settings) - Base.metadata.create_all(engine) - - session_factory = get_session_factory(engine) - - with transaction.manager: - dbsession = get_tm_session(session_factory, transaction.manager) - reindex_full_text_search(dbsession) diff --git a/src/privatim/forms/fields/fields.py b/src/privatim/forms/fields/fields.py index 02c72de..6a64ba9 100644 --- a/src/privatim/forms/fields/fields.py +++ b/src/privatim/forms/fields/fields.py @@ -3,6 +3,8 @@ import sedate from sqlalchemy import select + +from privatim.models.file import SearchableFile from privatim.static import init_tom_select from wtforms.utils import unset_value from wtforms.validators import DataRequired @@ -384,33 +386,33 @@ def append_entry_from_field_storage( # we fake the formdata for the new field # we use a werkzeug MultiDict because the WebOb version # needs to get wrapped to be usable in WTForms - formdata: MultiDict[str, RawFormValue] = MultiDict() + formdata: 'MultiDict[str, RawFormValue]' = MultiDict() name = f'{self.short_name}{self._separator}{len(self)}' formdata.add(name, fs) return self._add_entry(formdata) class _DummyFile: - file: GeneralFile | None + file: SearchableFile | None class UploadFileWithORMSupport(UploadField): - """ Extends the upload field with onegov.file support. """ + """ Extends the upload field with file support. """ - file_class: type[GeneralFile] + file_class: type[SearchableFile] def __init__(self, *args: Any, **kwargs: Any): self.file_class = kwargs.pop('file_class') super().__init__(*args, **kwargs) - def create(self) -> GeneralFile | None: + def create(self) -> SearchableFile | None: if not getattr(self, 'file', None): return None assert self.file is not None self.file.seek(0) assert self.filename is not None - return GeneralFile(filename=self.filename, content=self.file.read()) + return SearchableFile(filename=self.filename, content=self.file.read()) def populate_obj(self, obj: object, name: str) -> None: @@ -429,7 +431,7 @@ def populate_obj(self, obj: object, name: str) -> None: else: raise NotImplementedError(f"Unknown action: {self.action}") - def process_data(self, value: GeneralFile | None) -> None: + def process_data(self, value: SearchableFile | None) -> None: if value: try: @@ -450,8 +452,8 @@ def process_data(self, value: GeneralFile | None) -> None: class UploadMultipleFilesWithORMSupport(UploadMultipleField): """ Extends the upload multiple field with file support. """ - file_class: type[GeneralFile] - added_files: list[GeneralFile] + file_class: type[SearchableFile] + added_files: list[SearchableFile] upload_field_class = UploadFileWithORMSupport def __init__(self, *args: Any, **kwargs: Any): @@ -461,7 +463,7 @@ def __init__(self, *args: Any, **kwargs: Any): def populate_obj(self, obj: object, name: str) -> None: self.added_files = [] files = getattr(obj, name, ()) - output: list[GeneralFile] = [] + output: list[SearchableFile] = [] print(self.entries) for field, file in zip_longest(self.entries, files): diff --git a/src/privatim/layouts/action_menu.pt b/src/privatim/layouts/action_menu.pt index 6e5ef32..f4cde0d 100644 --- a/src/privatim/layouts/action_menu.pt +++ b/src/privatim/layouts/action_menu.pt @@ -1,5 +1,5 @@ -
    +