diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1a7c9ca..bc54001 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,6 +27,9 @@ jobs: libpq-dev \ make \ postgresql \ + libpoppler-cpp-dev \ + pkg-config \ + python3-dev \ --fix-missing - name: Checkout repo @@ -75,6 +78,9 @@ jobs: libpq-dev \ make \ postgresql \ + libpoppler-cpp-dev \ + pkg-config \ + python3-dev \ --fix-missing export LC_ALL="C.UTF-8" diff --git a/README.md b/README.md index d6d1f06..0bb42eb 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,19 @@ Getting Started --------------- ``` +sudo apt install postgresql libpq-dev python3-dev build-essential weasyprint +``` + +Create the PostgreSQL database: +``` +sudo -u postgres bash -c "psql <1.0.0 lxml-stubs types-babel @@ -124,6 +115,7 @@ test = pytest pytest-cov pytest-codecov + pytest-postgresql pytest-xdist pytest-sugar pyquery diff --git a/src/privatim/__init__.py b/src/privatim/__init__.py index 0f7b3f0..6eec6c8 100644 --- a/src/privatim/__init__.py +++ b/src/privatim/__init__.py @@ -1,5 +1,7 @@ from functools import partial from fanstatic import Fanstatic +from sqlalchemy.dialects.postgresql import TSVECTOR + from privatim.layouts.action_menu import ActionMenuEntry from pyramid.config import Configurator from pyramid_beaker import session_factory_from_settings @@ -69,7 +71,7 @@ def profile_pic(request: 'IRequest') -> str: user = request.user if not user: return '' - return request.route_url('download_general_file', id=user.picture.id) + return request.route_url('download_file', id=user.picture.id) config.add_request_method(profile_pic, 'profile_pic', property=True) config.add_request_method(MessageQueue, 'messages', reify=True) @@ -147,4 +149,9 @@ def upgrade(context: 'UpgradeContext'): # type: ignore[no-untyped-def] ), ) + context.operations.add_column( + 'consultations', + Column('searchable_text_de_CH', TSVECTOR()) + ) + context.commit() diff --git a/src/privatim/cache.py b/src/privatim/cache.py index b2a79da..3b28b84 100644 --- a/src/privatim/cache.py +++ b/src/privatim/cache.py @@ -5,6 +5,7 @@ from pyramid.threadlocal import get_current_request + from typing import TYPE_CHECKING if TYPE_CHECKING: from collections.abc import Callable diff --git a/src/privatim/cli/helpers.py b/src/privatim/cli/helpers.py new file mode 100644 index 0000000..583ae9d --- /dev/null +++ b/src/privatim/cli/helpers.py @@ -0,0 +1,86 @@ +import click +from pyramid.paster import bootstrap +from sqlalchemy import select, func + +from privatim.models.associated_file import SearchableAssociatedFiles +from privatim.models.file import SearchableFile +from privatim.orm import Base + + +@click.command() +@click.argument('config_uri') +def print_tsvectors(config_uri: str) -> None: + """ + Iterate over all models inheriting from SearchableAssociatedFiles + and print their tsvector of searchable_text. + """ + env = bootstrap(config_uri) + + with env['request'].tm: + db = env['request'].dbsession + seen = set() + for mapper in Base.registry.mappers: + cls = mapper.class_ + if issubclass(cls, SearchableAssociatedFiles) and cls not in seen: + seen.add(cls) + click.echo(f"\nProcessing model: {cls.__name__}") + stmt = select(cls.searchable_text_de_CH) + results = db.execute(stmt).fetchall() + for id, tsvector in results: + click.echo(f"ID: {id}") + click.echo(f"TSVector: {tsvector}") + click.echo("---") + + +@click.command() +@click.argument('config_uri') +def print_text(config_uri: str) -> None: + """ + Iterate over all models inheriting from SearchableAssociatedFiles + and print their tsvector of searchable_text. + """ + env = bootstrap(config_uri) + + with env['request'].tm: + db = env['request'].dbsession + seen = set() + for mapper in Base.registry.mappers: + cls = mapper.class_ + if issubclass(cls, SearchableAssociatedFiles) and cls not in seen: + seen.add(cls) + click.echo(f"\nProcessing model: {cls.__name__}") + texts2 = db.execute(select( + func.string_agg(SearchableFile.extract, ' ')).select_from( + cls).join(cls.files).group_by(cls.id)).all() + for content in texts2: + click.echo(f"text_contents: {content}") + click.echo("---") + + +@click.command() +@click.argument('config_uri') +def reindex(config_uri: str) -> None: + + env = bootstrap(config_uri) + + with env['request'].tm: + db = env['request'].dbsession + seen = set() + for mapper in Base.registry.mappers: + cls = mapper.class_ + if issubclass(cls, SearchableAssociatedFiles) and cls not in seen: + seen.add(cls) + click.echo(f"\nProcessing model: {cls.__name__}") + + stmt = select(cls) + results = db.execute(stmt).scalars().fetchall() + for instance in results: + assert isinstance(instance, cls) + name = getattr(instance, 'title', None) + if name is not None: + click.echo(f"\nReindexing model: {cls.__name__} with " + f"title: {name[:30]}") + else: + click.echo(f"\nReindexing model: {cls.__name__} with") + + instance.reindex_files() diff --git a/src/privatim/cli/initialize_db.py b/src/privatim/cli/initialize_db.py index bb5d655..815c49c 100644 --- a/src/privatim/cli/initialize_db.py +++ b/src/privatim/cli/initialize_db.py @@ -5,9 +5,10 @@ from privatim.layouts.layout import DEFAULT_TIMEZONE from privatim.models.consultation import Status, Tag +from privatim.models.file import SearchableFile from privatim.orm import get_engine from privatim.models import (Consultation, User, Meeting, WorkingGroup, - AgendaItem, GeneralFile) + AgendaItem) from privatim.orm import Base import click @@ -109,7 +110,7 @@ def add_example_content( pdf = here / 'sample-pdf-for-initialize-db/' / pdfname content = pdf.read_bytes() consultation = Consultation( - files=[GeneralFile(filename=pdfname, content=content)], + files=[SearchableFile(filename=pdfname, content=content)], title='Verordnung über den Einsatz elektronischer Mittel zur Ton- ' 'und Bildübertragung in Zivilverfahren (VEMZ)', description='Mit der Revision der Schweizerischen ' diff --git a/src/privatim/forms/add_comment.py b/src/privatim/forms/add_comment.py index 9e89592..0b32090 100644 --- a/src/privatim/forms/add_comment.py +++ b/src/privatim/forms/add_comment.py @@ -3,6 +3,7 @@ from privatim.forms.core import Form from privatim.i18n import _ + from typing import TYPE_CHECKING if TYPE_CHECKING: from pyramid.interfaces import IRequest diff --git a/src/privatim/forms/agenda_item_form.py b/src/privatim/forms/agenda_item_form.py index 44978b0..3418abb 100644 --- a/src/privatim/forms/agenda_item_form.py +++ b/src/privatim/forms/agenda_item_form.py @@ -1,11 +1,15 @@ +from sqlalchemy import select from wtforms import StringField from wtforms import validators +from wtforms.fields.choices import RadioField from wtforms.fields.simple import TextAreaField +from wtforms.validators import ValidationError from privatim.forms.core import Form from privatim.i18n import _ from privatim.models import Meeting + from typing import TYPE_CHECKING if TYPE_CHECKING: from pyramid.interfaces import IRequest @@ -40,3 +44,45 @@ def populate_obj(self, obj: 'AgendaItem') -> None: # type:ignore[override] super().populate_obj(obj) for name, field in self._fields.items(): field.populate_obj(obj, name) + + +class AgendaItemCopyForm(Form): + + def __init__( + self, + context: Meeting, + request: 'IRequest', + ) -> None: + + self._title = _('Select Destionation for Agenda Item') + + super().__init__( + request.POST, + obj=context, + meta={'context': context, 'request': request}, + ) + + all_meetings_for_choices = [ + (str(meeting.id), meeting.name) + # valid destination are all meetings except the one from which + # we are copying from + for meeting in request.dbsession.execute( + select(Meeting).where(Meeting.id != context.id) + ).scalars().all() + ] + if not all_meetings_for_choices: + self.copy_to.validators.append( + lambda form, field: ValidationError( + _('No valid destination meetings available.') + ) + ) + self.copy_to.choices = all_meetings_for_choices + + copy_to = RadioField( + label=_('Copy to'), validators=[validators.DataRequired()] + ) + + def populate_obj(self, obj: 'AgendaItem') -> None: # type:ignore[override] + super().populate_obj(obj) + for name, field in self._fields.items(): + field.populate_obj(obj, name) diff --git a/src/privatim/forms/consultation_form.py b/src/privatim/forms/consultation_form.py index cf1a93e..6ccbf1e 100644 --- a/src/privatim/forms/consultation_form.py +++ b/src/privatim/forms/consultation_form.py @@ -51,6 +51,10 @@ def __init__( ] self.status.choices = translated_choices + # If editing, populate the secondary_tags field + # if context and context.secondary_tags: + # self.secondary_tags.process_data(context.secondary_tags) + title = TextAreaField( _('Title'), validators=[DataRequired()], @@ -73,7 +77,6 @@ def __init__( render_kw={'rows': 6}, ) - # todo: finish field mapping # new: Beschluss decision = TextAreaField( _('Decision'), @@ -84,7 +87,7 @@ def __init__( _('Status'), choices=[] ) - cantons = SearchableSelectField( + secondary_tags = SearchableSelectField( _('Cantons'), choices=[('', '')] + CANTONS_SHORT, validators=[ diff --git a/src/privatim/forms/fields/fields.py b/src/privatim/forms/fields/fields.py index 2a29df7..477d5dc 100644 --- a/src/privatim/forms/fields/fields.py +++ b/src/privatim/forms/fields/fields.py @@ -3,6 +3,8 @@ import sedate from sqlalchemy import select + +from privatim.models.file import SearchableFile from privatim.static import init_tom_select from wtforms.utils import unset_value from wtforms.validators import DataRequired @@ -159,13 +161,20 @@ def process_formdata(self, valuelist: list['RawFormValue']) -> None: class SearchableSelectField(SelectMultipleField): - """A multiple select field with tom-select.js support. """ + """ + A multiple select field with tom-select.js support. + + Note: This is unrelated to PostgreSQL full-text search, which also uses + the term 'searchable'. + Note: you need to call form.raw_data() to actually get the choices as list + """ widget = ChosenSelectWidget(multiple=True) - def __call__(self, *args: Any, **kwargs: Any) -> Any: + def __call__(self, **kwargs: Any) -> Any: init_tom_select.need() - return super().__call__(*args, **kwargs) + self.data: list[str] = [] + return super().__call__(**kwargs) def process_data(self, value: list[object]) -> None: if value: @@ -386,33 +395,33 @@ def append_entry_from_field_storage( # we fake the formdata for the new field # we use a werkzeug MultiDict because the WebOb version # needs to get wrapped to be usable in WTForms - formdata = MultiDict() # type: ignore + formdata: MultiDict[str, 'RawFormValue'] = MultiDict() name = f'{self.short_name}{self._separator}{len(self)}' formdata.add(name, fs) return self._add_entry(formdata) class _DummyFile: - file: GeneralFile | None + file: SearchableFile | None class UploadFileWithORMSupport(UploadField): - """ Extends the upload field with onegov.file support. """ + """ Extends the upload field with file support. """ - file_class: type[GeneralFile] + file_class: type[SearchableFile] def __init__(self, *args: Any, **kwargs: Any): self.file_class = kwargs.pop('file_class') super().__init__(*args, **kwargs) - def create(self) -> GeneralFile | None: + def create(self) -> SearchableFile | None: if not getattr(self, 'file', None): return None assert self.file is not None self.file.seek(0) assert self.filename is not None - return GeneralFile(filename=self.filename, content=self.file.read()) + return SearchableFile(filename=self.filename, content=self.file.read()) def populate_obj(self, obj: object, name: str) -> None: @@ -431,7 +440,7 @@ def populate_obj(self, obj: object, name: str) -> None: else: raise NotImplementedError(f"Unknown action: {self.action}") - def process_data(self, value: GeneralFile | None) -> None: + def process_data(self, value: SearchableFile | None) -> None: if value: try: @@ -452,8 +461,8 @@ def process_data(self, value: GeneralFile | None) -> None: class UploadMultipleFilesWithORMSupport(UploadMultipleField): """ Extends the upload multiple field with file support. """ - file_class: type[GeneralFile] - added_files: list[GeneralFile] + file_class: type[SearchableFile] + added_files: list[SearchableFile] upload_field_class = UploadFileWithORMSupport def __init__(self, *args: Any, **kwargs: Any): @@ -463,7 +472,7 @@ def __init__(self, *args: Any, **kwargs: Any): def populate_obj(self, obj: object, name: str) -> None: self.added_files = [] files = getattr(obj, name, ()) - output: list[GeneralFile] = [] + output: list[SearchableFile] = [] print(self.entries) for field, file in zip_longest(self.entries, files): diff --git a/src/privatim/forms/filter_form.py b/src/privatim/forms/filter_form.py new file mode 100644 index 0000000..c01830c --- /dev/null +++ b/src/privatim/forms/filter_form.py @@ -0,0 +1,67 @@ +from wtforms import SelectField, BooleanField, DateField + +from privatim.forms.constants import cantons_named +from privatim.forms.core import Form +from wtforms.validators import Optional +from privatim.i18n import _ + + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pyramid.interfaces import IRequest + from wtforms import Field + + +def render_filter_field(field: 'Field') -> str: + if isinstance(field, BooleanField): + return field(class_="form-check-input") + else: + return field(class_="form-control") + + +class FilterForm(Form): + def __init__( + self, + request: 'IRequest', + ) -> None: + + self._title = _('Filter') + session = request.dbsession + super().__init__(request.POST, meta={'dbsession': session}) + + def get_type_fields(self) -> list[BooleanField]: + return [self.consultation, self.meeting, self.comment] + + def get_date_fields(self) -> list[tuple[str, DateField]]: + return [('datumVon', self.start_date), ('datumBis', self.end_date)] + + canton: SelectField = SelectField( + _('Canton'), + choices=[('all', _('all'))] + cantons_named, + validators=[Optional()], + render_kw={'class': 'form-select', 'id': 'kanton'}, + ) + + consultation: BooleanField = BooleanField( + _('Consultation'), + render_kw={'class': 'form-check-input', 'id': 'vernehmlassung'}, + ) + meeting: BooleanField = BooleanField( + _('Meeting'), render_kw={'class': 'form-check-input', 'id': 'sitzung'} + ) + comment: BooleanField = BooleanField( + _('Comment'), + render_kw={'class': 'form-check-input', 'id': 'kommentar'}, + ) + + start_date: DateField = DateField( + _('Date from'), + validators=[Optional()], + render_kw={'class': 'form-control', 'id': 'datumVon'}, + ) + end_date: DateField = DateField( + _('Date to'), + validators=[Optional()], + render_kw={'class': 'form-control', 'id': 'datumBis'}, + ) diff --git a/src/privatim/forms/meeting_form.py b/src/privatim/forms/meeting_form.py index 9799f43..3ad2bbd 100644 --- a/src/privatim/forms/meeting_form.py +++ b/src/privatim/forms/meeting_form.py @@ -10,6 +10,7 @@ from privatim.models import WorkingGroup from privatim.i18n import _ + from typing import TYPE_CHECKING if TYPE_CHECKING: from pyramid.interfaces import IRequest diff --git a/src/privatim/forms/search_form.py b/src/privatim/forms/search_form.py new file mode 100644 index 0000000..03b1f54 --- /dev/null +++ b/src/privatim/forms/search_form.py @@ -0,0 +1,30 @@ +from wtforms.fields.simple import SearchField +from wtforms.validators import DataRequired +from privatim.forms.core import Form +from privatim.i18n import _ + + +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from pyramid.interfaces import IRequest + + +class SearchForm(Form): + + def __init__( + self, + request: 'IRequest', + ) -> None: + session = request.dbsession + super().__init__( + request.POST, + meta={ + 'dbsession': session + } + ) + term = SearchField( + _('Search'), + [DataRequired()], + render_kw={ + }, + ) diff --git a/src/privatim/forms/widgets/checkbox_list_widget.py b/src/privatim/forms/widgets/checkbox_list_widget.py index 0be5e1f..0b9a4c4 100644 --- a/src/privatim/forms/widgets/checkbox_list_widget.py +++ b/src/privatim/forms/widgets/checkbox_list_widget.py @@ -1,6 +1,7 @@ from markupsafe import Markup from privatim.controls.controls import html_params + from typing import Any from typing import TYPE_CHECKING if TYPE_CHECKING: diff --git a/src/privatim/i18n/__init__.py b/src/privatim/i18n/__init__.py index ac587a2..c45bf5d 100644 --- a/src/privatim/i18n/__init__.py +++ b/src/privatim/i18n/__init__.py @@ -4,11 +4,15 @@ from .translation_string import TranslationStringFactory +locales = {'de_CH': 'german'} + + _ = TranslationStringFactory('privatim') __all__ = ( '_', 'LocaleNegotiator', 'pluralize', - 'translate' + 'translate', + 'locales', ) diff --git a/src/privatim/i18n/locale_negotiator.py b/src/privatim/i18n/locale_negotiator.py index ac5b8c2..c9c651d 100644 --- a/src/privatim/i18n/locale_negotiator.py +++ b/src/privatim/i18n/locale_negotiator.py @@ -1,8 +1,8 @@ -from typing import TYPE_CHECKING - from pyramid.interfaces import ILocaleNegotiator from zope.interface import implementer + +from typing import TYPE_CHECKING if TYPE_CHECKING: from pyramid.interfaces import IRequest diff --git a/src/privatim/i18n/translation_string.py b/src/privatim/i18n/translation_string.py index cf97848..1d9b1a1 100644 --- a/src/privatim/i18n/translation_string.py +++ b/src/privatim/i18n/translation_string.py @@ -1,7 +1,6 @@ from functools import update_wrapper from typing import Any from typing import Literal -from typing import TYPE_CHECKING from typing import overload import translationstring @@ -10,6 +9,8 @@ from .core import translate + +from typing import TYPE_CHECKING if TYPE_CHECKING: from markupsafe import HasHTML from typing import Protocol diff --git a/src/privatim/layouts/action_menu.pt b/src/privatim/layouts/action_menu.pt index 6e5ef32..f4cde0d 100644 --- a/src/privatim/layouts/action_menu.pt +++ b/src/privatim/layouts/action_menu.pt @@ -1,5 +1,5 @@ -
+