Feature/ogc 508 replace elastic search by postgres text search

CHANGES.md

@@ -227,7 +227,7 @@ all the necessary book keeping for traces and profiles
 
 ##### Improve ticker code.
 
-Adds fallback to ticker in case of websocket server shutdown and add 
+Adds fallback to ticker in case of websocket server shutdown and add
 timestamp to HEAD request to avoid being browser-cached.
 
 `Bugfix` | [093fdf43af](https://github.com/onegov/onegov-cloud/commit/093fdf43af92fb4ecc2de84f815db5a48a08f13a)
@@ -743,7 +743,7 @@ Removes reference to deleted 'sender' mail macro from mail template.
 
 ##### Fixes news not being displayed if it's the first item.
 
-Fix root-level page interpretation bug for news, 
+Fix root-level page interpretation bug for news,
 which was mistakenly being treated as falsy (index 0).
 
 `Bugfix` | [OGC-863](https://linear.app/onegovcloud/issue/OGC-863) | [dc42c72bf3](https://github.com/onegov/onegov-cloud/commit/dc42c72bf3229270ff262b2c38a58d61cb41cfcb)
@@ -880,8 +880,8 @@ Displays "access denied" instead of "page not found" if Invoice exists.
 
 ##### Don't use deferral on timestamp columns.
 
-Timestamps don't add a lot of data to queries but accessing them will 
-lead to a lot of additional queries. Also, nobody expects these 
+Timestamps don't add a lot of data to queries but accessing them will
+lead to a lot of additional queries. Also, nobody expects these
 timestamps to be deferred in the first place.
 
 `Bugfix` | [6288609773](https://github.com/onegov/onegov-cloud/commit/6288609773ffc7cc447db2dd267e7dd433c00ff1)
@@ -1374,7 +1374,7 @@ Documents = *.pdf (multiple)
 
 ##### Add content security policy tween.
 
-Allows to optionally add a websocket endpoint to the connect-src content 
+Allows to optionally add a websocket endpoint to the connect-src content
 security policy.
 
 `Feature` | [OGC-935](https://linear.app/onegovcloud/issue/OGC-935) | [9b2a006baf](https://github.com/onegov/onegov-cloud/commit/9b2a006baf374b7514a871bae18ddabf66326c8e)
@@ -2463,7 +2463,7 @@ Also cleanup the documentation.
 
 `Bugfix` | [OGC-589](https://linear.app/onegovcloud/issue/OGC-589) | [fb904c42d3](https://github.com/onegov/onegov-cloud/commit/fb904c42d33f08f8212215819a666912523f4260)
 
-### Core 
+### Core
 
 ##### Use autoapi for documentation.
 

src/onegov/activity/models/attendee.py

@@ -1,4 +1,5 @@
 from datetime import date
+
 from onegov.activity.models.booking import Booking
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import TimestampMixin
@@ -11,7 +12,6 @@
 from sqlalchemy import Date
 from sqlalchemy import Float
 from sqlalchemy import ForeignKey
-from sqlalchemy import Index
 from sqlalchemy import Integer
 from sqlalchemy import Numeric
 from sqlalchemy import Text
@@ -43,6 +43,10 @@ class Attendee(Base, TimestampMixin, ORMSearchable):
     }
     es_public = False
 
+    @property
+    def search_score(self):
+        return 3
+
     @property
     def es_suggestion(self):
         return self.name
@@ -179,7 +183,3 @@ def happiness(cls, period_id):
         order_by='Booking.created',
         backref='attendee'
     )
-
-    __table_args__ = (
-        Index('unique_child_name', 'username', 'name', unique=True),
-    )

src/onegov/agency/models/person.py

@@ -21,8 +21,6 @@ def es_public(self):
         'title': {'type': 'text'},
         'function': {'type': 'localized'},
         'email': {'type': 'text'},
-        'phone_internal': {'type': 'text'},
-        'phone_es': {'type': 'text'}
     }
 
     @property

src/onegov/agency/views/search.py

@@ -1,13 +1,23 @@
 from onegov.agency import AgencyApp
 from onegov.agency.layout import AgencySearchLayout
 from onegov.core.security import Public
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 
 
 @AgencyApp.html(model=Search, template='search.pt', permission=Public)
-def search(self, request):
+def agency_search(self, request):
     data = search_view(self, request)
     if isinstance(data, dict):
         data['layout'] = AgencySearchLayout(self, request)
     return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt',
+                permission=Public)
+def agency_search_postgres(self, request):
+    data = search_postgres_view(self, request)
+    if isinstance(data, dict):
+        data['layout'] = AgencySearchLayout(self, request)
+    return data

src/onegov/core/upgrade.py

@@ -480,6 +480,12 @@ def has_column(self, table: str, column: str) -> bool:
             table, schema=self.schema
         )}
 
+    def has_index(self, table: str, index: str) -> bool:
+        inspector = Inspector(self.operations_connection)
+        return index in {i['name'] for i in inspector.get_indexes(
+            table, schema=self.schema
+        )}
+
     def has_enum(self, enum: str) -> bool:
         return self.session.execute(f"""
             SELECT EXISTS (

src/onegov/directory/models/directory.py

@@ -17,10 +17,8 @@
 from onegov.file.utils import as_fileintent
 from onegov.form import flatten_fieldsets, parse_formcode, parse_form
 from onegov.search import SearchableContent
-from sqlalchemy import Column
+from sqlalchemy import Column, Text, Integer
 from sqlalchemy import func, exists, and_
-from sqlalchemy import Integer
-from sqlalchemy import Text
 from sqlalchemy.orm import object_session
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.attributes import InstrumentedAttribute
@@ -103,6 +101,10 @@ def count(self):
         backref='directory'
     )
 
+    @property
+    def search_score(self):
+        return 7
+
     @property
     def entry_cls_name(self):
         return 'DirectoryEntry'

src/onegov/directory/models/directory_entry.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.mixins import ContentMixin
 from onegov.core.orm.mixins import TimestampMixin
@@ -98,7 +100,7 @@ def keywords(self):
     def keywords(self, value):
         self._keywords = {k: '' for k in value} if value else None
 
-    @property
+    @hybrid_property
     def text(self):
         return self.directory.configuration.extract_searchable(self.values)
 

src/onegov/event/models/event.py

@@ -6,6 +6,7 @@
 from icalendar import Calendar as vCalendar
 from icalendar import Event as vEvent
 from icalendar import vRecur
+
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import associated
 from onegov.core.orm.mixins import content_property
@@ -106,6 +107,10 @@ class Event(Base, OccurrenceMixin, ContentMixin, TimestampMixin,
         EventFile, 'pdf', 'one-to-one', uselist=False, backref_suffix='pdf'
     )
 
+    @property
+    def search_score(self):
+        return 1
+
     def set_image(self, content, filename=None):
         self.set_blob('image', content, filename)
 

src/onegov/feriennet/models/activity.py

@@ -1,4 +1,7 @@
 from functools import cached_property
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.activity import Activity, ActivityCollection, Occasion
 from onegov.activity import PublicationRequestCollection
 from onegov.activity.models import DAYS
@@ -24,6 +27,10 @@ class VacationActivity(Activity, CoordinatesExtension, SearchableContent):
         'organiser': {'type': 'text'}
     }
 
+    @property
+    def search_score(self):
+        return 1
+
     @property
     def es_public(self):
         return self.state == 'accepted'
@@ -32,7 +39,7 @@ def es_public(self):
     def es_skip(self):
         return self.state == 'preview'
 
-    @property
+    @hybrid_property
     def organiser(self):
         organiser = [
             self.user.username,

src/onegov/feriennet/templates/mail_booking_accepted.pt

@@ -16,7 +16,7 @@
             <tal:b tal:content="structure bookings_link"></tal:b>
         </p>
         <br>
-        <p i18n:translate>Best regards</p>                    
+        <p i18n:translate>Best regards</p>
         <metal:b use-macro="layout.macros['sender']|nothing" />
     </tal:b>
 </div>

src/onegov/feriennet/templates/mail_booking_canceled.pt

@@ -15,7 +15,7 @@
             <tal:b tal:content="structure bookings_link"></tal:b>
         </p>
         <br>
-        <p i18n:translate>Best regards</p>                    
+        <p i18n:translate>Best regards</p>
         <metal:b use-macro="layout.macros['sender']|nothing" />
     </tal:b>
 </div>

src/onegov/file/models/file.py

@@ -6,6 +6,7 @@
 from contextlib import contextmanager
 from collections import defaultdict
 from depot.fields.sqlalchemy import UploadedFileField as UploadedFileFieldBase
+
 from onegov.core.crypto import random_token
 from onegov.core.orm import Base
 from onegov.core.orm.abstract import Associable
@@ -257,6 +258,10 @@ class File(Base, Associable, TimestampMixin):
         Index('files_by_type_and_order', 'type', 'order'),
     )
 
+    @property
+    def search_score(self) -> int:
+        return 10
+
     @hybrid_property
     def signature_timestamp(self) -> 'datetime | None':
         if self.signed:

src/onegov/file/models/fileset.py

@@ -6,6 +6,7 @@
 
 
 from typing import TYPE_CHECKING
+
 if TYPE_CHECKING:
     from .file import File
 

src/onegov/form/models/definition.py

@@ -140,6 +140,9 @@ class FormDefinition(Base, ContentMixin, TimestampMixin, Extendable):
     }
 
     @property
+    def search_score(self) -> int:
+        return 7
+
     def form_class(self) -> Type['Form']:
         """ Parses the form definition and returns a form class. """
 

src/onegov/fsi/models/course.py

@@ -1,4 +1,5 @@
 from arrow import utcnow
+
 from onegov.core.html import html_to_text
 from onegov.core.orm import Base
 from onegov.core.orm.types import UUID
@@ -32,6 +33,10 @@ class Course(Base, ORMSearchable):
     # hides the course in the collection for non-admins
     hidden_from_public = Column(Boolean, nullable=False, default=False)
 
+    @property
+    def search_score(self):
+        return 2
+
     @property
     def title(self):
         return self.name

src/onegov/fsi/models/course_attendee.py

@@ -1,3 +1,5 @@
+from sqlalchemy.ext.hybrid import hybrid_property
+
 from onegov.core.orm import Base
 from onegov.core.orm.types import UUID, JSON
 from sqlalchemy import Boolean
@@ -107,7 +109,7 @@ def __str__(self):
         cascade='all, delete-orphan'
     )
 
-    @property
+    @hybrid_property
     def title(self):
         return ' '.join((
             p for p in (
@@ -131,7 +133,7 @@ def role(self):
             return 'member'
         return self.user.role
 
-    @property
+    @hybrid_property
     def email(self):
         """Needs a switch for external users"""
         if not self.user_id:

src/onegov/fsi/models/course_event.py

@@ -22,13 +22,15 @@
 from onegov.fsi.models.course_subscription import subscription_table
 from onegov.search import ORMSearchable
 
-
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from .course import Course
     from .course_notification_template import (
-        CancellationTemplate, CourseNotificationTemplate, InfoTemplate,
-        ReminderTemplate, SubscriptionTemplate
+        CancellationTemplate,
+        CourseNotificationTemplate,
+        InfoTemplate,
+        ReminderTemplate,
+        SubscriptionTemplate
     )
 
 COURSE_EVENT_STATUSES = ('created', 'confirmed', 'canceled', 'planned')
@@ -83,7 +85,7 @@ def es_public(self):
     def title(self):
         return str(self)
 
-    @property
+    @hybrid_property
     def name(self):
         return self.course.name
 
@@ -95,7 +97,7 @@ def lead(self):
             f'{self.presenter_company}'
         )
 
-    @property
+    @hybrid_property
     def description(self):
         return self.course.description
 

src/onegov/fsi/views/search.py

@@ -1,15 +1,29 @@
 from onegov.core.security import Personal
 from onegov.fsi import FsiApp
-from onegov.org.models import Search
+from onegov.org.models import Search, SearchPostgres
 from onegov.org.views.search import search as search_view
+from onegov.org.views.search import search_postgres as search_postgres_view
 from onegov.org.views.search import suggestions as suggestions_view
+from onegov.org.views.search import suggestions_postgres as \
+    suggestions_postgres_view
 
 
 @FsiApp.html(model=Search, template='search.pt', permission=Personal)
 def search(self, request):
     return search_view(self, request)
 
 
+@FsiApp.html(model=SearchPostgres, template='search_postgres.pt',
+             permission=Personal)
+def search_postgres(self, request):
+    return search_postgres_view(self, request)
+
+
 @FsiApp.json(model=Search, name='suggest', permission=Personal)
 def suggestions(self, request):
     return suggestions_view(self, request)
+
+
+@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal)
+def suggestions_postgres(self, request):
+    return suggestions_postgres_view(self, request)

src/onegov/landsgemeinde/views/search.py

@@ -7,4 +7,5 @@
 
 @LandsgemeindeApp.html(model=Search, template='search.pt', permission=Public)
 def landsgemeinde_search(self, request):
+    # TODO: switch to postgres search
     return search(self, request, DefaultLayout(self, request))

src/onegov/newsletter/models.py

@@ -95,6 +95,9 @@ def validate_name(self, key: str, name: str) -> str:
         back_populates='newsletters')
 
     @property
+    def search_score(self) -> int:
+        return 6
+
     def open_recipients(self) -> tuple['Recipient', ...]:
         received = select([newsletter_recipients.c.recipient_id]).where(
             newsletter_recipients.c.newsletter_id == self.name)