Merge branch 'develop' into events

Dockerfile

@@ -3,6 +3,42 @@ ARG DJANGO_RELEASE_NAME
 ARG SENTRY_RELEASE_NAME
 ARG SENTRY_RELEASE_ENVIRONMENT
 
+#
+# wiki renderer image
+#
+FROM ruby:3.0 as wiki-base
+
+RUN apt-get update && \
+    # pip to install docutils, cmake to build github-linguist
+    apt-get install -y python3-pip cmake && \
+    python3 -m pip install docutils && \
+    apt-get remove -y python3-pip && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./wiki/Gemfile ./wiki/Gemfile.lock ./
+RUN bundle install
+
+RUN apt-get remove -y cmake && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /usr/src/app
+
+COPY ./wiki/ .
+
+ENTRYPOINT [ "./fetcher.rb" ]
+
+FROM wiki-base as wiki-stable
+
+ENV WIKI_GIT_URL=https://github.com/esnvutbrno/fiesta-plus.wiki.git
+ENV WIKI_STATIC_ASSETS_URL=/static/gh/
+
+ENV WIKI_DB_PATH=/usr/src/wiki/db
+ENV WIKI_DB_NAME=${WIKI_DB_PATH}/wiki.sqlite3
+ENV WIKI_STATIC_PATH=/usr/src/wiki/static
+ENV WIKI_STATIC_URL=/static/wiki/
+
+RUN mkdir -p $WIKI_STATIC_PATH && mkdir -p $WIKI_DB_PATH && ./fetcher.rb
+
 #
 # webpack image
 #
@@ -129,18 +165,20 @@ ARG DJANGO_RELEASE_NAME
 ENV DJANGO_RELEASE_NAME=${DJANGO_RELEASE_NAME}
 
 # need production configuration, but not all values are ready in env
-RUN bash -c "DJANGO_SECRET_KEY=\$RANDOM DJANGO_CONFIGURATION=LocalProduction python manage.py collectstatic --no-input"
+RUN bash -c "DJANGO_SECRET_KEY=\$RANDOM DJANGO_CONFIGURATION=LocalProduction python manage.py collectstatic --no-input --verbosity 3"
 
 # given by webpack compiled results
 COPY --from=webpack-stable /usr/src/build/webpack-stats.json ${DJANGO_BUILD_DIR}
 
+COPY --from=wiki-stable /usr/src/wiki/db/wiki.sqlite3 /usr/src/wiki/db/wiki.sqlite3
+
 # TODO: check opts https://www.uvicorn.org/#command-line-options
 CMD ["python -m gunicorn -b [::]:8000 fiesta.wsgi:application"]
 
 #
 # proxy image
 #
-FROM nginx:1.25.0-alpine as proxy-base
+FROM nginx:1.25.2-alpine as proxy-base
 
 RUN rm /etc/nginx/conf.d/default.conf
 
@@ -152,6 +190,7 @@ FROM proxy-base as proxy-stable
 # prepared by webpack and web builds during CD
 COPY --from=webpack-stable /usr/src/build/ /var/build/
 COPY --from=web-stable /usr/src/static/ /var/static/
+COPY --from=wiki-stable /usr/src/wiki/static /var/static/wiki/
 
 ARG STATIC_LOCATION_PATTERN="^/static/(.*)$$"
 ENV STATIC_LOCATION_PATTERN=${STATIC_LOCATION_PATTERN}

docker-compose.override.yml

@@ -39,18 +39,6 @@ services:
     volumes:
       - ./legacy.sql:/docker-entrypoint-initdb.d/legacy.sql
 
-  elastic:
-    volumes:
-      - ./elastic/elastic.yml:/usr/share/elasticsearch/config/elasticsearch.yml
-      - ./conf/elastic/elasticsearch.keystore:/usr/share/elasticsearch/config/elasticsearch.keystore
-      - ./conf/certs:/usr/share/elasticsearch/config/certs
-
-  kibana:
-    volumes:
-      - ./elastic/kibana.yml:/usr/share/kibana/config/kibana.yml
-      - ./conf/certs:/etc/kibana/certs
-
   wikifetcher:
     volumes:
       - ./wiki/:/usr/src/app/
-      - ./conf/certs:/usr/share/certs

docker-compose.prod.yml

@@ -22,13 +22,5 @@ services:
     ports:
       - "80:80"
       - "443:443"
-    # TODO: figure out before deployment, whereever/where should be certs stored
     volumes:
       - ./conf/certs:/etc/nginx/certs
-
-  # remove profiles so production env does have filled elastic search
-#  elastic:
-#    profiles: [ ]
-
-#  wikifetcher:
-#    profiles: [ ]

docker-compose.yml

@@ -16,6 +16,8 @@ services:
     volumes:
       # for dev and also produp
       - web_media_files:/usr/src/media
+      - wiki_db:/usr/src/wiki/db/
+      - wiki_static:/usr/src/wiki/static/wiki/
     environment:
       VIRTUAL_HOST: web.${ROOT_DOMAIN}
       VIRTUAL_PORT: 8000
@@ -32,7 +34,6 @@ services:
       - web
     volumes:
       - web_media_files:/var/media
-      - wiki_static_files:/var/wiki
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       PROXY_HOSTNAME_TARGET: "web"
@@ -98,46 +99,24 @@ services:
     ports:
       - "4306:3306"
 
-  elastic:
-    image: elasticsearch:7.17.0
-    profiles: [ "wiki" ]
-    volumes:
-      - elastic_data:/usr/share/elasticsearch/data
-    ports:
-      - "9200"
-
-  kibana:
-    image: kibana:7.17.0
-    profiles: [ "wiki" ]
-    depends_on:
-      - elastic
-    ports:
-      - "5601"
-    env_file:
-      - ./wiki/.env.base
-      - ./.env
-    environment:
-      VIRTUAL_HOST: kibana.${ROOT_DOMAIN}
-      VIRTUAL_PORT: 5601
-
   wikifetcher:
-    build: ./wiki
-    profiles: [ "wiki" ]
-    # when elastic is not ready yet, try it again
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: wiki-base
     restart: on-failure
-    depends_on:
-      - elastic
     volumes:
-      - wiki_static_files:/usr/src/static
+      - wiki_db:/usr/src/wiki/db/
+      - wiki_static:/usr/src/wiki/static/
     env_file:
       - .env
       - wiki/.env.base
 
 volumes:
   postgres15_data:
-  elastic_data:
   legacydb_data:
   webpack_build:
   web_static_files:
   web_media_files:
-  wiki_static_files:
+  wiki_db:
+  wiki_static:

fiesta/apps/utils/templatetags/navigation.py

@@ -6,13 +6,15 @@
 from django.contrib.auth import REDIRECT_FIELD_NAME
 from django.urls import reverse
 from django.utils.http import urlencode
+from django.utils.translation import gettext_lazy as _
 
 if typing.TYPE_CHECKING:
     from apps.plugins.middleware.plugin import HttpRequest
     from apps.plugins.models import Plugin
     from apps.plugins.plugin import BasePluginAppConfig
     from apps.sections.models import Section, SectionMembership
 
+
 register = template.Library()
 
 
@@ -27,13 +29,26 @@ class NavigationItemSpec(typing.NamedTuple):
 @register.simple_tag(takes_context=True)
 def get_navigation_items(context):
     from apps.plugins.utils import all_plugins_to_order
+    from apps.sections.models import SectionMembership
 
     request: HttpRequest = context["request"]
 
     membership: SectionMembership | None = request.membership
     section: Section | None = request.in_space_of_section
 
     if not membership:
+        if request.all_memberships.filter(
+            role__in=(SectionMembership.Role.ADMIN, SectionMembership.Role.EDITOR)
+        ).exists():
+            # TODO: consider better way
+            return [
+                NavigationItemSpec(
+                    _("Docs"),
+                    reverse("wiki:index"),
+                    [],
+                    request.path.startswith(reverse("wiki:index")),
+                ),
+            ]
         return []
 
     plugins: list[Plugin] = (
@@ -42,11 +57,14 @@ def get_navigation_items(context):
 
     orders = all_plugins_to_order()
 
-    for plugin in sorted(plugins, key=lambda p: (orders[p.app_label], p.app_label)):
-        apps: BasePluginAppConfig = plugin.app_config
-        item = apps.as_navigation_item(request=request, bound_plugin=plugin)
-        if item:
-            yield item
+    def _all_items():
+        for plugin in sorted(plugins, key=lambda p: (orders[p.app_label], p.app_label)):
+            apps: BasePluginAppConfig = plugin.app_config
+            item = apps.as_navigation_item(request=request, bound_plugin=plugin)
+            if item:
+                yield item
+
+    return list(_all_items())
 
 
 @register.simple_tag(takes_context=True)

fiesta/apps/wiki/elastic.py → fiesta/apps/wiki/adapter.py

@@ -5,6 +5,8 @@
 from django.core.exceptions import ObjectDoesNotExist
 from elasticsearch import Elasticsearch, Transport, Urllib3HttpConnection
 
+from apps.wiki import models
+
 
 class Revision(TypedDict):
     at: str
@@ -30,7 +32,7 @@ class SearchPage(TypedDict):
     highlight: list[str]
 
 
-class WikiElastic:
+class ElasticAdapter:
     def __init__(self, client: Elasticsearch):
         self._client = client
 
@@ -109,6 +111,36 @@ def _page_for_filename(self, file_name: str, *, base: str = "") -> Page | None:
         return None
 
 
+class LocalDbAdapter(ElasticAdapter):
+    def _page_for_filename(self, file_name: str, *, base: str = "") -> Page | None:
+        base = f"{base}/" if base and not file_name.startswith("_") else ""
+        return models.Page.objects.using("wiki").get(file__contains=f"{base}{file_name}.").__dict__
+
+    def search(self, term: str) -> list[SearchPage]:
+        results = models.Page.objects.using("wiki").raw(
+            """
+                SELECT
+                    snippet(wiki, 1, '<b>', '</b>', '...', 64) as highlight,
+                    rank as rank,
+                    w.*
+                FROM wiki AS w
+                WHERE content_plain MATCH %s
+                ORDER BY rank;
+            """,
+            [term.strip()],
+        )
+
+        return [
+            SearchPage(
+                source=page.__dict__,
+                highlight=page.highlight,
+                score=page.rank,
+            )
+            for page in results
+            if not page.file.startswith("_")
+        ]
+
+
 class LocalCATrustedTransportation(Transport):
     # TODO: really here?
     class CATrustedConnection(Urllib3HttpConnection):
@@ -124,4 +156,5 @@ def __init__(self, *args, **kwargs) -> None:
     transport_class=LocalCATrustedTransportation,
 )
 
-wiki_elastic = WikiElastic(client=es)
+wiki_adapter = LocalDbAdapter(client=es)
+# wiki_adapter = ElasticAdapter(client=es)

fiesta/apps/wiki/models.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+
+class Page(models.Model):
+    file = models.CharField(primary_key=True, max_length=255, db_column="file", verbose_name=_("file"))
+    title = models.CharField(max_length=255, db_column="title", verbose_name=_("title"))
+    content_html = models.TextField(db_column="content_html", verbose_name=_("content"))
+    content_plain = models.TextField(db_column="content_plain", verbose_name=_("content (plain)"))
+    toc = models.TextField(db_column="toc", verbose_name=_("table of contents"))
+
+    last_change = models.JSONField(db_column="last_change", verbose_name=_("last change"))
+
+    class Meta:
+        # generated by wiki dumper script, just queried by Django
+        managed = False
+        db_table = "wiki"
+        verbose_name = _("page")
+        verbose_name_plural = _("pages")
+
+    def __str__(self):
+        return self.title

fiesta/apps/wiki/templates/wiki/index.html

@@ -32,8 +32,9 @@
 
       {% block main %}
         {{ page.content_html|safe }}
-        <hr class="mt-10">
-        <div class="mt-5 mb-5 text-sm">{{ footer.content_html|safe }}</div>
+        {#        TODO: consider footer display #}
+        {#        <hr class="mt-10">#}
+        {#        <div class="mt-5 mb-5 text-sm">{{ footer.content_html|safe }}</div>#}
       {% endblock main %}
     </main>
 
@@ -48,14 +49,17 @@
         {% endif %}
         <div class="mt-5 text-xs items-center space-y-1 text-gray-600">
           {# TODO: links to conf #}
-          Last revision: {{ page.last_change.name }}
-          <a href="https://github.com/esnvutbrno/fiesta-plus/wiki/_compare/{{ page.last_change.parent_sha }}..{{ page.last_change.sha }}"
-             target="_blank"
-             rel="noreferrer nofollow noopener"
-             class="hover:underline"
-             title="{{ page.last_change.at|date_from_iso|date }}">
-            {{ page.last_change.at|date_from_iso|naturaltime }}
-          </a>
+          {% if page.last_change %}
+            Last revision:
+            {#          {{ page.last_change.name }}#}
+            <a href="https://github.com/esnvutbrno/fiesta-plus/wiki/_compare/{{ page.last_change.parent_sha }}..{{ page.last_change.sha }}"
+               target="_blank"
+               rel="noreferrer nofollow noopener"
+               class="hover:underline"
+               title="{{ page.last_change.at|date_from_iso|date }}">
+              {{ page.last_change.at|date_from_iso|naturaltime }}
+            </a>
+          {% endif %}
           <div class="border-t border-gray-100"></div>
           <a href="https://github.com/esnvutbrno/fiesta-plus/wiki/{{ page.file|strip_file_extension }}/_history"
              target="_blank"

fiesta/apps/wiki/templates/wiki/search.html

@@ -16,11 +16,7 @@ <h2 class="text-3xl font-medium font-gray-800 my-3">{{ hit.source.title }}</h2>
               {{ hit.score }}
             </span>
           </div>
-          <p class="text-gray-700 text-md">
-            {{ hit.highlight.content_plain|safeseq|join:"
-            <br>
-            "|safe }}
-          </p>
+          <p class="text-gray-700 text-md">{{ hit.highlight|safe }}</p>
         </a>
       {% endwith %}
     {% endfor %}