Feature/deepface (#123)

* chg ! use deepface library

.github/workflows/release.yml

@@ -105,6 +105,7 @@ jobs:
         with:
           env_vars: OS,PYTHON
           fail_ci_if_error: true
+          required_coverage: 66.0
           files: coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: false

.github/workflows/test.yml

@@ -152,6 +152,7 @@ jobs:
         with:
           env_vars: OS,PYTHON
           fail_ci_if_error: true
+          required_coverage: 66.0
           files: coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: false

docker/Dockerfile

@@ -178,7 +178,7 @@ RUN --mount=type=cache,target=/root/.uv-cache \
         --cache-dir=/root/.uv-cache \
         --python=/usr/local/bin/python \
         --python-preference=system \
-        --no-dev --no-editable --frozen
+        --no-dev --no-editable --frozen --extra distribution
 
 
 # ------------- dist -------------

docker/bin/docker-entrypoint.sh

@@ -3,6 +3,7 @@
 export MEDIA_ROOT="${MEDIA_ROOT:-/var/run/app/media}"
 export STATIC_ROOT="${STATIC_ROOT:-/var/run/app/static}"
 export DEFAULT_ROOT="${DEFAULT_ROOT:-/var/run/app/default}"
+export DEEPFACE_HOME="${DEEPFACE_HOME:-/var/run/app/deepface}"
 export UWSGI_PROCESSES="${UWSGI_PROCESSES:-"4"}"
 export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"hope_dedup_engine.config.settings"}"
 mkdir -p "${MEDIA_ROOT}" "${STATIC_ROOT}" "${DEFAULT_ROOT}" || echo "Cannot create dirs ${MEDIA_ROOT} ${STATIC_ROOT} ${DEFAULT_ROOT}"
@@ -15,10 +16,14 @@ if [ -d "${DEFAULT_ROOT}" ];then
   chown -R hope:unicef ${DEFAULT_ROOT}
 fi
 
+if [ -d "${DEEPFACE_HOME}" ];then
+  chown -R hope:unicef ${DEEPFACE_HOME}
+fi
 
 echo "MEDIA_ROOT  ${MEDIA_ROOT}"
 echo "STATIC_ROOT ${STATIC_ROOT}"
-echo "DEFAULT_ROOT  ${DEFAULT_ROOT}"
+echo "DEFAULT_ROOT ${DEFAULT_ROOT}"
+echo "DEEPFACE_HOME ${DEEPFACE_HOME}"
 echo "Docker run command: $1"
 
 case "$1" in
@@ -28,14 +33,16 @@ case "$1" in
       exit 0
       ;;
     worker)
-      gosu hope:unicef django-admin syncdnn || exit 1
 	    set -- tini -- "$@"
       set -- gosu hope:unicef celery -A hope_dedup_engine.config.celery worker -E --loglevel=ERROR --concurrency=4
       ;;
     beat)
 	    set -- tini -- "$@"
       set -- gosu hope:unicef celery -A hope_dedup_engine.config.celery beat --loglevel=ERROR --scheduler django_celery_beat.schedulers:DatabaseScheduler
       ;;
+    syncmodels)
+      gosu hope:unicef django-admin syncmodels || exit 1
+      ;;
     run)
       django-admin check --deploy || exit 1
 	    set -- tini -- "$@"

pyproject.toml

@@ -11,6 +11,7 @@ requires-python = ">=3.12"
 license = {text = "MIT"}
 
 dependencies = [
+    "deepface>=0.0.93",
     "Django",
     "celery[redis]",
     "django-admin-extra-buttons",
@@ -35,10 +36,7 @@ dependencies = [
     "sentry-sdk[celery,django]>=2.2.1",
     "social-auth-app-django",
     "social-auth-core",
-    "opencv-contrib-python-headless>=4.10.0.84",
-    "face-recognition>=1.3.0",
     "unicef-security",
-    "uwsgi>=2.0.25.1",
     "requests>=2.32.3",
     "numpy>=1.26.4,<2.0.0",
     "flower>=2.0.1",
@@ -47,6 +45,7 @@ dependencies = [
     "jsonschema>=4.23.0",
     "django-celery-boost==0.4.1",
     "django-svelte-jsoneditor>=0.4.2",
+    "tf-keras>=2.18.0",
 ]
 
 [project.optional-dependencies]
@@ -58,6 +57,10 @@ docs = [
     "mkdocs-gen-files>=0.5.0",
 ]
 
+distribution = [
+    "uwsgi>=2.0.28",
+]
+
 [tool.uv]
 package = true
 dev-dependencies = [

src/hope_dedup_engine/apps/api/admin/__init__.py

@@ -1,6 +1,7 @@
 from .config import ConfigAdmin  # noqa
 from .deduplicationset import DeduplicationSetAdmin  # noqa
-from .duplicate import DuplicateAdmin  # noqa
+from .finding import FindingAdmin  # noqa
 from .hdetoken import HDETokenAdmin  # noqa
+from .ignored_pair import IgnoredFilenamePairAdmin, IgnoredReferencePkPairAdmin  # noqa
 from .image import ImageAdmin  # noqa
 from .jobs import DedupJob  # noqa

src/hope_dedup_engine/apps/api/admin/config.py

@@ -2,22 +2,16 @@
 from typing import Any
 
 from django.contrib import messages
-from django.contrib.admin import ModelAdmin, register, site
-from django.core.exceptions import ValidationError
+from django.contrib.admin import ModelAdmin, register
 from django.db import models
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import redirect, render
 from django.urls import path, reverse
 
-from admin_extra_buttons.api import button
 from admin_extra_buttons.mixins import ExtraButtonsMixin
 from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
 
-from hope_dedup_engine.apps.api.forms import EditSchemaForm
 from hope_dedup_engine.apps.api.models import Config
-from hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
-from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
-from hope_dedup_engine.utils.security import is_root
 
 
 @register(Config)
@@ -31,15 +25,15 @@ class ConfigAdmin(ExtraButtonsMixin, ModelAdmin):
         }
     }
 
-    def get_changeform_initial_data(self, request: HttpRequest) -> dict[str, str]:
-        initial_data = super().get_changeform_initial_data(request)
-        initial_data["settings"] = {}
-        try:
-            schema = SchemaManager.get_or_create()
-            DefaultValidatingValidator(schema).validate(initial_data["settings"])
-        except ValidationError as e:
-            self.message_user(request, e.message, level=messages.ERROR)
-        return initial_data
+    # def get_changeform_initial_data(self, request: HttpRequest) -> dict[str, str]:
+    #     initial_data = super().get_changeform_initial_data(request)
+    #     initial_data["settings"] = {}
+    #     try:
+    #         schema = SchemaManager.get_or_create()
+    #         DefaultValidatingValidator(schema).validate(initial_data["settings"])
+    #     except ValidationError as e:
+    #         self.message_user(request, e.message, level=messages.ERROR)
+    #     return initial_data
 
     def get_urls(self):
         urls = super().get_urls()
@@ -49,11 +43,11 @@ def get_urls(self):
                 self.admin_site.admin_view(self.confirm_save),
                 name="confirm_save_config",
             ),
-            path(
-                "change-settings-schema/",
-                self.admin_site.admin_view(self.change_settings_schema),
-                name="change_settings_schema",
-            ),
+            # path(
+            #     "change-settings-schema/",
+            #     self.admin_site.admin_view(self.change_settings_schema),
+            #     name="change_settings_schema",
+            # ),
         ]
         return custom_urls + urls
 
@@ -92,40 +86,40 @@ def confirm_save(self, request, object_id) -> HttpResponse:  # pragma: no cover
             },
         )
 
-    @button(permission=is_root)
-    def change_settings_schema(
-        self, request: HttpRequest
-    ) -> HttpResponse:  # pragma: no cover
-        context = {
-            "opts": self.model._meta,
-            "site_header": site.site_header,
-            "title": "Change settings shema",
-            "trail_label": "Settings schema",
-            "has_view_permission": self.has_view_permission(request),
-        }
+    # @button(permission=is_root)
+    # def change_settings_schema(
+    #     self, request: HttpRequest
+    # ) -> HttpResponse:  # pragma: no cover
+    #     context = {
+    #         "opts": self.model._meta,
+    #         "site_header": site.site_header,
+    #         "title": "Change settings shema",
+    #         "trail_label": "Settings schema",
+    #         "has_view_permission": self.has_view_permission(request),
+    #     }
 
-        if request.method == "POST":
-            form = EditSchemaForm(request.POST)
-            if form.is_valid():
-                try:
-                    SchemaManager.save(form.cleaned_data["schema"])
-                except ValidationError as e:
-                    self.message_user(request, e.message, level=messages.ERROR)
-                else:
-                    self.message_user(request, "Schema has been updated.")
-                    return redirect(reverse("admin:api_config_changelist"))
-        else:
-            try:
-                form = EditSchemaForm(initial={"schema": SchemaManager.get_or_create()})
-            except ValidationError as e:
-                self.message_user(request, e.message, level=messages.ERROR)
-                return redirect(reverse("admin:api_config_changelist"))
+    #     if request.method == "POST":
+    #         form = EditSchemaForm(request.POST)
+    #         if form.is_valid():
+    #             try:
+    #                 SchemaManager.save(form.cleaned_data["schema"])
+    #             except ValidationError as e:
+    #                 self.message_user(request, e.message, level=messages.ERROR)
+    #             else:
+    #                 self.message_user(request, "Schema has been updated.")
+    #                 return redirect(reverse("admin:api_config_changelist"))
+    #     else:
+    #         try:
+    #             form = EditSchemaForm(initial={"schema": SchemaManager.get_or_create()})
+    #         except ValidationError as e:
+    #             self.message_user(request, e.message, level=messages.ERROR)
+    #             return redirect(reverse("admin:api_config_changelist"))
 
-        return render(
-            request,
-            "admin/api/config/change_settings_schema.html",
-            {
-                "form": form,
-                **context,
-            },
-        )
+    #     return render(
+    #         request,
+    #         "admin/api/config/change_settings_schema.html",
+    #         {
+    #             "form": form,
+    #             **context,
+    #         },
+    #     )

src/hope_dedup_engine/apps/api/admin/duplicate.py → src/hope_dedup_engine/apps/api/admin/finding.py

@@ -4,21 +4,23 @@
 from adminfilters.filters import DjangoLookupFilter, NumberFilter
 from adminfilters.mixin import AdminFiltersMixin
 
-from hope_dedup_engine.apps.api.models import Duplicate
+from hope_dedup_engine.apps.api.models import Finding
 
 
-@register(Duplicate)
-class DuplicateAdmin(AdminFiltersMixin, ModelAdmin):
+@register(Finding)
+class FindingAdmin(AdminFiltersMixin, ModelAdmin):
     list_display = (
         "id",
         "deduplication_set",
         "score",
         "first_reference_pk",
         "second_reference_pk",
+        "error",
     )
     list_filter = (
         ("deduplication_set", AutoCompleteFilter),
         ("score", NumberFilter),
+        ("error", NumberFilter),
         DjangoLookupFilter,
     )
 

src/hope_dedup_engine/apps/api/admin/ignored_pair.py

@@ -0,0 +1,25 @@
+from django.contrib import admin
+
+from adminfilters.autocomplete import AutoCompleteFilter
+from adminfilters.mixin import AdminFiltersMixin
+
+from hope_dedup_engine.apps.api.models import (
+    IgnoredFilenamePair,
+    IgnoredReferencePkPair,
+)
+
+
+class IgnoredPairBaseAdmin(AdminFiltersMixin, admin.ModelAdmin):
+    list_display = ("id", "first", "second", "deduplication_set")
+    list_filter = (("deduplication_set", AutoCompleteFilter),)
+    search_fields = ("first", "second")
+
+
+@admin.register(IgnoredReferencePkPair)
+class IgnoredReferencePkPairAdmin(IgnoredPairBaseAdmin):
+    pass
+
+
+@admin.register(IgnoredFilenamePair)
+class IgnoredFilenamePairAdmin(IgnoredPairBaseAdmin):
+    pass