diff --git a/VERSION b/VERSION index 227cea21..7ec1d6db 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 +2.1.0 diff --git a/apptax/migrations/versions/3c4762751898_taxref_tree_v2.py b/apptax/migrations/versions/3c4762751898_taxref_tree_v2.py new file mode 100644 index 00000000..6345caa0 --- /dev/null +++ b/apptax/migrations/versions/3c4762751898_taxref_tree_v2.py @@ -0,0 +1,90 @@ +"""create vm_taxref_tree v2 + +Revision ID: 3c4762751898 +Revises: 83d7105edb76 +Create Date: 2024-12-03 13:30:26.521216 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "3c4762751898" +down_revision = "83d7105edb76" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("DROP MATERIALIZED VIEW IF EXISTS taxonomie.vm_taxref_tree") + op.execute( + """ + CREATE MATERIALIZED VIEW taxonomie.vm_taxref_tree AS + WITH RECURSIVE + biota AS ( + SELECT + t.cd_nom, + t.cd_ref::TEXT::ltree AS path + FROM + taxonomie.taxref t + WHERE + t.cd_nom = 349525 + UNION ALL + SELECT + child.cd_nom AS cd_nom, + parent.path || child.cd_ref::TEXT AS path + FROM + taxonomie.taxref child + JOIN + taxonomie.taxref child_ref ON child.cd_ref = child_ref.cd_nom + JOIN + biota parent ON parent.cd_nom = child_ref.cd_sup + ), + orphans AS ( + SELECT + t.cd_nom, + t.cd_ref::TEXT::ltree AS path + FROM + taxonomie.taxref t + JOIN + taxonomie.taxref t_ref ON t.cd_ref = t_ref.cd_nom + LEFT JOIN + taxonomie.taxref parent ON t_ref.cd_sup = parent.cd_nom AND parent.cd_nom != t_ref.cd_nom + WHERE + parent.cd_nom IS NULL + ) + SELECT + cd_nom, + path + FROM + biota + UNION DISTINCT -- do not include biota twice + SELECT + cd_nom, + path + FROM + orphans + WITH DATA; + """ + ) + op.create_index( + index_name="taxref_tree_cd_nom_idx", + schema="taxonomie", + table_name="vm_taxref_tree", + columns=["cd_nom"], + unique=True, + ) + # required for these operators: <, <=, =, >=, >, @>, <@, @, ~, ? + op.create_index( + index_name="taxref_tree_path_idx", + schema="taxonomie", + table_name="vm_taxref_tree", + columns=["path"], + postgresql_using="gist", + ) + + +def downgrade(): + op.execute("DROP MATERIALIZED VIEW taxonomie.vm_taxref_tree") diff --git a/apptax/migrations/versions/83d7105edb76_taxref_tree.py b/apptax/migrations/versions/83d7105edb76_taxref_tree.py deleted file mode 100644 index caab1a5f..00000000 --- a/apptax/migrations/versions/83d7105edb76_taxref_tree.py +++ /dev/null @@ -1,77 +0,0 @@ -"""create vm_taxref_tree - -Revision ID: 83d7105edb76 -Revises: 44447746cacc -Create Date: 2024-10-05 17:40:11.302423 - -""" - -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = "83d7105edb76" -down_revision = "6a20cd1055ec" -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute( - """ - CREATE MATERIALIZED VIEW taxonomie.vm_taxref_tree AS - WITH RECURSIVE childs AS ( - SELECT - t.cd_nom, - t.cd_ref::TEXT::ltree AS path, - 1 AS path_length, - t_ref.cd_sup AS cd_sup - FROM - taxonomie.taxref t - JOIN taxonomie.taxref t_ref ON - t.cd_ref = t_ref.cd_nom - UNION ALL - SELECT - child.cd_nom AS cd_nom, - parent.cd_ref::TEXT || child.path AS path, - child.path_length + 1 AS path_length, - parent_ref.cd_sup AS cd_sup - FROM - childs child - JOIN taxonomie.taxref parent ON - child.cd_sup = parent.cd_nom - JOIN taxonomie.taxref parent_ref ON - parent.cd_ref = parent_ref.cd_nom - ) - SELECT - DISTINCT ON - (cd_nom) cd_nom, - path - FROM - childs - ORDER BY - cd_nom, - path_length DESC - WITH DATA; - """ - ) - op.create_index( - index_name="taxref_tree_cd_nom_idx", - schema="taxonomie", - table_name="vm_taxref_tree", - columns=["cd_nom"], - unique=True, - ) - # required for these operators: <, <=, =, >=, >, @>, <@, @, ~, ? - op.create_index( - index_name="taxref_tree_path_idx", - schema="taxonomie", - table_name="vm_taxref_tree", - columns=["path"], - postgresql_using="gist", - ) - - -def downgrade(): - op.execute("DROP MATERIALIZED VIEW taxonomie.vm_taxref_tree") diff --git a/apptax/migrations/versions/83d7105edb76_taxref_tree_v1.py b/apptax/migrations/versions/83d7105edb76_taxref_tree_v1.py new file mode 100644 index 00000000..386bdcfd --- /dev/null +++ b/apptax/migrations/versions/83d7105edb76_taxref_tree_v1.py @@ -0,0 +1,25 @@ +"""create vm_taxref_tree v1 + +Revision ID: 83d7105edb76 +Revises: 44447746cacc +Create Date: 2024-10-05 17:40:11.302423 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "83d7105edb76" +down_revision = "6a20cd1055ec" +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + op.execute("DROP MATERIALIZED VIEW IF EXISTS taxonomie.vm_taxref_tree") diff --git a/apptax/taxonomie/models.py b/apptax/taxonomie/models.py index ac7e2663..d0b244d5 100644 --- a/apptax/taxonomie/models.py +++ b/apptax/taxonomie/models.py @@ -216,7 +216,14 @@ def where_id_liste(cls, id_liste, *, query): def where_params(cls, filters=None, *, query): for filter in filters: - if hasattr(Taxref, filter) and filters[filter] != "": + # Test empty values + if not filters[filter]: + continue + + if hasattr(Taxref, filter) and isinstance(filters[filter], list): + col = getattr(Taxref, filter) + query = query.filter(col.in_(tuple(filters[filter]))) + elif hasattr(Taxref, filter) and filters[filter] != "": col = getattr(Taxref, filter) query = query.filter(col == filters[filter]) elif filter == "is_ref" and filters[filter] == "true": @@ -261,13 +268,17 @@ class BibListes(db.Model): @hybrid_property def nb_taxons(self): - return len(self.noms) + return db.session.scalar( + select([db.func.count(cor_nom_liste.c.cd_nom)]).where( + cor_nom_liste.c.id_liste == self.id_liste + ) + ) @nb_taxons.expression def nb_taxons(cls): return ( - db.select([db.func.count(cor_nom_liste.id_liste)]) - .where(BibListes.id_liste == cls.id_liste) + db.select([db.func.count(cor_nom_liste.c.cd_nom)]) + .where(cor_nom_liste.c.id_liste == cls.id_liste) .label("nb_taxons") ) diff --git a/apptax/taxonomie/routesbiblistes.py b/apptax/taxonomie/routesbiblistes.py index 512764c9..f0652634 100644 --- a/apptax/taxonomie/routesbiblistes.py +++ b/apptax/taxonomie/routesbiblistes.py @@ -3,16 +3,13 @@ import os import logging -from flask import Blueprint, request, current_app -from sqlalchemy import func, or_ -from sqlalchemy.orm import joinedload +from flask import Blueprint +from sqlalchemy import select -from pypnusershub import routes as fnauth from utils_flask_sqla.response import json_resp -from . import filemanager from . import db -from .models import BibListes, Taxref +from .models import BibListes from apptax.taxonomie.schemas import BibListesSchema adresses = Blueprint("bib_listes", __name__) @@ -24,26 +21,38 @@ @adresses.route("/", methods=["GET"]) @json_resp -def get_biblistes(id=None): +def get_biblistes(): """ retourne les contenu de bib_listes dans "data" et le nombre d'enregistrements dans "count" """ - data = db.session.query(BibListes).all() + biblistes_records = db.session.execute( + select( + BibListes.id_liste, + BibListes.code_liste, + BibListes.nom_liste, + BibListes.desc_liste, + BibListes.nb_taxons, + BibListes.regne, + BibListes.group2_inpn, + ) + ).all() biblistes_schema = BibListesSchema() - maliste = {"data": [], "count": 0} - maliste["count"] = len(data) - maliste["data"] = biblistes_schema.dump(data, many=True) - return maliste + biblistes_infos = { + "data": biblistes_schema.dump(biblistes_records, many=True), + "count": len(biblistes_records), + } + + return biblistes_infos @adresses.route("/", methods=["GET"], defaults={"group2_inpn": None}) @adresses.route("//", methods=["GET"]) def get_biblistesbyTaxref(regne, group2_inpn): - q = db.session.query(BibListes) + q = select(BibListes) if regne: q = q.where(BibListes.regne == regne) if group2_inpn: q = q.where(BibListes.group2_inpn == group2_inpn) - results = q.all() + results = db.session.scalars(q).all() return BibListesSchema().dump(results, many=True) diff --git a/apptax/taxonomie/routestaxref.py b/apptax/taxonomie/routestaxref.py index 8a46e45a..a31088c3 100644 --- a/apptax/taxonomie/routestaxref.py +++ b/apptax/taxonomie/routestaxref.py @@ -162,10 +162,18 @@ def get_taxref_list(): limit = request.values.get("limit", 20, int) page = request.values.get("page", 1, int) id_liste = None + cd_nom = None + if "id_liste" in request.values: - id_liste = request.values.get("id_liste").split(",") + id_liste = ( + request.values.get("id_liste").split(",") if request.values.get("id_liste") else None + ) + if "cd_nom" in request.values: + cd_nom = request.values.get("cd_nom").split(",") if request.values.get("cd_nom") else None + fields = request.values.get("fields", type=str, default=[]) parameters = request.values.to_dict() + parameters["cd_nom"] = cd_nom dump_options = {} if fields: @@ -178,10 +186,10 @@ def get_taxref_list(): joinedload_when_attributs = get_joinedload_when_attributs(fields=fields) query = Taxref.joined_load(fields).options(*joinedload_when_attributs) + query = Taxref.where_params(parameters, query=query) if id_liste and "-1" not in id_liste: query = Taxref.where_id_liste(id_liste, query=query) - count_filter = db.session.scalar( db.select(func.count()).select_from( Taxref.where_params(parameters, query=query), diff --git a/apptax/taxonomie/routestmedias.py b/apptax/taxonomie/routestmedias.py index 2856444a..f8ae7bfb 100644 --- a/apptax/taxonomie/routestmedias.py +++ b/apptax/taxonomie/routestmedias.py @@ -5,8 +5,8 @@ from flask import json, Blueprint, request, current_app, send_file, abort -from .models import TMedias -from .schemas import TMediasSchema +from .models import TMedias, BibTypesMedia +from .schemas import TMediasSchema, BibTypesMediaSchema from .filemanager import FILEMANAGER @@ -29,6 +29,19 @@ def get_tmedias(id=None): return TMediasSchema().dump(medias, many=True) +@adresses.route("/types", methods=["GET"]) +@adresses.route("/types/", methods=["GET"]) +def get_type_tmedias(id=None): + """ + Liste des types de mĂ©dias + """ + if id: + type_media = BibTypesMedia.query.get(id) + return BibTypesMediaSchema().dump(type_media) + types_media = BibTypesMedia.query.all() + return BibTypesMediaSchema().dump(types_media, many=True) + + @adresses.route("/bycdref/", methods=["GET"]) def get_tmediasbyTaxon(cd_ref): """ diff --git a/apptax/tests/fixtures.py b/apptax/tests/fixtures.py index e35103e7..aec6503f 100644 --- a/apptax/tests/fixtures.py +++ b/apptax/tests/fixtures.py @@ -95,7 +95,14 @@ def liste(): "code_liste": "TEST_LIST_Plantae", "nom_liste": "Liste test Plantae", "desc_liste": "Liste description", - "regne": "Plantea", + "regne": "Plantae", + }, + { + "code_liste": "TEST_LIST_Mousses", + "nom_liste": "Liste test Mousses", + "desc_liste": "Liste description", + "regne": "Plantae", + "group2_inpn": "Mousses", }, ] diff --git a/apptax/tests/test_biblistes.py b/apptax/tests/test_biblistes.py index 3e7c161f..784c293b 100644 --- a/apptax/tests/test_biblistes.py +++ b/apptax/tests/test_biblistes.py @@ -8,22 +8,13 @@ @pytest.mark.usefixtures("client_class", "temporary_transaction") class TestApiBibListe: - schema_cor_nom_liste = Schema( - { - "items": [{"cd_nom": int, "id_liste": int}], - "total": int, - "limit": int, - "page": int, - } - ) - schema_allnamebyListe = Schema( [ { "id_liste": int, "code_liste": str, "nom_liste": str, - "desc_liste": str, + "desc_liste": Or(str, None), "regne": Or(str, None), "group2_inpn": Or(str, None), "nb_taxons": int, @@ -31,7 +22,7 @@ class TestApiBibListe: ] ) - def test_get_biblistes(self): + def test_get_biblistes(self, listes): query_string = {"limit": 10} response = self.client.get( url_for( @@ -45,7 +36,6 @@ def test_get_biblistes(self): assert self.schema_allnamebyListe.is_valid(data["data"]) def test_get_biblistesbyTaxref(self, listes): - response = self.client.get( url_for("bib_listes.get_biblistesbyTaxref", regne="Animalia", group2_inpn=None), ) @@ -53,3 +43,11 @@ def test_get_biblistesbyTaxref(self, listes): data = [d for d in response.json if d["desc_liste"] == "Liste description"] self.schema_allnamebyListe.validate(data) assert len(data) == 1 + + response = self.client.get( + url_for("bib_listes.get_biblistesbyTaxref", regne="Plantae", group2_inpn="Mousses"), + ) + # Filter test list only + data = [d for d in response.json if d["desc_liste"] == "Liste description"] + self.schema_allnamebyListe.validate(data) + assert len(data) == 1 diff --git a/apptax/tests/test_media.py b/apptax/tests/test_media.py index d62b788c..69fb287a 100644 --- a/apptax/tests/test_media.py +++ b/apptax/tests/test_media.py @@ -15,6 +15,7 @@ AppUser, ) from pypnusershub.tests.utils import set_logged_user_cookie +from schema import Schema, Optional, Or from .fixtures import noms_example, attribut_example, liste @@ -40,6 +41,21 @@ def user(): @pytest.mark.usefixtures("client_class", "temporary_transaction") class TestAPIMedia: + + type_media_schema = Schema( + [{"desc_type_media": Or(None, str), "id_type": int, "nom_type_media": str}] + ) + + def test_get_type_tmedias(self): + response = self.client.get(url_for("t_media.get_type_tmedias")) + assert response.status_code == 200 + assert self.type_media_schema.is_valid(response.json) + + def test_get_type_tmedias_one(self): + response = self.client.get(url_for("t_media.get_type_tmedias", id=1)) + assert response.status_code == 200 + assert response.json["nom_type_media"] == "Photo_principale" + def test_get_tmediasbyTaxon(self, noms_example): response = self.client.get(url_for("t_media.get_tmediasbyTaxon", cd_ref=67111)) assert response.status_code == 200 diff --git a/apptax/tests/test_taxref.py b/apptax/tests/test_taxref.py index 7b4b8c16..fe78249d 100644 --- a/apptax/tests/test_taxref.py +++ b/apptax/tests/test_taxref.py @@ -305,3 +305,27 @@ def test_get_taxref_list(self, liste_with_names): assert response.status_code == 200 response_json = response.json assert len(response_json["items"]) == limit + + def test_get_taxref_list_filters_cd_noms(self): + # Test de la route taxref avec des filtres sur le cd_nom + # 1 cd_nom + response = self.client.get( + url_for("taxref.get_taxref_list"), + query_string={"cd_nom": 103536}, + ) + assert response.status_code == 200 + assert len(response.json["items"]) == 1 + + # Plusieurs cd_nom + response = self.client.get( + url_for("taxref.get_taxref_list"), + query_string={"cd_nom": "103536, 461885,2891"}, + ) + assert len(response.json["items"]) == 3 + + # Valeur vide + response = self.client.get( + url_for("taxref.get_taxref_list"), + query_string={"cd_nom": ""}, + ) + assert response.status_code == 200 diff --git a/docs/changelog.md b/docs/changelog.md index c013dcdf..b4d72d46 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,15 @@ # CHANGELOG +2.1.0 (2024-12-06) +------------------ + +**🚀 NouveautĂ©s** + +- Optimisation de la VM `vm_taxref_tree` pour en amĂ©liorer les performances et gĂ©rer diffĂ©rents cas oĂč des taxons locaux ont Ă©tĂ© ajoutĂ©s Ă  la table `taxref` (#587) +- Ajout d'une route `/tmedias/types/` renvoyant la liste des types de mĂ©dias (#588) +- AmĂ©lioration des performances de la route `/biblistes/` (#584) +- Ajout de la possibilitĂ© de filtrer la route `/taxref/` par une liste de cd_nom (#581) + 2.0.0 (2024-10-29) ------------------ diff --git a/docs/developpement.md b/docs/developpement.md index 35d0a6a6..eecaa88e 100644 --- a/docs/developpement.md +++ b/docs/developpement.md @@ -8,7 +8,8 @@ - limit (defaut = 50) : nombre d'Ă©lĂ©ments Ă  retourner - page (defaut = 0) : page Ă  retourner - is_ref (default = false) : ne retourne que les noms valides (cd_nom = cd_ref) - - id_liste + - id_liste : liste d'identifiant des listes + - cd_nom : liste de cd_nom - fields (permet de spĂ©cifier les champs renvoyĂ©s). Permet aussi de rĂ©cupĂ©rer les donnĂ©es secondaires non renvoyĂ©es par dĂ©faut, en les spĂ©cifiant explicitement (`fields=status,rang,medias,attributs,synonymes,listes`) - nomColonne : Permet de filtrer @@ -79,3 +80,18 @@ > Ces valeurs sont issues de la charte des codes couleurs pour les statuts Liste rouge, dĂ©finis internationalement par l'Union internationale pour la conservation de la nature (UICN) \ > (page 55) + +## MĂ©dias + +- `/tmedias/thumbnail/` : Retourne un mĂ©dia redimensionnĂ© aux dimensions spĂ©cifiĂ©es (vignette) + - Params : + - id_media : identifiant du mĂ©dia + - h (defaut = 300) : hauteur souhaitĂ©e + - w (defaut = 400) : largeur souhaitĂ©e + - regenerate : force la rĂ©gĂ©nĂ©ration du fichier thumbnail +- `/tmedias/types` : Retourne la liste des types de mĂ©dias +- `/tmedias/types/` : Retourne le dĂ©tail d'un type de mĂ©dia +- `/tmedias/` : Retourne la liste de tous les mĂ©dias de TaxHub + - Attention route non paginĂ©e et sans filtre, donc peut crasher si il y a beaucoup de mĂ©dias +- `/tmedias/` : Retourne le dĂ©tail d'un mĂ©dia +- `/tmedias/bycdref/` : Retourne la liste des mĂ©dias associĂ©s Ă  un taxon