Merge pull request #17 from PnX-SI/feat/geom_4326

Remove geojson_4326, add geom_4326

.github/workflows/pytest.yml

@@ -66,7 +66,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: "pip"
       - name: Install GDAL
         run: |
           sudo apt update

requirements.in

@@ -1,8 +1,9 @@
 alembic
 flask>=2.1
 flask-sqlalchemy
+flask-marshmallow
 python-dotenv
 sqlalchemy>=1.4,<2
 utils-flask-sqlalchemy>=0.3.0
-utils-flask-sqlalchemy-geo>=0.2.4
+utils-flask-sqlalchemy-geo>=0.2.8
 psycopg2

src/ref_geo/env.py

@@ -2,6 +2,7 @@
 from importlib import import_module
 
 from flask_sqlalchemy import SQLAlchemy
+from flask_marshmallow import Marshmallow
 
 
 db_path = environ.get("FLASK_SQLALCHEMY_DB")
@@ -14,4 +15,16 @@
     environ["FLASK_SQLALCHEMY_DB"] = f"{__name__}.db"
 
 
-__all__ = ["db"]
+ma_path = environ.get("FLASK_MARSHMALLOW")
+if ma_path and ma_path != f"{__name__}.ma":
+    ma_module_name, ma_object_name = ma_path.rsplit(".", 1)
+    ma_module = import_module(ma_module_name)
+    ma = getattr(ma_module, ma_object_name)
+else:
+    ma = Marshmallow()
+    environ["FLASK_MARSHMALLOW"] = f"{__name__}.ma"
+    ma.SQLAlchemySchema.OPTIONS_CLASS.session = db.session
+    ma.SQLAlchemyAutoSchema.OPTIONS_CLASS.session = db.session
+
+
+__all__ = ["db", "ma"]

src/ref_geo/migrations/utils.py

@@ -14,12 +14,26 @@
 schema = "ref_geo"
 
 
-"""
-Supprimer les zones d’un type donnée, e.g. 'DEP', 'COM', …
-"""
+def geom_4326_exists():
+    return (
+        op.get_bind()
+        .execute(
+            """
+            SELECT EXISTS (
+                SELECT 1
+                FROM information_schema.COLUMNS
+                WHERE table_schema = 'ref_geo' AND table_name='l_areas' AND column_name = 'geom_4326'
+            )
+            """
+        )
+        .scalar()
+    )
 
 
 def delete_area_with_type(area_type):
+    """
+    Supprimer les zones d’un type donnée, e.g. 'DEP', 'COM', …
+    """
     op.execute(
         f"""
         DELETE FROM {schema}.l_areas la
@@ -52,18 +66,33 @@ def create_temporary_grids_table(schema, temp_table_name):
 
 def insert_grids_and_drop_temporary_table(schema, temp_table_name, area_type):
     logger.info("Copy grids in l_areas…")
-    op.execute(
-        f"""
-        INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
-        SELECT
-            {schema}.get_id_area_type('{area_type}') AS id_type,
-            cd_sig,
-            code,
-            ST_Transform(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
-            geojson
-        FROM {schema}.{temp_table_name}
-    """
-    )
+    if geom_4326_exists():
+        # We insert geom and geom_4326 to avoid double conversion like 2154 → 3312 → 4326
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geom_4326)
+            SELECT
+                {schema}.get_id_area_type('{area_type}') AS id_type,
+                cd_sig,
+                code,
+                ST_Transform(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                ST_SetSRID(ST_GeomFromGeoJSON(geojson), 4326)
+            FROM {schema}.{temp_table_name}
+        """
+        )
+    else:  # legacy column geojson_4326
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
+            SELECT
+                {schema}.get_id_area_type('{area_type}') AS id_type,
+                cd_sig,
+                code,
+                ST_Transform(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                geojson
+            FROM {schema}.{temp_table_name}
+        """
+        )
     logger.info("Copy grids in li_grids…")
     op.execute(
         f"""

src/ref_geo/migrations/versions/0dfdbfbccd63_ref_geo_french_municipalities.py

@@ -10,6 +10,7 @@
 from ref_geo.migrations.utils import (
     schema,
     delete_area_with_type,
+    geom_4326_exists,
 )
 from utils_flask_sqla.migrations.utils import logger, open_remote_file
 
@@ -59,24 +60,44 @@ def upgrade():
         logger.info("Inserting municipalities data in temporary table…")
         cursor.copy_expert(f"COPY {schema}.{temp_table_name} FROM STDIN", geofile)
     logger.info("Copy municipalities in l_areas…")
-    op.execute(
-        f"""
-        INSERT INTO {schema}.l_areas (
-            id_type,
-            area_code,
-            area_name,
-            geom,
-            geojson_4326
+    if geom_4326_exists():
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (
+                id_type,
+                area_code,
+                area_name,
+                geom,
+                geom_4326
+            )
+            SELECT
+                {schema}.get_id_area_type('COM') AS id_type,
+                insee_com,
+                nom_com,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                ST_SetSRID(ST_GeomFromGeoJSON(geojson), 4326)
+            FROM {schema}.{temp_table_name}
+        """
+        )
+    else:
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (
+                id_type,
+                area_code,
+                area_name,
+                geom,
+                geojson_4326
+            )
+            SELECT
+                {schema}.get_id_area_type('COM') AS id_type,
+                insee_com,
+                nom_com,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                geojson
+            FROM {schema}.{temp_table_name}
+        """
         )
-        SELECT
-            {schema}.get_id_area_type('COM') AS id_type,
-            insee_com,
-            nom_com,
-            ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
-            geojson
-        FROM {schema}.{temp_table_name}
-    """
-    )
     logger.info("Copy municipalities in li_municipalities…")
     op.execute(
         f"""

src/ref_geo/migrations/versions/3fdaa1805575_ref_geo_french_departments.py

@@ -7,7 +7,7 @@
 from alembic import op
 from shutil import copyfileobj
 
-from ref_geo.migrations.utils import schema, delete_area_with_type
+from ref_geo.migrations.utils import schema, delete_area_with_type, geom_4326_exists
 from utils_flask_sqla.migrations.utils import logger, open_remote_file
 
 
@@ -50,18 +50,32 @@ def upgrade():
         logger.info("Inserting departments data in temporary table…")
         cursor.copy_expert(f"COPY {schema}.{temp_table_name} FROM STDIN", geofile)
     logger.info("Copy departments data in l_areas…")
-    op.execute(
-        f"""
-        INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
-        SELECT
-            {schema}.get_id_area_type('DEP') AS id_type,
-            insee_dep,
-            nom_dep,
-            ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
-            geojson
-        FROM {schema}.{temp_table_name}
-    """
-    )
+    if geom_4326_exists():
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geom_4326)
+            SELECT
+                {schema}.get_id_area_type('DEP') AS id_type,
+                insee_dep,
+                nom_dep,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                ST_SetSRID(ST_GeomFromGeoJSON(geojson), 4326)
+            FROM {schema}.{temp_table_name}
+        """
+        )
+    else:
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
+            SELECT
+                {schema}.get_id_area_type('DEP') AS id_type,
+                insee_dep,
+                nom_dep,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                geojson
+            FROM {schema}.{temp_table_name}
+        """
+        )
     logger.info("Re-indexing…")
     op.execute(f"REINDEX INDEX {schema}.index_l_areas_geom")
     logger.info("Dropping temporary departments table…")