Merge pull request #60 from davidmegginson/HDXDSYS-842

- add idps table, view, and VAT

changelog.md

@@ -5,11 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.8.16 (in progress)
+
+### Added
+
+- `idps` endpoint, view, and VAT
+- new constraint: `greater\_than\_constraint`
+
 ## 0.8.15
 
 ### Added
 
-- `coverage\_view`, which shows the locations available for each API endpoint
+- `data\_availability\_view`, which shows the locations available for each API endpoint
 
 ### Changed
 

src/hapi_schema/db_idps.py

@@ -0,0 +1,138 @@
+"""IDPs subcategory table and view."""
+
+from datetime import datetime
+
+from sqlalchemy import (
+    DateTime,
+    ForeignKey,
+    Integer,
+    select,
+)
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+from sqlalchemy.sql.expression import literal
+
+from hapi_schema.db_admin1 import DBAdmin1
+from hapi_schema.db_admin2 import DBAdmin2
+from hapi_schema.db_location import DBLocation
+from hapi_schema.db_resource import DBResource
+from hapi_schema.utils.base import Base
+from hapi_schema.utils.constraints import (
+    greater_than_constraint,
+    population_constraint,
+    reference_period_constraint,
+)
+from hapi_schema.utils.enums import DTMAssessmentType, build_enum_using_values
+from hapi_schema.utils.view_params import ViewParams
+
+
+class DBIDPs(Base):
+    __tablename__ = "idps"
+    __table_args__ = (
+        greater_than_constraint("reporting_round", 0),
+        population_constraint(),
+        reference_period_constraint(),
+    )
+
+    resource_hdx_id = mapped_column(
+        ForeignKey("resource.hdx_id", onupdate="CASCADE", ondelete="CASCADE"),
+        nullable=False,
+    )
+
+    admin2_ref = mapped_column(
+        ForeignKey("admin2.id", onupdate="CASCADE"), primary_key=True
+    )
+
+    assessment_type: Mapped[DTMAssessmentType] = mapped_column(
+        build_enum_using_values(DTMAssessmentType), primary_key=True
+    )
+
+    reporting_round: Mapped[int] = mapped_column(Integer, nullable=True)
+
+    population: Mapped[int] = mapped_column(
+        Integer, nullable=False, index=True
+    )
+
+    reference_period_start: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, index=True
+    )
+
+    reference_period_end: Mapped[datetime] = mapped_column(
+        DateTime, nullable=True, index=True
+    )
+
+    resource = relationship(DBResource)
+    admin2 = relationship(DBAdmin2)
+
+
+view_params_idps = ViewParams(
+    name="idps_view",
+    metadata=Base.metadata,
+    selectable=select(
+        *DBIDPs.__table__.columns,
+        DBLocation.code.label("location_code"),
+        DBLocation.name.label("location_name"),
+        DBLocation.has_hrp.label("has_hrp"),
+        DBLocation.in_gho.label("in_gho"),
+        DBAdmin1.code.label("admin1_code"),
+        DBAdmin1.name.label("admin1_name"),
+        DBAdmin1.is_unspecified.label("admin1_is_unspecified"),
+        DBAdmin1.location_ref.label("location_ref"),
+        DBAdmin2.code.label("admin2_code"),
+        DBAdmin2.name.label("admin2_name"),
+        DBAdmin2.is_unspecified.label("admin2_is_unspecified"),
+        DBAdmin2.admin1_ref.label("admin1_ref"),
+    ).select_from(
+        DBIDPs.__table__.join(
+            DBAdmin2.__table__,
+            DBIDPs.admin2_ref == DBAdmin2.id,
+            isouter=True,
+        )
+        .join(
+            DBAdmin1.__table__,
+            DBAdmin2.admin1_ref == DBAdmin1.id,
+            isouter=True,
+        )
+        .join(
+            DBLocation.__table__,
+            DBAdmin1.location_ref == DBLocation.id,
+            isouter=True,
+        )
+    ),
+)
+
+# Results format: category, subcategory, location_name, location_code, admin1_name, admin1_code, admin2_name, admin2_code, hapi_updated_date
+availability_stmt_idps = (
+    select(
+        literal("affected-people").label("category"),
+        literal("idps").label("subcategory"),
+        DBLocation.name.label("location_name"),
+        DBLocation.code.label("location_code"),
+        DBAdmin1.name.label("admin1_name"),
+        DBAdmin1.code.label("admin1_code"),
+        DBAdmin2.name.label("admin2_name"),
+        DBAdmin2.code.label("admin2_code"),
+        DBResource.hapi_updated_date,
+    )
+    .select_from(
+        DBIDPs.__table__.join(
+            DBAdmin2.__table__,
+            DBIDPs.admin2_ref == DBAdmin2.id,
+            isouter=True,
+        )
+        .join(
+            DBAdmin1.__table__,
+            DBAdmin2.admin1_ref == DBAdmin1.id,
+            isouter=True,
+        )
+        .join(
+            DBLocation.__table__,
+            DBAdmin1.location_ref == DBLocation.id,
+            isouter=True,
+        )
+        .join(
+            DBResource.__table__,
+            DBIDPs.resource_hdx_id == DBResource.hdx_id,
+        )
+    )
+    .distinct()
+)

src/hapi_schema/db_views_as_tables.py

@@ -270,6 +270,35 @@ class DBHumanitarianNeedsVAT(Base):
     admin1_ref: Mapped[int] = mapped_column(Integer)
 
 
+class DBIDPsVAT(Base):
+    __tablename__ = "idps_vat"
+    resource_hdx_id: Mapped[str] = mapped_column(String(36))
+    admin2_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
+    assessment_type: Mapped[str] = mapped_column(String(32), primary_key=True)
+    reporting_round: Mapped[int] = mapped_column(Integer, nullable=True)
+    population: Mapped[int] = mapped_column(Integer, index=True)
+    reference_period_start: Mapped[datetime] = mapped_column(
+        DateTime, primary_key=True
+    )
+    reference_period_end: Mapped[datetime] = mapped_column(
+        DateTime,
+        index=True,
+        nullable=True,
+    )
+    location_code: Mapped[str] = mapped_column(String(128), index=True)
+    location_name: Mapped[str] = mapped_column(String(512), index=True)
+    has_hrp: Mapped[bool] = mapped_column(Boolean)
+    in_gho: Mapped[bool] = mapped_column(Boolean)
+    admin1_code: Mapped[str] = mapped_column(String(128), index=True)
+    admin1_name: Mapped[str] = mapped_column(String(512), index=True)
+    admin1_is_unspecified: Mapped[bool] = mapped_column(Boolean)
+    location_ref: Mapped[int] = mapped_column(Integer)
+    admin2_code: Mapped[str] = mapped_column(String(128), index=True)
+    admin2_name: Mapped[str] = mapped_column(String(512), index=True)
+    admin2_is_unspecified: Mapped[bool] = mapped_column(Boolean)
+    admin1_ref: Mapped[int] = mapped_column(Integer)
+
+
 class DBLocationVAT(Base):
     __tablename__ = "location_vat"
     id: Mapped[int] = mapped_column(Integer, primary_key=True)

src/hapi_schema/utils/constraints.py

@@ -45,6 +45,15 @@ def non_negative_constraint(
     return CheckConstraint(sqltext=sqltext, name=f"{var_name}_constraint")
 
 
+def greater_than_constraint(
+    var_name: str,
+    min_value: 0,
+) -> CheckConstraint:
+    """Require a column to be non-negative."""
+    sqltext = f"{var_name} >= {min_value}"
+    return CheckConstraint(sqltext=sqltext, name=f"{var_name}_constraint")
+
+
 def percentage_constraint(var_name: str) -> CheckConstraint:
     sqltext = f"{var_name} >= 0. AND {var_name} <= 100."
     return CheckConstraint(sqltext=sqltext, name=f"{var_name}_constraint")

src/hapi_schema/utils/enums.py

@@ -23,6 +23,11 @@ class Gender(str, PythonEnum):
     ALL = "all"
 
 
+class DTMAssessmentType(str, PythonEnum):
+    BASELINE = "BA"
+    SITE = "SA"
+
+
 class DisabledMarker(str, PythonEnum):
     YES = "y"
     NO = "n"

tests/conftest.py

@@ -23,6 +23,7 @@
 from hapi_schema.db_humanitarian_needs import (
     DBHumanitarianNeeds,
 )
+from hapi_schema.db_idps import DBIDPs
 from hapi_schema.db_location import DBLocation
 from hapi_schema.db_national_risk import (
     DBNationalRisk,
@@ -51,6 +52,7 @@
 from sample_data.data_food_security import data_food_security
 from sample_data.data_funding import data_funding
 from sample_data.data_humanitarian_needs import data_humanitarian_needs
+from sample_data.data_idps import data_idps
 from sample_data.data_location import data_location
 from sample_data.data_national_risk import data_national_risk
 from sample_data.data_operational_presence import data_operational_presence
@@ -92,6 +94,7 @@ def session():
 
     session.execute(insert(DBConflictEvent), data_conflict_event)
     session.execute(insert(DBFunding), data_funding)
+    session.execute(insert(DBIDPs), data_idps)
     session.execute(insert(DBNationalRisk), data_national_risk)
     session.execute(insert(DBPopulation), data_population)
     session.execute(insert(DBOperationalPresence), data_operational_presence)

tests/sample_data/data_idps.py

@@ -0,0 +1,13 @@
+from datetime import datetime
+
+data_idps = [
+    dict(
+        resource_hdx_id="90deb235-1bf5-4bae-b231-3393222c2d01",
+        admin2_ref=2,
+        assessment_type="BA",
+        reporting_round=18,
+        population=25000,
+        reference_period_start=datetime(2024, 1, 1),
+        reference_period_end=datetime(2024, 12, 31),
+    ),
+]

tests/test_idps.py

@@ -0,0 +1,94 @@
+from datetime import datetime
+
+import pytest
+from hdx.database import Database
+
+from hapi_schema.db_idps import (
+    DBIDPs,
+    view_params_idps,
+)
+from hapi_schema.views import prepare_hapi_views
+
+
+def test_idps_view(run_view_test):
+    """Check idps view has all columns."""
+    view_idps = Database.prepare_view(view_params_idps.__dict__)
+    run_view_test(
+        view=view_idps,
+        whereclause=(
+            view_idps.c.resource_hdx_id
+            == "90deb235-1bf5-4bae-b231-3393222c2d01",
+            view_idps.c.admin2_ref == 2,
+            view_idps.c.assessment_type == "BA",
+            view_idps.c.reporting_round == 18,
+            view_idps.c.population == 25000,
+            view_idps.c.admin2_code == "FOO-001-XXX",
+            view_idps.c.admin1_code == "FOO-001",
+            view_idps.c.location_code == "FOO",
+        ),
+    )
+
+
+def test_idps_availability(run_view_test):
+    view_availability = prepare_hapi_views()
+    run_view_test(
+        view=view_availability,
+        whereclause=(
+            view_availability.c.category == "affected-people",
+            view_availability.c.subcategory == "idps",
+            view_availability.c.location_code == "FOO",
+            view_availability.c.admin1_code == "FOO-001",
+            view_availability.c.admin2_code == "FOO-001-XXX",
+            view_availability.c.hapi_updated_date == datetime(2023, 6, 1),
+        ),
+    )
+
+
+@pytest.fixture
+def base_parameters():
+    return dict(
+        resource_hdx_id="90deb235-1bf5-4bae-b231-3393222c2d01",
+        admin2_ref=2,
+        assessment_type="BA",
+        reporting_round=18,
+        population=25000,
+        reference_period_start=datetime(2020, 1, 1),
+        reference_period_end=datetime(2020, 1, 2),
+    )
+
+
+def test_reporting_round_constraint(run_constraints_test, base_parameters):
+    """Check that the reporting round is greater than 0"""
+    modified_params = {**base_parameters, "reporting_round": -1}
+    run_constraints_test(
+        new_rows=[
+            DBIDPs(**modified_params),
+        ],
+        expected_constraint="reporting_round_constraint",
+    )
+
+
+def test_population_positive(run_constraints_test, base_parameters):
+    """Check that the population value is positive"""
+    modified_params = {**base_parameters, "population": -1}
+    run_constraints_test(
+        new_rows=[
+            DBIDPs(**modified_params),
+        ],
+        expected_constraint="population_constraint",
+    )
+
+
+def test_reference_period_constraint(run_constraints_test, base_parameters):
+    """Check that reference_period_end cannot be less than start"""
+    modified_params = {
+        **base_parameters,
+        **dict(
+            reference_period_start=datetime(2023, 1, 2),
+            reference_period_end=datetime(2023, 1, 1),
+        ),
+    }
+    run_constraints_test(
+        new_rows=[DBIDPs(**modified_params)],
+        expected_constraint="reference_period_constraint",
+    )

tests/test_views_as_tables.py

@@ -7,6 +7,7 @@
 from hapi_schema.db_food_security import view_params_food_security
 from hapi_schema.db_funding import view_params_funding
 from hapi_schema.db_humanitarian_needs import view_params_humanitarian_needs
+from hapi_schema.db_idps import view_params_idps
 from hapi_schema.db_location import view_params_location
 from hapi_schema.db_national_risk import view_params_national_risk
 from hapi_schema.db_operational_presence import (
@@ -218,6 +219,28 @@ def test_humanitarian_needs_vat(
     run_primary_keys_test("humanitarian_needs_vat", expected_primary_keys)
 
 
+def test_idps_vat(run_indexes_test, run_columns_test, run_primary_keys_test):
+    """Check that funding_vat is correct - columns match, expected indexes present"""
+    expected_primary_keys = [
+        "admin2_ref",
+        "assessment_type",
+        "reference_period_start",
+    ]
+    expected_indexes = [
+        "population",
+        "reference_period_end",
+        "location_code",
+        "location_name",
+        "admin1_name",
+        "admin1_code",
+        "admin2_name",
+        "admin2_code",
+    ]
+    run_columns_test("idps_vat", "idps_view", view_params_idps)
+    run_primary_keys_test("idps_vat", expected_primary_keys)
+    run_indexes_test("idps_vat", expected_indexes)
+
+
 def test_location_vat(
     run_indexes_test, run_columns_test, run_primary_keys_test
 ):