Add facility to obtain WFP exchange rates

Allow current and historic rates to be passed to Currency.setup() to prefill the cache

pyproject.toml

@@ -50,12 +50,16 @@ Homepage = "https://github.com/OCHA-DAP/hdx-python-country"
 [project.optional-dependencies]
 test = ["pytest", "pytest-cov"]
 dev = ["pre-commit"]
+wfp = ["data-bridges-client@git+ssh://[email protected]/WFP-VAM/DataBridgesAPI@dev#egg=data-bridges-client"]
 
 
 #########
 # Hatch #
 #########
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 # Build
 
 [tool.hatch.build.targets.wheel]
@@ -76,7 +80,7 @@ version_scheme = "python-simplified-semver"
 # Tests
 
 [tool.hatch.envs.test]
-features = ["test"]
+features = ["test", "wfp"]
 
 [tool.hatch.envs.test.scripts]
 test = """

requirements.txt

@@ -2,17 +2,22 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --all-extras --output-file=requirements.txt --resolver=backtracking pyproject.toml
+#    pip-compile --all-extras --output-file=requirements.txt pyproject.toml
 #
 annotated-types==0.6.0
     # via pydantic
+anyio==4.3.0
+    # via httpx
 attrs==23.2.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
 certifi==2024.2.2
-    # via requests
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
@@ -24,7 +29,11 @@ click==8.1.7
 colorama==0.4.6
     # via typer
 coverage[toml]==7.4.3
-    # via pytest-cov
+    # via
+    #   coverage
+    #   pytest-cov
+data-bridges-client @ git+ssh://[email protected]/WFP-VAM/DataBridgesAPI@dev
+    # via hdx-python-country (pyproject.toml)
 distlib==0.3.8
     # via virtualenv
 et-xmlfile==1.1.0
@@ -33,14 +42,23 @@ filelock==3.13.1
     # via virtualenv
 frictionless==5.16.1
     # via hdx-python-utilities
+h11==0.14.0
+    # via httpcore
 hdx-python-utilities==3.6.5
     # via hdx-python-country (pyproject.toml)
+httpcore==1.0.4
+    # via httpx
+httpx==0.27.0
+    # via data-bridges-client
 humanize==4.9.0
     # via frictionless
 identify==2.5.35
     # via pre-commit
 idna==3.6
-    # via requests
+    # via
+    #   anyio
+    #   httpx
+    #   requests
 ijson==3.2.3
     # via hdx-python-utilities
 iniconfig==2.0.0
@@ -87,8 +105,10 @@ ply==3.11
     #   libhxl
 pre-commit==3.6.2
     # via hdx-python-country (pyproject.toml)
-pydantic==2.6.2
-    # via frictionless
+pydantic==2.6.3
+    # via
+    #   data-bridges-client
+    #   frictionless
 pydantic-core==2.16.3
     # via pydantic
 pygments==2.17.2
@@ -105,6 +125,7 @@ pytest-cov==4.1.0
     # via hdx-python-country (pyproject.toml)
 python-dateutil==2.8.2
     # via
+    #   data-bridges-client
     #   frictionless
     #   hdx-python-utilities
     #   libhxl
@@ -128,7 +149,7 @@ requests-file==2.0.0
     # via hdx-python-utilities
 rfc3986==2.0.0
     # via frictionless
-rich==13.7.0
+rich==13.7.1
     # via typer
 ruamel-yaml==0.18.6
     # via hdx-python-utilities
@@ -142,6 +163,10 @@ six==1.16.0
     # via
     #   isodate
     #   python-dateutil
+sniffio==1.3.1
+    # via
+    #   anyio
+    #   httpx
 stringcase==1.2.0
     # via frictionless
 structlog==24.1.0
@@ -153,9 +178,12 @@ tabulate==0.9.0
 text-unidecode==1.3
     # via python-slugify
 typer[all]==0.9.0
-    # via frictionless
+    # via
+    #   frictionless
+    #   typer
 typing-extensions==4.10.0
     # via
+    #   data-bridges-client
     #   frictionless
     #   pydantic
     #   pydantic-core
@@ -164,8 +192,9 @@ unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.1
+urllib3==2.0.7
     # via
+    #   data-bridges-client
     #   libhxl
     #   requests
 validators==0.22.0

src/hdx/location/__init__.py

@@ -1 +1,17 @@
+from datetime import datetime
+
 from ._version import version as __version__  # noqa: F401
+from hdx.utilities.dateparse import get_timestamp_from_datetime
+
+
+def get_int_timestamp(date: datetime) -> int:
+    """
+    Get integer timestamp from datetime object
+
+    Args:
+        date (datetime): datetime object
+
+    Returns:
+        int: Integer timestamp
+    """
+    return int(round(get_timestamp_from_datetime(date)))

src/hdx/location/adminlevel.py

@@ -58,7 +58,7 @@ def __init__(
             "admin_name_replacements", {}
         )
         self.admin_fuzzy_dont = admin_config.get("admin_fuzzy_dont", list())
-        self.pcodes = list()
+        self.pcodes = []
         self.pcode_lengths = {}
         self.name_to_pcode = {}
         self.pcode_to_name = {}
@@ -634,7 +634,7 @@ def output_matches(self) -> List[str]:
         Returns:
             List[str]: List of matches
         """
-        output = list()
+        output = []
         for match in sorted(self.matches):
             line = f"{match[0]} - {match[1]}: Matching ({match[4]}) {match[2]} to {match[3]} on map"
             logger.info(line)
@@ -647,7 +647,7 @@ def output_ignored(self) -> List[str]:
         Returns:
             List[str]: List of ignored
         """
-        output = list()
+        output = []
         for ignored in sorted(self.ignored):
             if len(ignored) == 2:
                 line = f"{ignored[0]} - Ignored {ignored[1]}!"
@@ -663,7 +663,7 @@ def output_errors(self) -> List[str]:
         Returns:
             List[str]: List of errors
         """
-        output = list()
+        output = []
         for error in sorted(self.errors):
             if len(error) == 2:
                 line = f"{error[0]} - Could not find {error[1]} in map names!"

src/hdx/location/country.py

@@ -69,8 +69,8 @@ class Country:
     _countriesdata = None
     _ochaurl_default = "https://docs.google.com/spreadsheets/d/1NjSI2LaS3SqbgYc0HdD8oIb7lofGtiHgoKKATCpwVdY/export?format=csv&gid=1088874596"
     _ochaurl = _ochaurl_default
-    _country_name_overrides = dict()
-    _country_name_mappings = dict()
+    _country_name_overrides = {}
+    _country_name_mappings = {}
 
     @classmethod
     def _add_countriesdata(cls, iso3: str, hxlcountry: hxl.Row) -> Dict:
@@ -167,16 +167,16 @@ def set_countriesdata(cls, countries: str) -> None:
         Returns:
             None
         """
-        cls._countriesdata = dict()
-        cls._countriesdata["countries"] = dict()
-        cls._countriesdata["iso2iso3"] = dict()
-        cls._countriesdata["m49iso3"] = dict()
-        cls._countriesdata["countrynames2iso3"] = dict()
-        cls._countriesdata["regioncodes2countries"] = dict()
-        cls._countriesdata["regioncodes2names"] = dict()
-        cls._countriesdata["regionnames2codes"] = dict()
-        cls._countriesdata["aliases"] = dict()
-        cls._countriesdata["currencies"] = dict()
+        cls._countriesdata = {}
+        cls._countriesdata["countries"] = {}
+        cls._countriesdata["iso2iso3"] = {}
+        cls._countriesdata["m49iso3"] = {}
+        cls._countriesdata["countrynames2iso3"] = {}
+        cls._countriesdata["regioncodes2countries"] = {}
+        cls._countriesdata["regioncodes2names"] = {}
+        cls._countriesdata["regionnames2codes"] = {}
+        cls._countriesdata["aliases"] = {}
+        cls._countriesdata["currencies"] = {}
 
         for key, value in cls._country_name_mappings.items():
             cls._countriesdata["countrynames2iso3"][
@@ -891,4 +891,4 @@ def get_countries_in_region(
 
         if exception is not None:
             raise exception
-        return list()
+        return []

src/hdx/location/currency.py

@@ -1,10 +1,11 @@
 """Currency conversion"""
 import logging
+from copy import copy
 from datetime import datetime, timezone
 from typing import Dict, Optional, Union
 
+from . import get_int_timestamp
 from hdx.utilities.dateparse import (
-    get_timestamp_from_datetime,
     now_utc,
     parse_date,
 )
@@ -44,19 +45,6 @@ class Currency:
     _fixed_now = None
     _threshold = 1.3
 
-    @classmethod
-    def _get_int_timestamp(cls, date: datetime) -> int:
-        """
-        Get integer timestamp from datetime object
-
-        Args:
-            date (datetime): datetime object
-
-        Returns:
-            int: Integer timestamp
-        """
-        return int(round(get_timestamp_from_datetime(date)))
-
     @classmethod
     def setup(
         cls,
@@ -69,6 +57,8 @@ def setup(
         no_historic: bool = False,
         fixed_now: Optional[datetime] = None,
         log_level: int = logging.DEBUG,
+        current_rates_cache: Dict = {"USD": 1},
+        historic_rates_cache: Dict = {},
     ) -> None:
         """
         Setup the sources. If you wish to use a static fallback file by setting
@@ -85,13 +75,15 @@ def setup(
             no_historic (bool): Do not set up historic rates. Defaults to False.
             fixed_now (Optional[datetime]): Use a fixed datetime for now. Defaults to None (use datetime.now()).
             log_level (int): Level at which to log messages. Defaults to logging.DEBUG.
+            current_rates_cache (Dict): Pre-populate current rates cache with given values. Defaults to {"USD": 1}.
+            historic_rates_cache (Dict): Pre-populate historic rates cache with given values. Defaults to {}.
 
         Returns:
             None
         """
 
-        cls._cached_current_rates = {"USD": 1}
-        cls._cached_historic_rates = dict()
+        cls._cached_current_rates = copy(current_rates_cache)
+        cls._cached_historic_rates = copy(historic_rates_cache)
         cls._rates_api = primary_rates_url
         cls._secondary_rates = None
         cls._secondary_historic = None
@@ -131,10 +123,10 @@ def setup(
                 filename="historic_rates.csv",
                 logstr="secondary historic exchange rates",
             )
-            cls._secondary_historic = dict()
+            cls._secondary_historic = {}
             for row in iterator:
                 currency = row["Currency"]
-                date = cls._get_int_timestamp(parse_date(row["Date"]))
+                date = get_int_timestamp(parse_date(row["Date"]))
                 rate = float(row["Rate"])
                 dict_of_dicts_add(
                     cls._secondary_historic, currency, date, rate
@@ -281,7 +273,7 @@ def _get_primary_rate(
             else:
                 now = now_utc()
                 get_close = False
-            timestamp = cls._get_int_timestamp(now)
+            timestamp = get_int_timestamp(now)
         else:
             get_close = True
         data = cls._get_primary_rates_data(currency, timestamp)
@@ -479,7 +471,7 @@ def get_historic_rate(
             )
         else:
             date = date.astimezone(timezone.utc)
-        timestamp = cls._get_int_timestamp(date)
+        timestamp = get_int_timestamp(date)
         if currency_data is not None:
             fx_rate = currency_data.get(timestamp)
             if fx_rate is not None:

src/hdx/location/phonetics.py

@@ -11,7 +11,7 @@ def match(
         possible_names: ListTuple,
         name: str,
         alternative_name: Optional[str] = None,
-        transform_possible_names: ListTuple[Callable] = list(),
+        transform_possible_names: ListTuple[Callable] = [],
         threshold: int = 2,
     ) -> Optional[int]:
         """