More sophisticated algorithm that uses secondary historic rate for co…

…mparison if available

.config/coveragerc

@@ -14,4 +14,4 @@ exclude_also =
     if 0:
     if __name__ == .__main__.:
     if TYPE_CHECKING:
-    @(abc\.)?abstractmethod
+    @(abc\.)?abstractmethod

CONTRIBUTING.md

@@ -20,7 +20,7 @@ you make a git commit:
 
     pre-commit install
 
-The configuration file for this project is in a 
+The configuration file for this project is in a
 non-start location. Thus, you will need to edit your
 `.git/hooks/pre-commit` file to reflect this. Change
 the line that begins with `ARGS` to:
@@ -29,7 +29,7 @@ the line that begins with `ARGS` to:
 
 With pre-commit, all code is formatted according to
 [black]("https://github.com/psf/black") and
-[ruff]("https://github.com/charliermarsh/ruff") guidelines. 
+[ruff]("https://github.com/charliermarsh/ruff") guidelines.
 
 To check if your changes pass pre-commit without committing, run:
 
@@ -46,8 +46,8 @@ Follow the example set out already in ``api.rst`` as you write the documentation
 ## Packages
 
 [pip-tools](https://github.com/jazzband/pip-tools) is used for
-package management.  If you’ve introduced a new package to the 
-source code (i.e.anywhere in `src/`), please add it to the 
+package management.  If you’ve introduced a new package to the
+source code (i.e.anywhere in `src/`), please add it to the
 `project.dependencies` section of
 `pyproject.toml` with any known version constraints.
 

LICENSE

@@ -23,4 +23,4 @@ SOFTWARE.
 Regular expressions from countrycode
 (https://github.com/vincentarelbundock/countrycode) incorporated and relicensed
 under The MIT License (MIT) with the kind permission of Vincent Arel-Bundock,
-Nils Enevoldsen, and CJ Yetman.
+Nils Enevoldsen, and CJ Yetman.

requirements.txt

@@ -17,7 +17,7 @@ cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.3.1
+charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via typer
@@ -29,15 +29,15 @@ distlib==0.3.7
     # via virtualenv
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.13.0
+filelock==3.13.1
     # via virtualenv
 frictionless==5.16.0
     # via hdx-python-utilities
 hdx-python-utilities==3.6.2
     # via hdx-python-country (pyproject.toml)
 humanize==4.8.0
     # via frictionless
-identify==2.5.30
+identify==2.5.31
     # via pre-commit
 idna==3.4
     # via requests
@@ -130,7 +130,7 @@ rfc3986==2.0.0
     # via frictionless
 rich==13.6.0
     # via typer
-ruamel-yaml==0.18.2
+ruamel-yaml==0.18.5
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
@@ -173,7 +173,7 @@ validators==0.22.0
     # via frictionless
 virtualenv==20.24.6
     # via pre-commit
-wheel==0.41.2
+wheel==0.41.3
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities

src/hdx/location/Countries & Territories Taxonomy MVP - C&T Taxonomy with HXL Tags.csv

@@ -255,4 +255,4 @@ ID,HRinfo ID,RW ID,m49 numerical code,FTS API ID,Appears in UNTERM list,Appears
 255,346,178,580,166,,,N,N,MP,MNP,,,Northern Mariana Islands,,Northern Mariana Islands (the),,,,,Northern Mariana Islands (The United States of America),Northern Mariana Islands,,,,,,,0,25.56053736,152.1251419,9,Oceania,57,Micronesia,,,mariana,High,USD,1986-01-01,255 - Northern Mariana Islands
 256,,,680,,,,N,N,,,,,Sark,,,,,,,,Sark,,,,,,,,,,150,Europe,154,Northern Europe,830,Channel Islands,sark,,GBP,1974-01-01,256 - Sark
 265,425,246,850,245,,,N,N,VI,VIR,,,United States Virgin Islands,,Virgin Islands (U.S.),,"Virgin Islands, U.S.","Virgin Islands, U.S.",,,United States Virgin Islands,,,,,,,0,17.39137826,-64.67699419,19,Americas,419,Latin America and the Caribbean,29,Caribbean,^(?=.*\bu\.?\s?s).*virgin|^(?=.*states).*virgin,High,USD,1974-01-01,265 - United States Virgin Islands
-266,,,,,,,N,N,,,XK,XKX,Kosovo,,,,,,,,Kosovo,,,,,,,,,,150,Europe,39,Southern Europe,,,kosovo,,EUR,1974-01-01,266 - Kosovo
+266,,,,,,,N,N,,,XK,XKX,Kosovo,,,,,,,,Kosovo,,,,,,,,,,150,Europe,39,Southern Europe,,,kosovo,,EUR,1974-01-01,266 - Kosovo

src/hdx/location/currency.py

@@ -42,6 +42,7 @@ class Currency:
     _retriever = None
     _log_level = logging.DEBUG
     _fixed_now = None
+    _threshold = 1.3
 
     @classmethod
     def _get_int_timestamp(cls, date: datetime) -> int:
@@ -173,37 +174,90 @@ def _get_primary_rates_data(
             return None
 
     @classmethod
-    def _get_adjclose(cls, indicators: Dict) -> Optional[float]:
+    def _get_adjclose(
+        cls, indicators: Dict, currency: str, timestamp: int
+    ) -> Optional[float]:
         """
         Get the adjusted close fx rate from the indicators dictionary returned
         from the Yahoo API.
 
         Args:
             indicators (Dict): Indicators dictionary from Yahoo API
+            currency (str): Currency
+            timestamp (int): Timestamp to use for fx conversion
 
         Returns:
             Optional[float]: Adjusted close fx rate or None
         """
         adjclose = indicators["adjclose"][0].get("adjclose")
         if adjclose is None:
             return None
+
+        def beyond_threshold(x, y):
+            if max(x, y) / min(x, y) > cls._threshold:
+                return True
+            return False
+
+        def within_threshold(x, y):
+            if max(x, y) / min(x, y) > cls._threshold:
+                return False
+            return True
+
+        # Compare adjclose to other variables returned by Yahoo API
         adjclose = adjclose[0]
-        # compare with high and low to reveal errors from Yahoo feed
         quote = indicators["quote"][0]
+        open = quote.get("open")
+        fraction_ok = True
+        if open:
+            open = open[0]
+            if beyond_threshold(adjclose, open):
+                fraction_ok = False
         high = quote.get("high")
-        low = quote.get("low")
-        if high and low:
+        if high:
             high = high[0]
+            if beyond_threshold(adjclose, high):
+                fraction_ok = False
+        low = quote.get("low")
+        if low:
             low = low[0]
-            if adjclose > high:
-                diff = adjclose / high
-                if diff > 1.1:
-                    adjclose = low + (high - low) / 2
-            elif adjclose < low:
-                diff = low / adjclose
-                if diff > 1.1:
-                    adjclose = low + (high - low) / 2
-        return adjclose
+            if beyond_threshold(adjclose, low):
+                fraction_ok = False
+        if fraction_ok:
+            # if no discrepancies, adjclose is ok
+            return adjclose
+
+        if cls._no_historic:
+            secondary_fx_rate = None
+        else:
+            secondary_fx_rate = cls._get_secondary_historic_rate(
+                currency, timestamp
+            )
+        if not secondary_fx_rate:
+            # compare with high and low to reveal errors from Yahoo feed
+            if high and low:
+                if within_threshold(low, high):
+                    return low + (high - low) / 2
+            return None
+
+        # compare with secondary historic rate
+        if within_threshold(adjclose, secondary_fx_rate):
+            return adjclose
+        # if adjclose is wacky, find another value to return that is ok
+        if high and low:
+            if within_threshold(high, secondary_fx_rate) and within_threshold(
+                low, secondary_fx_rate
+            ):
+                return low + (high - low) / 2
+        if open:
+            if within_threshold(open, secondary_fx_rate):
+                return open
+        if high:
+            if within_threshold(high, secondary_fx_rate):
+                return high
+        if low:
+            if within_threshold(low, secondary_fx_rate):
+                return low
+        return secondary_fx_rate
 
     @classmethod
     def _get_primary_rate(
@@ -234,7 +288,7 @@ def _get_primary_rate(
         if not data:
             return None
         if get_close:
-            return cls._get_adjclose(data["indicators"])
+            return cls._get_adjclose(data["indicators"], currency, timestamp)
         return data["meta"]["regularMarketPrice"]
 
     @classmethod