save use saved fucntionality by passing retriever

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hdx-python-utilities>=3.7.3",
+    "hdx-python-utilities>=3.7.4",
     "libhxl>=5.2.1",
     "pyphonetics",
 ]

requirements.txt

@@ -24,17 +24,17 @@ distlib==0.3.8
     # via virtualenv
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.15.4
+filelock==3.16.1
     # via virtualenv
-frictionless==5.17.1
+frictionless==5.18.0
     # via hdx-python-utilities
-hdx-python-utilities==3.7.3
+hdx-python-utilities==3.7.4
     # via hdx-python-country (pyproject.toml)
 humanize==4.10.0
     # via frictionless
-identify==2.6.0
+identify==2.6.1
     # via pre-commit
-idna==3.8
+idna==3.10
     # via requests
 ijson==3.3.0
     # via hdx-python-utilities
@@ -74,7 +74,7 @@ packaging==24.1
     # via pytest
 petl==1.7.15
     # via frictionless
-platformdirs==4.2.2
+platformdirs==4.3.6
     # via virtualenv
 pluggy==1.5.0
     # via pytest
@@ -84,15 +84,15 @@ ply==3.11
     #   libhxl
 pre-commit==3.8.0
     # via hdx-python-country (pyproject.toml)
-pydantic==2.9.0
+pydantic==2.9.2
     # via frictionless
-pydantic-core==2.23.2
+pydantic-core==2.23.4
     # via pydantic
 pygments==2.18.0
     # via rich
 pyphonetics==0.5.3
     # via hdx-python-country (pyproject.toml)
-pytest==8.3.2
+pytest==8.3.3
     # via
     #   hdx-python-country (pyproject.toml)
     #   pytest-cov
@@ -127,7 +127,7 @@ requests-file==2.1.0
     # via hdx-python-utilities
 rfc3986==2.0.0
     # via frictionless
-rich==13.8.0
+rich==13.9.1
     # via typer
 rpds-py==0.20.0
     # via
@@ -163,19 +163,17 @@ typing-extensions==4.12.2
     #   pydantic
     #   pydantic-core
     #   typer
-tzdata==2024.1
-    # via pydantic
 unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.2
+urllib3==2.2.3
     # via
     #   libhxl
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.26.3
+virtualenv==20.26.6
     # via pre-commit
 wheel==0.44.0
     # via libhxl

src/hdx/location/adminlevel.py

@@ -8,7 +8,9 @@
 
 from hdx.location.country import Country
 from hdx.location.phonetics import Phonetics
+from hdx.utilities.base_downloader import DownloadError
 from hdx.utilities.dictandlist import dict_of_sets_add
+from hdx.utilities.retriever import Retrieve
 from hdx.utilities.text import multiple_replace, normalise
 from hdx.utilities.typehint import ListTuple
 
@@ -34,6 +36,7 @@ class AdminLevel:
         admin_config (Dict): Configuration dictionary. Defaults to {}.
         admin_level (int): Admin level. Defaults to 1.
         admin_level_overrides (Dict): Countries at other admin levels.
+        retriever (Optional[Retrieve]): Retriever object to use for loading/saving files. Defaults to None.
     """
 
     pcode_regex = re.compile(r"^([a-zA-Z]{2,3})(\d+)$")
@@ -47,9 +50,11 @@ def __init__(
         admin_config: Dict = {},
         admin_level: int = 1,
         admin_level_overrides: Dict = {},
+        retriever: Optional[Retrieve] = None,
     ) -> None:
         self.admin_level = admin_level
         self.admin_level_overrides = admin_level_overrides
+        self.retriever: Optional[Retrieve] = retriever
         self.countries_fuzzy_try = admin_config.get("countries_fuzzy_try")
         self.admin_name_mappings = admin_config.get("admin_name_mappings", {})
         self.admin_name_replacements = admin_config.get(
@@ -101,23 +106,35 @@ def set_default_admin_url(cls, admin_url: Optional[str] = None) -> None:
             admin_url = cls._admin_url_default
         cls._admin_url = admin_url
 
-    @classmethod
-    def get_libhxl_dataset(cls, admin_url: str = _admin_url) -> hxl.Dataset:
+    def get_libhxl_dataset(self, url: str = _admin_url) -> hxl.Dataset:
         """
         Get libhxl Dataset object given a URL which defaults to global p-codes
         dataset on HDX.
 
         Args:
-            admin_url (str): URL from which to load data. Defaults to global p-codes dataset.
+            admin_url (str): URL from which to load data.
 
         Returns:
             None
         """
+        if self.retriever:
+            try:
+                url_to_use = self.retriever.download_file(url)
+            except DownloadError:
+                logger.exception(
+                    f"Setup of libhxl Dataset object with {url} failed!"
+                )
+                raise
+        else:
+            url_to_use = url
         try:
-            return hxl.data(admin_url, InputOptions(encoding="utf-8"))
+            return hxl.data(
+                url_to_use,
+                InputOptions(InputOptions(allow_local=True, encoding="utf-8")),
+            )
         except HXLIOException:
             logger.exception(
-                f"Setup of libhxl Dataset object with {admin_url} failed!"
+                f"Setup of libhxl Dataset object with {url} failed!"
             )
             raise
 

tests/hdx/location/test_adminlevel.py

@@ -3,28 +3,34 @@
 from os.path import join
 
 import pytest
-from hxl.input import HXLIOException
 
 from hdx.location.adminlevel import AdminLevel
+from hdx.utilities.downloader import Download
 from hdx.utilities.loader import load_yaml
+from hdx.utilities.path import temp_dir
+from hdx.utilities.retriever import Retrieve
 
 
 class TestAdminLevel:
+    @pytest.fixture(scope="class")
+    def fixtures_dir(self):
+        return join("tests", "fixtures")
+
     @pytest.fixture(scope="function")
-    def config(self):
-        return load_yaml(join("tests", "fixtures", "adminlevel.yaml"))
+    def config(self, fixtures_dir):
+        return load_yaml(join(fixtures_dir, "adminlevel.yaml"))
 
     @pytest.fixture(scope="function")
-    def config_parent(self):
-        return load_yaml(join("tests", "fixtures", "adminlevelparent.yaml"))
+    def config_parent(self, fixtures_dir):
+        return load_yaml(join(fixtures_dir, "adminlevelparent.yaml"))
 
     @pytest.fixture(scope="function")
-    def url(self):
-        return "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-country/blank_adm_name/tests/fixtures/global_pcodes_adm_1_2.csv"
+    def url(self, fixtures_dir):
+        return join(fixtures_dir, "download-global-pcodes-adm-1-2.csv")
 
     @pytest.fixture(scope="function")
-    def formats_url(self):
-        return "https://raw.githubusercontent.com/OCHA-DAP/hdx-python-country/blank_adm_name/tests/fixtures/global_pcode_lengths.csv"
+    def formats_url(self, fixtures_dir):
+        return join(fixtures_dir, "download-global-pcode-lengths.csv")
 
     def test_adminlevel(self, config):
         adminone = AdminLevel(config)
@@ -367,16 +373,35 @@ def test_adminlevel_parent(self, config_parent):
             "MWI", "Blantyre city", parent="MW3", logname="test"
         ) == (None, False)
 
-    def test_adminlevel_with_url(self, config, url):
+    def test_adminlevel_with_url(self, config, url, fixtures_dir):
         adminone = AdminLevel(config)
-        with pytest.raises(HXLIOException):
+        with pytest.raises(FileNotFoundError):
             adminone.setup_from_url("fake_url")
         AdminLevel.set_default_admin_url()
         assert AdminLevel._admin_url == AdminLevel._admin_url_default
         AdminLevel.set_default_admin_url(url)
         assert AdminLevel._admin_url == url
         adminone.setup_from_url(countryiso3s=("YEM",))
         assert len(adminone.get_pcode_list()) == 22
+
+        with temp_dir(
+            "TestAdminLevelRetriever",
+            delete_on_success=True,
+            delete_on_failure=False,
+        ) as tempdir:
+            with Download(user_agent="test") as downloader:
+                retriever = Retrieve(
+                    downloader,
+                    tempdir,
+                    fixtures_dir,
+                    tempdir,
+                    save=False,
+                    use_saved=True,
+                )
+                adminone = AdminLevel(config, retriever=retriever)
+                adminone.setup_from_url(countryiso3s=("YEM",))
+                assert len(adminone.get_pcode_list()) == 22
+
         adminone = AdminLevel(config)
         adminone.setup_from_url()
         assert adminone.get_admin_level("YEM") == 1