From 4f9abd9c5a8235dd98f164006884848651362292 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Mon, 10 Jun 2024 12:41:18 -0500 Subject: [PATCH 01/16] Formatting, type annotations - autopep8 - Added type annotations where the language server wasn't inferring them - Turned the check for api_filename into a guard clause --- tools/RAiDER/models/credentials.py | 143 ++++++++++++++++------------- 1 file changed, 79 insertions(+), 64 deletions(-) diff --git a/tools/RAiDER/models/credentials.py b/tools/RAiDER/models/credentials.py index 85f8eb2d7..9433cfed3 100644 --- a/tools/RAiDER/models/credentials.py +++ b/tools/RAiDER/models/credentials.py @@ -13,44 +13,51 @@ import os from pathlib import Path from platform import system +from typing import Dict, Optional, Tuple # Filename for the hidden file per model -API_FILENAME = {'ERA5' : 'cdsapirc', - 'ERA5T' : 'cdsapirc', - 'HRES' : 'ecmwfapirc', - 'GMAO' : 'netrc', - 'HRRR' : None - } +API_FILENAME: Dict[str, Optional[str]] = { + 'ERA5': 'cdsapirc', + 'ERA5T': 'cdsapirc', + 'HRES': 'ecmwfapirc', + 'GMAO': 'netrc', + 'HRRR': None +} # API urls -API_URLS = {'cdsapirc' : 'https://cds.climate.copernicus.eu/api/v2', - 'ecmwfapirc' : 'https://api.ecmwf.int/v1', - 'netrc' : 'urs.earthdata.nasa.gov'} +API_URLS = { + 'cdsapirc': 'https://cds.climate.copernicus.eu/api/v2', + 'ecmwfapirc': 'https://api.ecmwf.int/v1', + 'netrc': 'urs.earthdata.nasa.gov' +} # api credentials dict API_CREDENTIALS_DICT = { - 'cdsapirc' : {'api' : """\ + 'cdsapirc': { + 'api': """\ \nurl: {host}\ \nkey: {uid}:{key} - """, - 'help_url' : 'https://cds.climate.copernicus.eu/api-how-to' - }, - 'ecmwfapirc' : {'api' : """{{\ + """, + 'help_url': 'https://cds.climate.copernicus.eu/api-how-to' + }, + 'ecmwfapirc': { + 'api': """{{\ \n"url" : "{host}",\ \n"key" : "{key}",\ \n"email" : "{uid}"\ \n}} - """, - 'help_url' : 'https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key' - }, - 'netrc' : {'api' : """\ + """, + 'help_url': 'https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key' + }, + 'netrc': { + 'api': """\ \nmachine {host}\ \n login {uid}\ \n password {key}\ - """, - 'help_url': 'https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+cURL+And+Wget' - } - } + """, + 'help_url': 'https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+cURL+And+Wget' + } +} # system environmental variables for API credentials ''' @@ -62,7 +69,9 @@ ''' # Check if API enviroments exists -def _check_envs(model): + + +def _check_envs(model: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: if model in ('ERA5', 'ERA5T'): uid = os.getenv('RAIDER_ECMWF_ERA5_UID') key = os.getenv('RAIDER_ECMWF_ERA5_API_KEY') @@ -76,20 +85,21 @@ def _check_envs(model): host = API_URLS['ecmwfapirc'] elif model in ('GMAO',): - uid = os.getenv('EARTHDATA_USERNAME') # same as in DockerizedTopsApp + uid = os.getenv('EARTHDATA_USERNAME') # same as in DockerizedTopsApp key = os.getenv('EARTHDATA_PASSWORD') host = API_URLS['netrc'] - else: # for HRRR + else: # for HRRR uid, key, host = None, None, None return uid, key, host + # Check and write MODEL API_RC_FILE for downloading weather model data def check_api(model: str, UID: str = None, KEY: str = None, - output_dir : str = '~/', + output_dir: str = '~/', update_flag: bool = False) -> None: # Weather model API filename @@ -103,47 +113,52 @@ def check_api(model: str, URL = API_URLS[api_filename] # Get hidden ext for Windows - hidden_ext = '_' if system()=="Windows" else '.' + hidden_ext = '_' if system() == "Windows" else '.' # skip below if model is HRRR as it does not need API - if api_filename: - # Check if the credential api file exists - api_filename_path = Path(output_dir) / (hidden_ext + api_filename) - api_filename_path = api_filename_path.expanduser() - - # if update flag is on, overwrite existing file - if update_flag is True: - api_filename_path.unlink(missing_ok=True) - - # Check if API_RC file already exists - if api_filename_path.exists(): - return None - - # if it does not exist, put UID/KEY inserted, create it - elif not api_filename_path.exists() and UID and KEY: - # Create file with inputs, do it only once - print(f'Writing {api_filename_path} locally!') - api_filename_path.write_text(API_CREDENTIALS_DICT[api_filename]['api'].format(uid=UID, - key=KEY, - host=URL)) - api_filename_path.chmod(0o000600) - # Raise ERROR message + if api_filename is None: + return + + # Check if the credential api file exists + api_filename_path = Path(output_dir) / (hidden_ext + api_filename) + api_filename_path = api_filename_path.expanduser() + + # if update flag is on, overwrite existing file + if update_flag is True: + api_filename_path.unlink(missing_ok=True) + + # Check if API_RC file already exists + if api_filename_path.exists(): + return + # if it does not exist, put UID/KEY inserted, create it + elif not api_filename_path.exists() and UID and KEY: + # Create file with inputs, do it only once + print(f'Writing {api_filename_path} locally!') + api_filename_path.write_text( + API_CREDENTIALS_DICT[api_filename]['api'].format( + uid=UID, + key=KEY, + host=URL + ) + ) + api_filename_path.chmod(0o000600) + # Raise ERROR message + else: + help_url = API_CREDENTIALS_DICT[api_filename]['help_url'] + + # Raise ERROR in case only UID or KEY is inserted + if UID is not None and KEY is None: + raise ValueError(f'ERROR: API UID not inserted' + f' or does not exist in ENVIRONMENTALS!') + elif UID is None and KEY is not None: + raise ValueError(f'ERROR: API KEY not inserted' + f' or does not exist in ENVIRONMENTALS!') else: - help_url = API_CREDENTIALS_DICT[api_filename]['help_url'] - - # Raise ERROR in case only UID or KEY is inserted - if UID is not None and KEY is None: - raise ValueError(f'ERROR: API UID not inserted' - f' or does not exist in ENVIRONMENTALS!') - elif UID is None and KEY is not None: - raise ValueError(f'ERROR: API KEY not inserted' - f' or does not exist in ENVIRONMENTALS!') - else: - #Raise ERROR is both UID/KEY are none - raise ValueError( - f'{api_filename_path}, API ENVIRONMENTALS' - f' and API UID and KEY, do not exist !!' - f'\nGet API info from ' + '\033[1m' f'{help_url}' + '\033[0m, and add it!') + # Raise ERROR is both UID/KEY are none + raise ValueError( + f'{api_filename_path}, API ENVIRONMENTALS' + f' and API UID and KEY, do not exist !!' + f'\nGet API info from ' + '\033[1m' f'{help_url}' + '\033[0m, and add it!') def setup_from_env(): From 124a00700a58badf9e7e0ed6595472fee80b4195 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:16:51 -0500 Subject: [PATCH 02/16] autopep8 --- test/test_credentials.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/test/test_credentials.py b/test/test_credentials.py index caf0c7a60..95c2dcdb5 100644 --- a/test/test_credentials.py +++ b/test/test_credentials.py @@ -3,23 +3,26 @@ from RAiDER.models import credentials # Test checking/creating ECMWF_RC CAPI file + + def test_ecmwfApi_createFile(): import ecmwfapi - #Check extension for hidden files - hidden_ext = '_' if system()=="Windows" else '.' + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' # Test creation of ~/.ecmwfapirc file - ecmwf_file = os.path.expanduser('./') + hidden_ext + credentials.API_FILENAME['HRES'] + ecmwf_file = os.path.expanduser( + './') + hidden_ext + credentials.API_FILENAME['HRES'] credentials.check_api('HRES', 'dummy', 'dummy', './', update_flag=True) - assert os.path.exists(ecmwf_file) == True,f'{ecmwf_file} does not exist' + assert os.path.exists(ecmwf_file) == True, f'{ecmwf_file} does not exist' # Get existing ECMWF_API_RC env if exist default_ecmwf_file = os.getenv("ECMWF_API_RC_FILE") if default_ecmwf_file is None: default_ecmwf_file = ecmwfapi.api.DEFAULT_RCFILE_PATH - #Set it to current dir to avoid overwriting ~/.ecmwfapirc file + # Set it to current dir to avoid overwriting ~/.ecmwfapirc file os.environ["ECMWF_API_RC_FILE"] = ecmwf_file key, url, uid = ecmwfapi.api.get_apikey_values() @@ -27,8 +30,8 @@ def test_ecmwfApi_createFile(): os.environ["ECMWF_API_RC_FILE"] = default_ecmwf_file os.remove(ecmwf_file) - #Check if API is written correctly - assert uid == 'dummy', f'{ecmwf_file}: UID was not written correctly' + # Check if API is written correctly + assert uid == 'dummy', f'{ecmwf_file}: UID was not written correctly' assert key == 'dummy', f'{ecmwf_file}: KEY was not written correctly' @@ -37,13 +40,14 @@ def test_ecmwfApi_createFile(): def test_cdsApi_createFile(): import cdsapi - #Check extension for hidden files - hidden_ext = '_' if system()=="Windows" else '.' + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' # Test creation of .cdsapirc file in current dir - cds_file = os.path.expanduser('./') + hidden_ext + credentials.API_FILENAME['ERA5'] + cds_file = os.path.expanduser( + './') + hidden_ext + credentials.API_FILENAME['ERA5'] credentials.check_api('ERA5', 'dummy', 'dummy', './', update_flag=True) - assert os.path.exists(cds_file) == True,f'{cds_file} does not exist' + assert os.path.exists(cds_file), f'{cds_file} does not exist' # Check the content cds_credentials = cdsapi.api.read_config(cds_file) @@ -52,20 +56,21 @@ def test_cdsApi_createFile(): # Remove local API file os.remove(cds_file) - assert uid == 'dummy', f'{cds_file}: UID was not written correctly' + assert uid == 'dummy', f'{cds_file}: UID was not written correctly' assert key == 'dummy', f'{cds_file}: KEY was not written correctly' # Test checking/creating EARTHDATA_RC API file def test_netrcApi_createFile(): import netrc - #Check extension for hidden files - hidden_ext = '_' if system()=="Windows" else '.' + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' # Test creation of ~/.cdsapirc file - netrc_file = os.path.expanduser('./') + hidden_ext + credentials.API_FILENAME['GMAO'] + netrc_file = os.path.expanduser( + './') + hidden_ext + credentials.API_FILENAME['GMAO'] credentials.check_api('GMAO', 'dummy', 'dummy', './', update_flag=True) - assert os.path.exists(netrc_file) == True,f'{netrc_file} does not exist' + assert os.path.exists(netrc_file) == True, f'{netrc_file} does not exist' # Check the content host = 'urs.earthdata.nasa.gov' @@ -75,5 +80,5 @@ def test_netrcApi_createFile(): # Remove local API file os.remove(netrc_file) - assert uid == 'dummy', f'{netrc_file}: UID was not written correctly' + assert uid == 'dummy', f'{netrc_file}: UID was not written correctly' assert key == 'dummy', f'{netrc_file}: KEY was not written correctly' From 7ec6592ccf542dde5edc6c15d1017d0f02ff51c8 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:56:55 -0500 Subject: [PATCH 03/16] Refactor - Use a match statement in _check_envs where appropriate - Refer to API credential files as rc files - Switch non-const variable names to lowercase --- tools/RAiDER/models/credentials.py | 133 ++++++++++++++--------------- 1 file changed, 64 insertions(+), 69 deletions(-) diff --git a/tools/RAiDER/models/credentials.py b/tools/RAiDER/models/credentials.py index 9433cfed3..c6ef11174 100644 --- a/tools/RAiDER/models/credentials.py +++ b/tools/RAiDER/models/credentials.py @@ -15,8 +15,8 @@ from platform import system from typing import Dict, Optional, Tuple -# Filename for the hidden file per model -API_FILENAME: Dict[str, Optional[str]] = { +# Filename for the rc file for each model +RC_FILENAMES: Dict[str, Optional[str]] = { 'ERA5': 'cdsapirc', 'ERA5T': 'cdsapirc', 'HRES': 'ecmwfapirc', @@ -24,14 +24,12 @@ 'HRRR': None } -# API urls API_URLS = { 'cdsapirc': 'https://cds.climate.copernicus.eu/api/v2', 'ecmwfapirc': 'https://api.ecmwf.int/v1', 'netrc': 'urs.earthdata.nasa.gov' } -# api credentials dict API_CREDENTIALS_DICT = { 'cdsapirc': { 'api': """\ @@ -68,99 +66,96 @@ ''' -# Check if API enviroments exists - - +# Check if the user has the environment variables for a given weather model API def _check_envs(model: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: - if model in ('ERA5', 'ERA5T'): - uid = os.getenv('RAIDER_ECMWF_ERA5_UID') - key = os.getenv('RAIDER_ECMWF_ERA5_API_KEY') - host = API_URLS['cdsapirc'] - - elif model in ('HRES',): - uid = os.getenv('RAIDER_HRES_EMAIL') - key = os.getenv('RAIDER_HRES_API_KEY') - host = os.getenv('RAIDER_HRES_URL') - if host is None: - host = API_URLS['ecmwfapirc'] - - elif model in ('GMAO',): - uid = os.getenv('EARTHDATA_USERNAME') # same as in DockerizedTopsApp - key = os.getenv('EARTHDATA_PASSWORD') - host = API_URLS['netrc'] - - else: # for HRRR - uid, key, host = None, None, None + match model: + case 'ERA5' | 'ERA5T': + uid = os.getenv('RAIDER_ECMWF_ERA5_UID') + key = os.getenv('RAIDER_ECMWF_ERA5_API_KEY') + host = API_URLS['cdsapirc'] + case 'HRES': + uid = os.getenv('RAIDER_HRES_EMAIL') + key = os.getenv('RAIDER_HRES_API_KEY') + host = os.getenv('RAIDER_HRES_URL') + if host is None: + host = API_URLS['ecmwfapirc'] + case 'GMAO': + # same as in DockerizedTopsApp + uid = os.getenv('EARTHDATA_USERNAME') + key = os.getenv('EARTHDATA_PASSWORD') + host = API_URLS['netrc'] + case _: # for HRRR + uid, key, host = None, None, None return uid, key, host -# Check and write MODEL API_RC_FILE for downloading weather model data +# Check if the user has the rc file necessary for a given weather model API def check_api(model: str, - UID: str = None, - KEY: str = None, + uid: Optional[str] = None, + key: Optional[str] = None, output_dir: str = '~/', - update_flag: bool = False) -> None: + update_rc_file: bool = False) -> None: + + # Weather model API RC filename + # Typically stored in home dir as a hidden file + rc_filename = RC_FILENAMES[model] - # Weather model API filename - # typically stored in home dir as hidden file - api_filename = API_FILENAME[model] + # If the given API does not require an rc file, return (nothing to do) + if rc_filename is None: + return - # Get API credential from os.env if UID/KEY are not inserted - if UID is None and KEY is None: - UID, KEY, URL = _check_envs(model) + # Get credentials from env vars if uid/key is not passed in + if uid is None and key is None: + uid, key, url = _check_envs(model) else: - URL = API_URLS[api_filename] + url = API_URLS[rc_filename] # Get hidden ext for Windows hidden_ext = '_' if system() == "Windows" else '.' - # skip below if model is HRRR as it does not need API - if api_filename is None: - return - - # Check if the credential api file exists - api_filename_path = Path(output_dir) / (hidden_ext + api_filename) - api_filename_path = api_filename_path.expanduser() + # Get the target rc file's path + rc_path = Path(output_dir) / (hidden_ext + rc_filename) + rc_path = rc_path.expanduser() # if update flag is on, overwrite existing file - if update_flag is True: - api_filename_path.unlink(missing_ok=True) + if update_rc_file: + rc_path.unlink(missing_ok=True) # Check if API_RC file already exists - if api_filename_path.exists(): + if rc_path.exists(): return - # if it does not exist, put UID/KEY inserted, create it - elif not api_filename_path.exists() and UID and KEY: + # if it does not exist, put uid/key inserted, create it + elif not rc_path.exists() and uid and key: # Create file with inputs, do it only once - print(f'Writing {api_filename_path} locally!') - api_filename_path.write_text( - API_CREDENTIALS_DICT[api_filename]['api'].format( - uid=UID, - key=KEY, - host=URL + print(f'Writing {rc_path} locally!') + rc_path.write_text( + API_CREDENTIALS_DICT[rc_filename]['api'].format( + uid=uid, + key=key, + host=url ) ) - api_filename_path.chmod(0o000600) + rc_path.chmod(0o000600) # Raise ERROR message else: - help_url = API_CREDENTIALS_DICT[api_filename]['help_url'] - - # Raise ERROR in case only UID or KEY is inserted - if UID is not None and KEY is None: - raise ValueError(f'ERROR: API UID not inserted' - f' or does not exist in ENVIRONMENTALS!') - elif UID is None and KEY is not None: - raise ValueError(f'ERROR: API KEY not inserted' - f' or does not exist in ENVIRONMENTALS!') + help_url = API_CREDENTIALS_DICT[rc_filename]['help_url'] + + # Raise ERROR in case only username or password is present + if uid is None and key is not None: + raise ValueError('ERROR: API uid not inserted' + ' or does not exist in ENVIRONMENTALS!') + elif uid is not None and key is None: + raise ValueError('ERROR: API key not inserted' + ' or does not exist in ENVIRONMENTALS!') else: - # Raise ERROR is both UID/KEY are none + # Raise ERROR if both UID/KEY are none raise ValueError( - f'{api_filename_path}, API ENVIRONMENTALS' - f' and API UID and KEY, do not exist !!' + f'{rc_path}, API ENVIRONMENTALS' + ' and API UID and KEY, do not exist !!' f'\nGet API info from ' + '\033[1m' f'{help_url}' + '\033[0m, and add it!') def setup_from_env(): - for model in API_FILENAME.keys(): + for model in RC_FILENAMES.keys(): check_api(model) From 7c3d209809766470ec4292a4a737d73f2657271b Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:04:15 -0500 Subject: [PATCH 04/16] Add entry for #652 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d2856f6..83d8073a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Changed +* Changed the behavior of `RAiDER.models.credentials.check_api` to not overwrite + the user's API credential files + ## [0.5.1] ### Changed * Use hyp3-lib v3* to download orbits to be able to distribute load across ESA and ASF. Can be easily swapped out for `sentineleof` in future release. From a259c7a19ee112895c939f6f3b41d41995ebb808 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:07:40 -0500 Subject: [PATCH 05/16] Add link to #652 --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83d8073a0..64043bd67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,9 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Changed -* Changed the behavior of `RAiDER.models.credentials.check_api` to not overwrite - the user's API credential files +* [652](https://github.com/dbekaert/RAiDER/pull/652) Changed the behavior of + `RAiDER.models.credentials.check_api` to not overwrite the user's API + credential files. ## [0.5.1] ### Changed From 05beba58d97b4479c0eeae7ad4a3d6d4fabe8aff Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Tue, 11 Jun 2024 19:48:55 -0500 Subject: [PATCH 06/16] Update .netrc instead of overwriting Behavior of this function changed so that it will no longer overwrite .netrc. .netrc may contain more than one set of credentials, so we have to take care not to destroy them while updating the one relevant to RAiDER/GMAO. The more simple RC files that only ever hold one set of credentials will still just be overwritten, but now RAiDER will know when it's dealing with .netrc/GMAO and parse it and only replace the part that pertains to GMAO's host URL. --- tools/RAiDER/models/credentials.py | 160 +++++++++++++++-------------- 1 file changed, 82 insertions(+), 78 deletions(-) diff --git a/tools/RAiDER/models/credentials.py b/tools/RAiDER/models/credentials.py index c6ef11174..5f7b1bb5e 100644 --- a/tools/RAiDER/models/credentials.py +++ b/tools/RAiDER/models/credentials.py @@ -7,14 +7,18 @@ cdsapirc ERA5, ERA5T uid key https://cds.climate.copernicus.eu/api/v2 ecmwfapirc HRES email key https://api.ecmwf.int/v1 netrc GMAO, MERRA2 username password urs.earthdata.nasa.gov - HRRR [public access] + HRRR [public access] ''' import os +import re from pathlib import Path from platform import system from typing import Dict, Optional, Tuple +from tools.RAiDER import logger + + # Filename for the rc file for each model RC_FILENAMES: Dict[str, Optional[str]] = { 'ERA5': 'cdsapirc', @@ -24,136 +28,136 @@ 'HRRR': None } -API_URLS = { - 'cdsapirc': 'https://cds.climate.copernicus.eu/api/v2', - 'ecmwfapirc': 'https://api.ecmwf.int/v1', - 'netrc': 'urs.earthdata.nasa.gov' -} - -API_CREDENTIALS_DICT = { +APIS = { 'cdsapirc': { 'api': """\ - \nurl: {host}\ - \nkey: {uid}:{key} +url: {host}\n\ +key: {uid}:{key}\n\ """, - 'help_url': 'https://cds.climate.copernicus.eu/api-how-to' + 'help_url': 'https://cds.climate.copernicus.eu/api-how-to', + 'default_host': 'https://cds.climate.copernicus.eu/api/v2' }, 'ecmwfapirc': { - 'api': """{{\ - \n"url" : "{host}",\ - \n"key" : "{key}",\ - \n"email" : "{uid}"\ - \n}} + 'api': """\ +{\n\ + "url" : "{host}",\n\ + "key" : "{key}",\n\ + "email" : "{uid}"\n\ +}\n\ """, - 'help_url': 'https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key' + 'help_url': 'https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key', + 'default_host': 'https://api.ecmwf.int/v1' }, 'netrc': { 'api': """\ - \nmachine {host}\ - \n login {uid}\ - \n password {key}\ +machine {host}\n\ + login {uid}\n\ + password {key}\n\ """, - 'help_url': 'https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+cURL+And+Wget' + 'help_url': 'https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+cURL+And+Wget', + 'default_host': 'urs.earthdata.nasa.gov' } } -# system environmental variables for API credentials -''' -ENV variables in cdsapi and ecmwapir - -cdsapi ['cdsapirc'] : CDSAPI_KEY [UID:KEY], 'CDSAPI_URL' -ecmwfapir [ecmwfapirc] : 'ECMWF_API_KEY', 'ECMWF_API_EMAIL','ECMWF_API_URL' -''' - -# Check if the user has the environment variables for a given weather model API -def _check_envs(model: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: +# Get the environment variables for a given weather model API +def _get_envs(model: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: match model: case 'ERA5' | 'ERA5T': uid = os.getenv('RAIDER_ECMWF_ERA5_UID') key = os.getenv('RAIDER_ECMWF_ERA5_API_KEY') - host = API_URLS['cdsapirc'] + host = APIS['cdsapirc']['default_host'] case 'HRES': uid = os.getenv('RAIDER_HRES_EMAIL') key = os.getenv('RAIDER_HRES_API_KEY') - host = os.getenv('RAIDER_HRES_URL') - if host is None: - host = API_URLS['ecmwfapirc'] + host = os.getenv('RAIDER_HRES_URL', + APIS['ecmwfapirc']['default_host']) case 'GMAO': # same as in DockerizedTopsApp uid = os.getenv('EARTHDATA_USERNAME') key = os.getenv('EARTHDATA_PASSWORD') - host = API_URLS['netrc'] + host = APIS['netrc']['default_host'] case _: # for HRRR uid, key, host = None, None, None - return uid, key, host -# Check if the user has the rc file necessary for a given weather model API def check_api(model: str, uid: Optional[str] = None, key: Optional[str] = None, output_dir: str = '~/', update_rc_file: bool = False) -> None: - # Weather model API RC filename # Typically stored in home dir as a hidden file rc_filename = RC_FILENAMES[model] - # If the given API does not require an rc file, return (nothing to do) + # If the given API does not require an rc file, then there is nothing to do + # (e.g., HRRR) if rc_filename is None: return - # Get credentials from env vars if uid/key is not passed in - if uid is None and key is None: - uid, key, url = _check_envs(model) - else: - url = API_URLS[rc_filename] - - # Get hidden ext for Windows - hidden_ext = '_' if system() == "Windows" else '.' - # Get the target rc file's path + hidden_ext = '_' if system() == "Windows" else '.' rc_path = Path(output_dir) / (hidden_ext + rc_filename) rc_path = rc_path.expanduser() - # if update flag is on, overwrite existing file - if update_rc_file: - rc_path.unlink(missing_ok=True) - - # Check if API_RC file already exists - if rc_path.exists(): + # If the RC file doesn't exist, then create it. + # But if it does exist, only update it if the user requests. + if rc_path.exists() and not update_rc_file: return - # if it does not exist, put uid/key inserted, create it - elif not rc_path.exists() and uid and key: - # Create file with inputs, do it only once - print(f'Writing {rc_path} locally!') - rc_path.write_text( - API_CREDENTIALS_DICT[rc_filename]['api'].format( - uid=uid, - key=key, - host=url - ) - ) - rc_path.chmod(0o000600) - # Raise ERROR message + + # Get credentials from env vars if uid and key are not passed in + if uid is None and key is None: + uid, key, url = _get_envs(model) else: - help_url = API_CREDENTIALS_DICT[rc_filename]['help_url'] + url = APIS[rc_filename]['default_host'] - # Raise ERROR in case only username or password is present + # Check for invalid inputs + if uid is None or key is None: + help_url = APIS[rc_filename]['help_url'] if uid is None and key is not None: - raise ValueError('ERROR: API uid not inserted' - ' or does not exist in ENVIRONMENTALS!') + raise ValueError( + f'ERROR: {model} API UID not provided in RAiDER arguments and ' + 'not present in environment variables.\n' + f'See info for this model\'s API at \033[1m{help_url}\033[0m' + ) elif uid is not None and key is None: - raise ValueError('ERROR: API key not inserted' - ' or does not exist in ENVIRONMENTALS!') + raise ValueError( + f'ERROR: {model} API key not provided in RAiDER arguments and ' + 'not present in environment variables.\n' + f'See info for this model\'s API at \033[1m{help_url}\033[0m' + ) else: - # Raise ERROR if both UID/KEY are none raise ValueError( - f'{rc_path}, API ENVIRONMENTALS' - ' and API UID and KEY, do not exist !!' - f'\nGet API info from ' + '\033[1m' f'{help_url}' + '\033[0m, and add it!') + f'ERROR: {model} API credentials not provided in RAiDER ' + 'arguments and not present in environment variables.\n' + f'See info for this model\'s API at \033[1m{help_url}\033[0m' + ) + + # Create file with the API credentials + logger.warn(f'{model} API credentials not found in {rc_path}') + entry = APIS[rc_filename]['api'].format(uid=uid, key=key, host=url) + match model: + case 'ERA5' | 'ERA5T' | 'HRES': + # These RC files only ever contain one set of credentials, so + # it can just be overwritten when updating + rc_path.write_text(entry) + case 'GMAO': + # This RC file may contain more than one set of credentials, so + # extra care needs to be taken to make sure we only touch the + # credentials that belong to this URL: + # If an entry for this URL already exists, replace it, otherwise + # add a new one to the end of the file + rc_contents = rc_path.read_text().replace('\r\n', '\n') + if rc_contents[-1] != '\n': + rc_contents += '\n' + rc_contents = re.sub( + f'(machine ({url})\\n\\s*login \\w+\\n\\s*password \\w+\\n?)|$', + entry, + rc_contents + ) + rc_path.write_text(rc_contents) + rc_path.chmod(0o000600) def setup_from_env(): From 8611224bbeb652f2ce156e035087561e1f575644 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Tue, 11 Jun 2024 19:49:15 -0500 Subject: [PATCH 07/16] Add to-do list for new tests --- test/test_credentials.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test_credentials.py b/test/test_credentials.py index 95c2dcdb5..85b1ab1ae 100644 --- a/test/test_credentials.py +++ b/test/test_credentials.py @@ -2,9 +2,7 @@ from platform import system from RAiDER.models import credentials -# Test checking/creating ECMWF_RC CAPI file - - +# Test checking/creating ECMWF_RC API file def test_ecmwfApi_createFile(): import ecmwfapi @@ -35,8 +33,7 @@ def test_ecmwfApi_createFile(): assert key == 'dummy', f'{ecmwf_file}: KEY was not written correctly' -# Test checking/creating Copernicus Climate Data Store -# CDS_RC CAPI file +# Test checking/creating Copernicus Climate Data Store CDS_RC API file def test_cdsApi_createFile(): import cdsapi @@ -82,3 +79,8 @@ def test_netrcApi_createFile(): assert uid == 'dummy', f'{netrc_file}: UID was not written correctly' assert key == 'dummy', f'{netrc_file}: KEY was not written correctly' + +# TODO(garlic-os): Tests for check_api() update=False: +# - Check if the file is not updated if it already exists +# - Check if the file is created if it does not exist +# - Check that environment variables are picked up when present From f18547046fb48d16d53c55ba47f56b6420059c87 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:08:05 -0500 Subject: [PATCH 08/16] Add tests for credentials.py - test_credentials.py moved to credentials/test_createFile.py - credentials/test_updateFalse.py: verifies behavior of credentials.check_api while update flag is False - (STAR OF THE SHOW) credentials/test_updateTrue.py: verifies correct behavior while update flag is True, including that .netrc credentials are NOT CLOBBERED --- test/__init__.py | 11 ++- test/credentials/test_createFile.py | 99 +++++++++++++++++++++++ test/credentials/test_updateFalse.py | 67 ++++++++++++++++ test/credentials/test_updateTrue.py | 116 +++++++++++++++++++++++++++ test/test_credentials.py | 86 -------------------- 5 files changed, 292 insertions(+), 87 deletions(-) create mode 100644 test/credentials/test_createFile.py create mode 100644 test/credentials/test_updateFalse.py create mode 100644 test/credentials/test_updateTrue.py delete mode 100644 test/test_credentials.py diff --git a/test/__init__.py b/test/__init__.py index 0faf8b712..69d49ca1a 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -2,6 +2,8 @@ import pytest import subprocess import shutil +import string +import random from contextlib import contextmanager from pathlib import Path @@ -77,4 +79,11 @@ def makeLatLonGrid(bbox, reg, out_dir, spacing=0.1): def make_delay_name(weather_model_name, date, time, kind='ztd'): assert kind in 'ztd std ray'.split(), 'Incorrect type of delays.' - return f'{weather_model_name}_tropo_{date}T{time.replace(":", "")}_{kind}.nc' \ No newline at end of file + return f'{weather_model_name}_tropo_{date}T{time.replace(":", "")}_{kind}.nc' + + +def random_string( + length: int = 32, + alphabet: str = string.ascii_letters + string.digits +) -> str: + return ''.join(random.choices(alphabet, k=length)) diff --git a/test/credentials/test_createFile.py b/test/credentials/test_createFile.py new file mode 100644 index 000000000..88fa83e90 --- /dev/null +++ b/test/credentials/test_createFile.py @@ -0,0 +1,99 @@ +''' +When update_rc_file is either True or False, the relevant API RC file should be +created if it doesn't exist. +''' +import os +from pathlib import Path +from platform import system +from RAiDER.models import credentials +from test import random_string + + +def test_cds(): + import cdsapi + + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['ERA5']) + rc_path = rc_path.expanduser() + rc_path.unlink(missing_ok=True) + + test_uid = random_string() + test_key = random_string() + + # Test creation of .cdsapirc file in current dir + credentials.check_api('ERA5', test_uid, test_key, './', update_rc_file=False) + assert rc_path.exists(), f'{rc_path} was not created' + + # Check the content + cds_credentials = cdsapi.api.read_config(rc_path) + uid, key = cds_credentials['key'].split(':') + + # Remove local API file + rc_path.unlink() + + assert uid == test_uid, f'{rc_path}: UID was not written correctly' + assert key == test_key, f'{rc_path}: KEY was not written correctly' + + +def test_ecmwf(): + import ecmwfapi + + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['HRES']) + rc_path = rc_path.expanduser() + rc_path.unlink(missing_ok=True) + + test_uid = random_string() + test_key = random_string() + + # Test creation of .ecmwfapirc file + credentials.check_api('HRES', test_uid, test_key, './', update_rc_file=False) + assert rc_path.exists(), f'{rc_path} does not exist' + + # Get current ECMWF API RC file path + old_rc_path = os.getenv("ECMWF_API_RC_FILE", ecmwfapi.api.DEFAULT_RCFILE_PATH) + + # Point ecmwfapi to current dir to avoid overwriting ~/.ecmwfapirc + os.environ["ECMWF_API_RC_FILE"] = str(rc_path) + key, _, uid = ecmwfapi.api.get_apikey_values() + + # Point ecmwfapi back to previous value and remove local API file + os.environ["ECMWF_API_RC_FILE"] = old_rc_path + rc_path.unlink() + + # Check if API is written correctly + assert uid == test_uid, f'{rc_path}: UID was not written correctly' + assert key == test_key, f'{rc_path}: KEY was not written correctly' + + +def test_netrc(): + import netrc + + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['GMAO']) + rc_path = rc_path.expanduser() + rc_path.unlink(missing_ok=True) + + test_uid = random_string() + test_key = random_string() + + # Test creation of .netrc file + credentials.check_api('GMAO', test_uid, test_key, './', update_rc_file=False) + assert os.path.exists(rc_path), f'{rc_path} does not exist' + + # Check the content + host = 'urs.earthdata.nasa.gov' + netrc_credentials = netrc.netrc(rc_path) + uid, _, key = netrc_credentials.authenticators(host) + + # Remove local API file + rc_path.unlink() + + assert uid == test_uid, f'{rc_path}: UID was not written correctly' + assert key == test_key, f'{rc_path}: KEY was not written correctly' diff --git a/test/credentials/test_updateFalse.py b/test/credentials/test_updateFalse.py new file mode 100644 index 000000000..ad9d3cad5 --- /dev/null +++ b/test/credentials/test_updateFalse.py @@ -0,0 +1,67 @@ +''' +When update_rc_file is False, the RC file should NOT be modified if it already +exists. +''' +from pathlib import Path +from platform import system +from RAiDER.models import credentials + + +def test_cds(): + # Get extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + # Get the target rc file's path + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['ERA5']) + rc_path = rc_path.expanduser() + + # Write some example text to test for + rc_path.write_text('dummy') + + # Test creation of .cdsapirc file in current dir + credentials.check_api('ERA5', None, None, './', update_rc_file=False) + + # Assert the content was unchanged + content = rc_path.read_text() + rc_path.unlink() + assert content == 'dummy', f'{rc_path} was modified' + + +def test_ecmwf(): + # Get extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + # Get the target rc file's path + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['HRES']) + rc_path = rc_path.expanduser() + + # Write some example text to test for + rc_path.write_text('dummy') + + # Test creation of .ecmwfapirc file + credentials.check_api('HRES', None, None, './', update_rc_file=False) + + # Assert the content was unchanged + content = rc_path.read_text() + rc_path.unlink() + assert content == 'dummy', f'{rc_path} was modified' + + +def test_netrc(): + # Get extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + # Get the target rc file's path + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['GMAO']) + rc_path = rc_path.expanduser() + + # Write some example text to test for + rc_path.write_text('dummy') + + # Test creation of .netrc file + credentials.check_api('GMAO', None, None, './', update_rc_file=False) + + # Assert the content was unchanged + content = rc_path.read_text() + rc_path.unlink() + assert content == 'dummy', f'{rc_path} was modified' diff --git a/test/credentials/test_updateTrue.py b/test/credentials/test_updateTrue.py new file mode 100644 index 000000000..26159c713 --- /dev/null +++ b/test/credentials/test_updateTrue.py @@ -0,0 +1,116 @@ +''' +When update_rc_file is True, the RC file should be: +- updated if it already exists, +- created if it doesn't, +- and for .netrc files, it should ONLY update the set of credentials related to + the given weather model's API URL. +''' + +from pathlib import Path +from platform import system +from RAiDER.models import credentials +from test import random_string + + +def test_cds(): + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['ERA5']) + rc_path = rc_path.expanduser() + + template = ( + 'url: https://cds.climate.copernicus.eu/api/v2\n' + 'key: {uid}:{key}\n' + ) + + test_uid = random_string() + test_key = random_string() + credentials.check_api('ERA5', test_uid, test_key, './', update_rc_file=True) + + expected_content = template.format(uid=test_uid, key=test_key) + actual_content = rc_path.read_text() + rc_path.unlink() + + assert ( + expected_content == actual_content, + f'{rc_path} was not updated correctly' + ) + + +def test_ecmwf(): + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['HRES']) + rc_path = rc_path.expanduser() + + template = ( + '{{\n' + ' "url" : "https://api.ecmwf.int/v1",\n' + ' "key" : "{key}",\n' + ' "email" : "{uid}"\n' + '}}\n' + ) + + # Simulate a .ecmwfapirc file + rc_path.write_text(template.format( + uid=random_string(), + key=random_string(), + )) + + test_uid = random_string() + test_key = random_string() + credentials.check_api('HRES', test_uid, test_key, './', update_rc_file=True) + + expected_content = template.format(uid=test_uid, key=test_key) + actual_content = rc_path.read_text() + rc_path.unlink() + + assert ( + expected_content == actual_content, + f'{rc_path} was not updated correctly' + ) + + +def test_netrc(): + # Check extension for hidden files + hidden_ext = '_' if system() == "Windows" else '.' + + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['GMAO']) + rc_path = rc_path.expanduser() + + template = ( + 'machine example.com\n' + ' login johndoe\n' + ' password hunter2\n' + 'machine urs.earthdata.nasa.gov\n' + ' login {uid}\n' + ' password {key}\n' + 'machine 127.0.0.1\n' + ' login bobsmith\n' + ' password dolphins\n' + ) + + # Simulate a .netrc file with multiple sets of credentials. + # The ones not for urs.earthdata.nasa.gov should NOT be touched + rc_path.write_text(template.format( + uid=random_string(), + key=random_string(), + )) + + test_uid = random_string() + test_key = random_string() + credentials.check_api('GMAO', test_uid, test_key, './', update_rc_file=True) + + # Check the content + expected_content = template.format(uid=test_uid, key=test_key) + actual_content = rc_path.read_text() + + # Remove local API file + rc_path.unlink() + + assert ( + expected_content == actual_content, + f'{rc_path} was not updated correctly' + ) diff --git a/test/test_credentials.py b/test/test_credentials.py deleted file mode 100644 index 85b1ab1ae..000000000 --- a/test/test_credentials.py +++ /dev/null @@ -1,86 +0,0 @@ -import os -from platform import system -from RAiDER.models import credentials - -# Test checking/creating ECMWF_RC API file -def test_ecmwfApi_createFile(): - import ecmwfapi - - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - # Test creation of ~/.ecmwfapirc file - ecmwf_file = os.path.expanduser( - './') + hidden_ext + credentials.API_FILENAME['HRES'] - credentials.check_api('HRES', 'dummy', 'dummy', './', update_flag=True) - assert os.path.exists(ecmwf_file) == True, f'{ecmwf_file} does not exist' - - # Get existing ECMWF_API_RC env if exist - default_ecmwf_file = os.getenv("ECMWF_API_RC_FILE") - if default_ecmwf_file is None: - default_ecmwf_file = ecmwfapi.api.DEFAULT_RCFILE_PATH - - # Set it to current dir to avoid overwriting ~/.ecmwfapirc file - os.environ["ECMWF_API_RC_FILE"] = ecmwf_file - key, url, uid = ecmwfapi.api.get_apikey_values() - - # Return to default_ecmwf_file and remove local API file - os.environ["ECMWF_API_RC_FILE"] = default_ecmwf_file - os.remove(ecmwf_file) - - # Check if API is written correctly - assert uid == 'dummy', f'{ecmwf_file}: UID was not written correctly' - assert key == 'dummy', f'{ecmwf_file}: KEY was not written correctly' - - -# Test checking/creating Copernicus Climate Data Store CDS_RC API file -def test_cdsApi_createFile(): - import cdsapi - - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - # Test creation of .cdsapirc file in current dir - cds_file = os.path.expanduser( - './') + hidden_ext + credentials.API_FILENAME['ERA5'] - credentials.check_api('ERA5', 'dummy', 'dummy', './', update_flag=True) - assert os.path.exists(cds_file), f'{cds_file} does not exist' - - # Check the content - cds_credentials = cdsapi.api.read_config(cds_file) - uid, key = cds_credentials['key'].split(':') - - # Remove local API file - os.remove(cds_file) - - assert uid == 'dummy', f'{cds_file}: UID was not written correctly' - assert key == 'dummy', f'{cds_file}: KEY was not written correctly' - -# Test checking/creating EARTHDATA_RC API file -def test_netrcApi_createFile(): - import netrc - - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - # Test creation of ~/.cdsapirc file - netrc_file = os.path.expanduser( - './') + hidden_ext + credentials.API_FILENAME['GMAO'] - credentials.check_api('GMAO', 'dummy', 'dummy', './', update_flag=True) - assert os.path.exists(netrc_file) == True, f'{netrc_file} does not exist' - - # Check the content - host = 'urs.earthdata.nasa.gov' - netrc_credentials = netrc.netrc(netrc_file) - uid, _, key = netrc_credentials.authenticators(host) - - # Remove local API file - os.remove(netrc_file) - - assert uid == 'dummy', f'{netrc_file}: UID was not written correctly' - assert key == 'dummy', f'{netrc_file}: KEY was not written correctly' - -# TODO(garlic-os): Tests for check_api() update=False: -# - Check if the file is not updated if it already exists -# - Check if the file is created if it does not exist -# - Check that environment variables are picked up when present From e2191c2f2680753d97aae4d58df350fdfccf41f3 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:08:44 -0500 Subject: [PATCH 09/16] Fix bugs identified by tests --- tools/RAiDER/models/credentials.py | 77 +++++++++++++++--------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/tools/RAiDER/models/credentials.py b/tools/RAiDER/models/credentials.py index 5f7b1bb5e..ff0f0f1f1 100644 --- a/tools/RAiDER/models/credentials.py +++ b/tools/RAiDER/models/credentials.py @@ -11,12 +11,11 @@ ''' import os -import re from pathlib import Path from platform import system from typing import Dict, Optional, Tuple -from tools.RAiDER import logger +from RAiDER.logger import logger # Filename for the rc file for each model @@ -25,35 +24,36 @@ 'ERA5T': 'cdsapirc', 'HRES': 'ecmwfapirc', 'GMAO': 'netrc', + 'MERRA2': 'netrc', 'HRRR': None } APIS = { 'cdsapirc': { - 'api': """\ -url: {host}\n\ -key: {uid}:{key}\n\ - """, + 'template': ( + 'url: {host}\n' + 'key: {uid}:{key}\n' + ), 'help_url': 'https://cds.climate.copernicus.eu/api-how-to', 'default_host': 'https://cds.climate.copernicus.eu/api/v2' }, 'ecmwfapirc': { - 'api': """\ -{\n\ - "url" : "{host}",\n\ - "key" : "{key}",\n\ - "email" : "{uid}"\n\ -}\n\ - """, + 'template': ( + '{{\n' + ' "url" : "{host}",\n' + ' "key" : "{key}",\n' + ' "email" : "{uid}"\n' + '}}\n' + ), 'help_url': 'https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key', 'default_host': 'https://api.ecmwf.int/v1' }, 'netrc': { - 'api': """\ -machine {host}\n\ - login {uid}\n\ - password {key}\n\ - """, + 'template': ( + 'machine {host}\n' + ' login {uid}\n' + ' password {key}\n' + ), 'help_url': 'https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+cURL+And+Wget', 'default_host': 'urs.earthdata.nasa.gov' } @@ -72,7 +72,7 @@ def _get_envs(model: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: key = os.getenv('RAIDER_HRES_API_KEY') host = os.getenv('RAIDER_HRES_URL', APIS['ecmwfapirc']['default_host']) - case 'GMAO': + case 'GMAO' | 'MERRA2': # same as in DockerizedTopsApp uid = os.getenv('EARTHDATA_USERNAME') key = os.getenv('EARTHDATA_PASSWORD') @@ -135,28 +135,27 @@ def check_api(model: str, ) # Create file with the API credentials - logger.warn(f'{model} API credentials not found in {rc_path}') - entry = APIS[rc_filename]['api'].format(uid=uid, key=key, host=url) - match model: - case 'ERA5' | 'ERA5T' | 'HRES': + if update_rc_file: + logger.info(f'Updating {model} API credentials in {rc_path}') + else: + logger.warning(f'{model} API credentials not found in {rc_path}; creating') + rc_type = RC_FILENAMES[model] + match rc_type: + case 'cdsapirc' | 'ecmwfapirc': # These RC files only ever contain one set of credentials, so - # it can just be overwritten when updating + # they can just be overwritten when updating. + template = APIS[rc_filename]['template'] + entry = template.format(host=url, uid=uid, key=key) rc_path.write_text(entry) - case 'GMAO': - # This RC file may contain more than one set of credentials, so - # extra care needs to be taken to make sure we only touch the - # credentials that belong to this URL: - # If an entry for this URL already exists, replace it, otherwise - # add a new one to the end of the file - rc_contents = rc_path.read_text().replace('\r\n', '\n') - if rc_contents[-1] != '\n': - rc_contents += '\n' - rc_contents = re.sub( - f'(machine ({url})\\n\\s*login \\w+\\n\\s*password \\w+\\n?)|$', - entry, - rc_contents - ) - rc_path.write_text(rc_contents) + case 'netrc': + # This type of RC file may contain more than one set of credentials, + # so extra care needs to be taken to make sure we only touch the + # one that belongs to this URL. + import netrc + rc_path.touch() + netrc_credentials = netrc.netrc(rc_path) + netrc_credentials.hosts[url] = (uid, None, key) + rc_path.write_text(str(netrc_credentials)) rc_path.chmod(0o000600) From 70498fe904d013aa91728e2cf0aafe1b973621ee Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Wed, 26 Jun 2024 15:52:07 -0500 Subject: [PATCH 10/16] Parametrize tests - Reduces code redundancy - Makes it trivial to add new models to the tests, and as such ERA5T and MERRA2 are now tested explicitly along with the others in the update flag tests --- test/credentials/test_createFile.py | 18 ++-- test/credentials/test_updateFalse.py | 51 ++-------- test/credentials/test_updateTrue.py | 147 +++++++++++---------------- 3 files changed, 80 insertions(+), 136 deletions(-) diff --git a/test/credentials/test_createFile.py b/test/credentials/test_createFile.py index 88fa83e90..08d831ba3 100644 --- a/test/credentials/test_createFile.py +++ b/test/credentials/test_createFile.py @@ -12,10 +12,12 @@ def test_cds(): import cdsapi + model_name = 'ERA5' + # Check extension for hidden files hidden_ext = '_' if system() == "Windows" else '.' - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['ERA5']) + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) rc_path = rc_path.expanduser() rc_path.unlink(missing_ok=True) @@ -23,7 +25,7 @@ def test_cds(): test_key = random_string() # Test creation of .cdsapirc file in current dir - credentials.check_api('ERA5', test_uid, test_key, './', update_rc_file=False) + credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) assert rc_path.exists(), f'{rc_path} was not created' # Check the content @@ -40,10 +42,12 @@ def test_cds(): def test_ecmwf(): import ecmwfapi + model_name = 'HRES' + # Check extension for hidden files hidden_ext = '_' if system() == "Windows" else '.' - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['HRES']) + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) rc_path = rc_path.expanduser() rc_path.unlink(missing_ok=True) @@ -51,7 +55,7 @@ def test_ecmwf(): test_key = random_string() # Test creation of .ecmwfapirc file - credentials.check_api('HRES', test_uid, test_key, './', update_rc_file=False) + credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) assert rc_path.exists(), f'{rc_path} does not exist' # Get current ECMWF API RC file path @@ -73,10 +77,12 @@ def test_ecmwf(): def test_netrc(): import netrc + model_name = 'GMAO' + # Check extension for hidden files hidden_ext = '_' if system() == "Windows" else '.' - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['GMAO']) + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) rc_path = rc_path.expanduser() rc_path.unlink(missing_ok=True) @@ -84,7 +90,7 @@ def test_netrc(): test_key = random_string() # Test creation of .netrc file - credentials.check_api('GMAO', test_uid, test_key, './', update_rc_file=False) + credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) assert os.path.exists(rc_path), f'{rc_path} does not exist' # Check the content diff --git a/test/credentials/test_updateFalse.py b/test/credentials/test_updateFalse.py index ad9d3cad5..29597c4e3 100644 --- a/test/credentials/test_updateFalse.py +++ b/test/credentials/test_updateFalse.py @@ -2,64 +2,27 @@ When update_rc_file is False, the RC file should NOT be modified if it already exists. ''' +import pytest + from pathlib import Path from platform import system from RAiDER.models import credentials -def test_cds(): - # Get extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - # Get the target rc file's path - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['ERA5']) - rc_path = rc_path.expanduser() - - # Write some example text to test for - rc_path.write_text('dummy') - - # Test creation of .cdsapirc file in current dir - credentials.check_api('ERA5', None, None, './', update_rc_file=False) - - # Assert the content was unchanged - content = rc_path.read_text() - rc_path.unlink() - assert content == 'dummy', f'{rc_path} was modified' - - -def test_ecmwf(): - # Get extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - # Get the target rc file's path - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['HRES']) - rc_path = rc_path.expanduser() - - # Write some example text to test for - rc_path.write_text('dummy') - - # Test creation of .ecmwfapirc file - credentials.check_api('HRES', None, None, './', update_rc_file=False) - - # Assert the content was unchanged - content = rc_path.read_text() - rc_path.unlink() - assert content == 'dummy', f'{rc_path} was modified' - - -def test_netrc(): +@pytest.mark.parametrize('model_name', 'ERA5 ERA5T HRES GMAO MERRA2'.split()) +def test_updateFalse(model_name): # Get extension for hidden files hidden_ext = '_' if system() == "Windows" else '.' # Get the target rc file's path - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['GMAO']) + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) rc_path = rc_path.expanduser() # Write some example text to test for rc_path.write_text('dummy') - # Test creation of .netrc file - credentials.check_api('GMAO', None, None, './', update_rc_file=False) + # Test creation of this model's RC file in current dir + credentials.check_api(model_name, None, None, './', update_rc_file=False) # Assert the content was unchanged content = rc_path.read_text() diff --git a/test/credentials/test_updateTrue.py b/test/credentials/test_updateTrue.py index 26159c713..d921e0937 100644 --- a/test/credentials/test_updateTrue.py +++ b/test/credentials/test_updateTrue.py @@ -5,6 +5,7 @@ - and for .netrc files, it should ONLY update the set of credentials related to the given weather model's API URL. ''' +import pytest from pathlib import Path from platform import system @@ -12,56 +13,73 @@ from test import random_string -def test_cds(): +@pytest.mark.parametrize( + 'model_name,template', + [ + ( + 'ERA5', ( + 'url: https://cds.climate.copernicus.eu/api/v2\n' + 'key: {uid}:{key}\n' + ) + ), + ( + 'ERA5T', ( + 'url: https://cds.climate.copernicus.eu/api/v2\n' + 'key: {uid}:{key}\n' + ) + ), + ( + 'HRES', ( + '{{\n' + ' "url" : "https://api.ecmwf.int/v1",\n' + ' "key" : "{key}",\n' + ' "email" : "{uid}"\n' + '}}\n' + ) + ), + ( + # Simulate a .netrc file with multiple sets of credentials. + # The ones not for urs.earthdata.nasa.gov should NOT be touched. + # Indentation is done with TABS, as that is what the netrc package + # generates. + 'GMAO', ( + 'machine example.com\n' + ' login johndoe\n' + ' password hunter2\n' + 'machine urs.earthdata.nasa.gov\n' + ' login {uid}\n' + ' password {key}\n' + 'machine 127.0.0.1\n' + ' login bobsmith\n' + ' password dolphins\n' + ) + ), + ( + 'MERRA2', ( + 'machine example.com\n' + ' login johndoe\n' + ' password hunter2\n' + 'machine urs.earthdata.nasa.gov\n' + ' login {uid}\n' + ' password {key}\n' + 'machine 127.0.0.1\n' + ' login bobsmith\n' + ' password dolphins\n' + ) + ), + ] +) +def test_updateTrue(model_name, template): # Check extension for hidden files hidden_ext = '_' if system() == "Windows" else '.' - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['ERA5']) + # Get the target rc file's path + rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) rc_path = rc_path.expanduser() - template = ( - 'url: https://cds.climate.copernicus.eu/api/v2\n' - 'key: {uid}:{key}\n' - ) - - test_uid = random_string() - test_key = random_string() - credentials.check_api('ERA5', test_uid, test_key, './', update_rc_file=True) - - expected_content = template.format(uid=test_uid, key=test_key) - actual_content = rc_path.read_text() - rc_path.unlink() - - assert ( - expected_content == actual_content, - f'{rc_path} was not updated correctly' - ) - - -def test_ecmwf(): - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['HRES']) - rc_path = rc_path.expanduser() - - template = ( - '{{\n' - ' "url" : "https://api.ecmwf.int/v1",\n' - ' "key" : "{key}",\n' - ' "email" : "{uid}"\n' - '}}\n' - ) - - # Simulate a .ecmwfapirc file - rc_path.write_text(template.format( - uid=random_string(), - key=random_string(), - )) - test_uid = random_string() test_key = random_string() - credentials.check_api('HRES', test_uid, test_key, './', update_rc_file=True) + credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=True) expected_content = template.format(uid=test_uid, key=test_key) actual_content = rc_path.read_text() @@ -71,46 +89,3 @@ def test_ecmwf(): expected_content == actual_content, f'{rc_path} was not updated correctly' ) - - -def test_netrc(): - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES['GMAO']) - rc_path = rc_path.expanduser() - - template = ( - 'machine example.com\n' - ' login johndoe\n' - ' password hunter2\n' - 'machine urs.earthdata.nasa.gov\n' - ' login {uid}\n' - ' password {key}\n' - 'machine 127.0.0.1\n' - ' login bobsmith\n' - ' password dolphins\n' - ) - - # Simulate a .netrc file with multiple sets of credentials. - # The ones not for urs.earthdata.nasa.gov should NOT be touched - rc_path.write_text(template.format( - uid=random_string(), - key=random_string(), - )) - - test_uid = random_string() - test_key = random_string() - credentials.check_api('GMAO', test_uid, test_key, './', update_rc_file=True) - - # Check the content - expected_content = template.format(uid=test_uid, key=test_key) - actual_content = rc_path.read_text() - - # Remove local API file - rc_path.unlink() - - assert ( - expected_content == actual_content, - f'{rc_path} was not updated correctly' - ) From 25c897aabf748087fed017f037b0a3d39c41b24e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 5 Jul 2024 12:15:00 -0700 Subject: [PATCH 11/16] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c5750707..2a455e4f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.5.2] ### Changed * [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`. ### Fixed From 792c6212dad9fd39bcc7b9f64da3ec00e7465c95 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Fri, 5 Jul 2024 21:20:09 -0500 Subject: [PATCH 12/16] Parametrize tests --- test/credentials/test_createFile.py | 98 ++++++++++------------------- 1 file changed, 33 insertions(+), 65 deletions(-) diff --git a/test/credentials/test_createFile.py b/test/credentials/test_createFile.py index 08d831ba3..a20632922 100644 --- a/test/credentials/test_createFile.py +++ b/test/credentials/test_createFile.py @@ -2,6 +2,7 @@ When update_rc_file is either True or False, the relevant API RC file should be created if it doesn't exist. ''' +import pytest import os from pathlib import Path from platform import system @@ -9,55 +10,15 @@ from test import random_string -def test_cds(): +def get_creds_cds(rc_path): import cdsapi - - model_name = 'ERA5' - - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) - rc_path = rc_path.expanduser() - rc_path.unlink(missing_ok=True) - - test_uid = random_string() - test_key = random_string() - - # Test creation of .cdsapirc file in current dir - credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) - assert rc_path.exists(), f'{rc_path} was not created' - - # Check the content cds_credentials = cdsapi.api.read_config(rc_path) uid, key = cds_credentials['key'].split(':') + return uid, key - # Remove local API file - rc_path.unlink() - - assert uid == test_uid, f'{rc_path}: UID was not written correctly' - assert key == test_key, f'{rc_path}: KEY was not written correctly' - -def test_ecmwf(): +def get_creds_ecmwf(rc_path): import ecmwfapi - - model_name = 'HRES' - - # Check extension for hidden files - hidden_ext = '_' if system() == "Windows" else '.' - - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) - rc_path = rc_path.expanduser() - rc_path.unlink(missing_ok=True) - - test_uid = random_string() - test_key = random_string() - - # Test creation of .ecmwfapirc file - credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) - assert rc_path.exists(), f'{rc_path} does not exist' - # Get current ECMWF API RC file path old_rc_path = os.getenv("ECMWF_API_RC_FILE", ecmwfapi.api.DEFAULT_RCFILE_PATH) @@ -67,39 +28,46 @@ def test_ecmwf(): # Point ecmwfapi back to previous value and remove local API file os.environ["ECMWF_API_RC_FILE"] = old_rc_path - rc_path.unlink() - - # Check if API is written correctly - assert uid == test_uid, f'{rc_path}: UID was not written correctly' - assert key == test_key, f'{rc_path}: KEY was not written correctly' + return uid, key -def test_netrc(): +def get_creds_netrc(rc_path): import netrc - - model_name = 'GMAO' - - # Check extension for hidden files + host = 'urs.earthdata.nasa.gov' + netrc_credentials = netrc.netrc(rc_path) + uid, _, key = netrc_credentials.authenticators(host) + return uid, key + + +@pytest.mark.parametrize( + 'model_name,get_creds', + ( + ('ERA5', get_creds_cds), + ('ERA5T', get_creds_cds), + ('HRES', get_creds_ecmwf), + ('GMAO', get_creds_netrc), + ('MERRA2', get_creds_netrc) + ) +) +def test_createFile(model_name, get_creds): + # Get the rc file's path hidden_ext = '_' if system() == "Windows" else '.' - - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) + rc_filename = credentials.RC_FILENAMES[model_name] + if rc_filename is None: + return + rc_path = Path('./') / (hidden_ext + rc_filename) rc_path = rc_path.expanduser() rc_path.unlink(missing_ok=True) test_uid = random_string() test_key = random_string() - # Test creation of .netrc file + # Test creation of the rc file credentials.check_api(model_name, test_uid, test_key, './', update_rc_file=False) - assert os.path.exists(rc_path), f'{rc_path} does not exist' - - # Check the content - host = 'urs.earthdata.nasa.gov' - netrc_credentials = netrc.netrc(rc_path) - uid, _, key = netrc_credentials.authenticators(host) + assert rc_path.exists(), f'{rc_path} does not exist' - # Remove local API file + # Check if API is written correctly + uid, key = get_creds(rc_path) rc_path.unlink() - assert uid == test_uid, f'{rc_path}: UID was not written correctly' - assert key == test_key, f'{rc_path}: KEY was not written correctly' + assert key == test_key, f'{rc_path}: KEY was not written correctly' \ No newline at end of file From ee6ed3f2a7e59c16b6b7799a30aac11a6f6d0c6b Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Fri, 5 Jul 2024 21:20:34 -0500 Subject: [PATCH 13/16] Add None check for RC_FILENAMES --- test/credentials/test_updateFalse.py | 9 +++++---- test/credentials/test_updateTrue.py | 10 +++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/credentials/test_updateFalse.py b/test/credentials/test_updateFalse.py index 29597c4e3..822c58ff9 100644 --- a/test/credentials/test_updateFalse.py +++ b/test/credentials/test_updateFalse.py @@ -11,11 +11,12 @@ @pytest.mark.parametrize('model_name', 'ERA5 ERA5T HRES GMAO MERRA2'.split()) def test_updateFalse(model_name): - # Get extension for hidden files + # Get the rc file's path hidden_ext = '_' if system() == "Windows" else '.' - - # Get the target rc file's path - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) + rc_filename = credentials.RC_FILENAMES[model_name] + if rc_filename is None: + return + rc_path = Path('./') / (hidden_ext + rc_filename) rc_path = rc_path.expanduser() # Write some example text to test for diff --git a/test/credentials/test_updateTrue.py b/test/credentials/test_updateTrue.py index d921e0937..9ed65fbc2 100644 --- a/test/credentials/test_updateTrue.py +++ b/test/credentials/test_updateTrue.py @@ -70,12 +70,12 @@ ] ) def test_updateTrue(model_name, template): - # Check extension for hidden files + # Get the rc file's path hidden_ext = '_' if system() == "Windows" else '.' - - # Get the target rc file's path - rc_path = Path('./') / (hidden_ext + credentials.RC_FILENAMES[model_name]) - rc_path = rc_path.expanduser() + rc_filename = credentials.RC_FILENAMES[model_name] + if rc_filename is None: + return + rc_path = Path('./') / (hidden_ext + rc_filename) test_uid = random_string() test_key = random_string() From 98f5b940159b28d59ef5b6ba2163380dab0ba98c Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Fri, 5 Jul 2024 21:20:41 -0500 Subject: [PATCH 14/16] Create test_envVars.py --- test/credentials/test_envVars.py | 108 +++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 test/credentials/test_envVars.py diff --git a/test/credentials/test_envVars.py b/test/credentials/test_envVars.py new file mode 100644 index 000000000..9c8bcd071 --- /dev/null +++ b/test/credentials/test_envVars.py @@ -0,0 +1,108 @@ +''' +Environment variables specific to each model are accepted iff uid and key +arguments are None. +''' + +import pytest +from pathlib import Path +from platform import system +from RAiDER.models import credentials +from test import random_string + +@pytest.mark.parametrize( + 'model_name,template,env_var_name_uid,env_var_name_key', + [ + ( + 'ERA5', ( + 'url: https://cds.climate.copernicus.eu/api/v2\n' + 'key: {uid}:{key}\n' + ), + 'RAIDER_ECMWF_ERA5_UID', + 'RAIDER_ECMWF_ERA5_API_KEY' + ), + ( + 'ERA5T', ( + 'url: https://cds.climate.copernicus.eu/api/v2\n' + 'key: {uid}:{key}\n' + ), + 'RAIDER_ECMWF_ERA5_UID', + 'RAIDER_ECMWF_ERA5_API_KEY' + ), + ( + 'HRES', ( + '{{\n' + ' "url" : "https://api.ecmwf.int/v1",\n' + ' "key" : "{key}",\n' + ' "email" : "{uid}"\n' + '}}\n' + ), + 'RAIDER_HRES_EMAIL', + 'RAIDER_HRES_API_KEY' + ), + ( + # Simulate a .netrc file with multiple sets of credentials. + # The ones not for urs.earthdata.nasa.gov should NOT be touched. + # Indentation is done with TABS, as that is what the netrc package + # generates. + 'GMAO', ( + 'machine example.com\n' + ' login johndoe\n' + ' password hunter2\n' + 'machine urs.earthdata.nasa.gov\n' + ' login {uid}\n' + ' password {key}\n' + 'machine 127.0.0.1\n' + ' login bobsmith\n' + ' password dolphins\n' + ), + 'EARTHDATA_USERNAME', + 'EARTHDATA_PASSWORD' + ), + ( + 'MERRA2', ( + 'machine example.com\n' + ' login johndoe\n' + ' password hunter2\n' + 'machine urs.earthdata.nasa.gov\n' + ' login {uid}\n' + ' password {key}\n' + 'machine 127.0.0.1\n' + ' login bobsmith\n' + ' password dolphins\n' + ), + 'EARTHDATA_USERNAME', + 'EARTHDATA_PASSWORD' + ), + ] +) +def test_envVars( + monkeypatch, + model_name, + template, + env_var_name_uid, + env_var_name_key +): + hidden_ext = '_' if system() == "Windows" else '.' + rc_filename = credentials.RC_FILENAMES[model_name] + if rc_filename is None: + return + rc_path = Path('./') / (hidden_ext + rc_filename) + rc_path = rc_path.expanduser() + rc_path.unlink(missing_ok=True) + + test_uid = random_string() + test_key = random_string() + + with monkeypatch.context() as mp: + mp.setenv(env_var_name_uid, test_uid) + mp.setenv(env_var_name_key, test_key) + credentials.check_api(model_name, None, None, './', update_rc_file=False) + + expected_content = template.format(uid=test_uid, key=test_key) + actual_content = rc_path.read_text() + rc_path.unlink() + + assert ( + expected_content == actual_content, + f'{rc_path} was not updated correctly' + ) From e0bf8dde43ca548184fb281ac9227a3d4e8e8c25 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Fri, 5 Jul 2024 21:35:33 -0500 Subject: [PATCH 15/16] Refactor match back to if-else chain for 3.9 --- tools/RAiDER/models/credentials.py | 64 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/tools/RAiDER/models/credentials.py b/tools/RAiDER/models/credentials.py index ff0f0f1f1..70900bcaa 100644 --- a/tools/RAiDER/models/credentials.py +++ b/tools/RAiDER/models/credentials.py @@ -62,23 +62,22 @@ # Get the environment variables for a given weather model API def _get_envs(model: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: - match model: - case 'ERA5' | 'ERA5T': - uid = os.getenv('RAIDER_ECMWF_ERA5_UID') - key = os.getenv('RAIDER_ECMWF_ERA5_API_KEY') - host = APIS['cdsapirc']['default_host'] - case 'HRES': - uid = os.getenv('RAIDER_HRES_EMAIL') - key = os.getenv('RAIDER_HRES_API_KEY') - host = os.getenv('RAIDER_HRES_URL', - APIS['ecmwfapirc']['default_host']) - case 'GMAO' | 'MERRA2': - # same as in DockerizedTopsApp - uid = os.getenv('EARTHDATA_USERNAME') - key = os.getenv('EARTHDATA_PASSWORD') - host = APIS['netrc']['default_host'] - case _: # for HRRR - uid, key, host = None, None, None + if model in ('ERA5', 'ERA5T'): + uid = os.getenv('RAIDER_ECMWF_ERA5_UID') + key = os.getenv('RAIDER_ECMWF_ERA5_API_KEY') + host = APIS['cdsapirc']['default_host'] + elif model == 'HRES': + uid = os.getenv('RAIDER_HRES_EMAIL') + key = os.getenv('RAIDER_HRES_API_KEY') + host = os.getenv('RAIDER_HRES_URL', + APIS['ecmwfapirc']['default_host']) + elif model in ('GMAO', 'MERRA2'): + # same as in DockerizedTopsApp + uid = os.getenv('EARTHDATA_USERNAME') + key = os.getenv('EARTHDATA_PASSWORD') + host = APIS['netrc']['default_host'] + else: # for HRRR + uid, key, host = None, None, None return uid, key, host @@ -140,22 +139,21 @@ def check_api(model: str, else: logger.warning(f'{model} API credentials not found in {rc_path}; creating') rc_type = RC_FILENAMES[model] - match rc_type: - case 'cdsapirc' | 'ecmwfapirc': - # These RC files only ever contain one set of credentials, so - # they can just be overwritten when updating. - template = APIS[rc_filename]['template'] - entry = template.format(host=url, uid=uid, key=key) - rc_path.write_text(entry) - case 'netrc': - # This type of RC file may contain more than one set of credentials, - # so extra care needs to be taken to make sure we only touch the - # one that belongs to this URL. - import netrc - rc_path.touch() - netrc_credentials = netrc.netrc(rc_path) - netrc_credentials.hosts[url] = (uid, None, key) - rc_path.write_text(str(netrc_credentials)) + if rc_type in ('cdsapirc', 'ecmwfapirc'): + # These RC files only ever contain one set of credentials, so + # they can just be overwritten when updating. + template = APIS[rc_filename]['template'] + entry = template.format(host=url, uid=uid, key=key) + rc_path.write_text(entry) + elif rc_type == 'netrc': + # This type of RC file may contain more than one set of credentials, + # so extra care needs to be taken to make sure we only touch the + # one that belongs to this URL. + import netrc + rc_path.touch() + netrc_credentials = netrc.netrc(rc_path) + netrc_credentials.hosts[url] = (uid, None, key) + rc_path.write_text(str(netrc_credentials)) rc_path.chmod(0o000600) From 542494e5c96820c6282151750949bc574c569270 Mon Sep 17 00:00:00 2001 From: Nate Kean <14845347+garlic-os@users.noreply.github.com> Date: Fri, 5 Jul 2024 21:40:13 -0500 Subject: [PATCH 16/16] Add type annotations --- test/credentials/test_createFile.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/credentials/test_createFile.py b/test/credentials/test_createFile.py index a20632922..57f539e20 100644 --- a/test/credentials/test_createFile.py +++ b/test/credentials/test_createFile.py @@ -2,6 +2,8 @@ When update_rc_file is either True or False, the relevant API RC file should be created if it doesn't exist. ''' +from typing import Tuple + import pytest import os from pathlib import Path @@ -10,14 +12,14 @@ from test import random_string -def get_creds_cds(rc_path): +def get_creds_cds(rc_path: Path) -> Tuple[str, str]: import cdsapi cds_credentials = cdsapi.api.read_config(rc_path) uid, key = cds_credentials['key'].split(':') return uid, key -def get_creds_ecmwf(rc_path): +def get_creds_ecmwf(rc_path: Path) -> Tuple[str, str]: import ecmwfapi # Get current ECMWF API RC file path old_rc_path = os.getenv("ECMWF_API_RC_FILE", ecmwfapi.api.DEFAULT_RCFILE_PATH) @@ -31,7 +33,7 @@ def get_creds_ecmwf(rc_path): return uid, key -def get_creds_netrc(rc_path): +def get_creds_netrc(rc_path: Path) -> Tuple[str, str]: import netrc host = 'urs.earthdata.nasa.gov' netrc_credentials = netrc.netrc(rc_path)