Skip to content

Commit

Permalink
Merge pull request #712 from oda-hub/external-resolver-fixed
Browse files Browse the repository at this point in the history 
in case local resolver fails, fallback to the external resolver
  • Loading branch information
@burnout87
burnout87 authored Oct 3, 2024
2 parents eb99daa + 7a39559 commit 2d00c76
Show file tree
Hide file tree
Showing 8 changed files with 269 additions and 43 deletions.
155 changes: 124 additions & 31 deletions cdci_data_analysis/analysis/drupal_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
from enum import Enum, auto
from astropy.coordinates import SkyCoord, Angle
from astropy import units as u
from astroquery.simbad import Simbad
import xml.etree.ElementTree as ET

from cdci_data_analysis.analysis import tokenHelper
from ..analysis.exceptions import RequestNotUnderstood, InternalError, RequestNotAuthorized
Expand Down Expand Up @@ -551,11 +553,14 @@ def post_content_to_gallery(decoded_token,
if update_astro_entity:
auto_update = kwargs.pop('auto_update', 'False') == 'True'
if auto_update is True:
name_resolver_url = disp_conf.name_resolver_url
local_name_resolver_url = disp_conf.local_name_resolver_url
external_name_resolver_url = disp_conf.external_name_resolver_url
entities_portal_url = disp_conf.entities_portal_url
resolved_obj = resolve_name(name_resolver_url=name_resolver_url,
resolved_obj = resolve_name(local_name_resolver_url=local_name_resolver_url,
external_name_resolver_url=external_name_resolver_url,
entities_portal_url=entities_portal_url,
name=src_name)
name=src_name,
sentry_dsn=sentry_dsn)
if resolved_obj is not None:
msg = ''
if 'message' in resolved_obj:
Expand Down Expand Up @@ -1488,39 +1493,127 @@ def check_matching_coords(source_1_name, source_1_coord_ra, source_1_coord_dec,
return False


def resolve_name(name_resolver_url: str, entities_portal_url: str = None, name: str = None):
def resolve_name(local_name_resolver_url: str, external_name_resolver_url: str, entities_portal_url: str = None, name: str = None, sentry_dsn=None):
resolved_obj = {}
if name is not None:
quoted_name = urllib.parse.quote(name.strip())
res = requests.get(name_resolver_url.format(quoted_name))
if res.status_code == 200:
returned_resolved_obj = res.json()
if 'success' in returned_resolved_obj:
local_name_resolver_url_formatted = local_name_resolver_url.format(quoted_name)
try:
res = requests.get(local_name_resolver_url_formatted)
if res.status_code == 200:
returned_resolved_obj = res.json()
if 'success' in returned_resolved_obj:
resolved_obj['name'] = name.replace('_', ' ')
if returned_resolved_obj['success']:
logger.info(f"object {name} successfully resolved")
if 'ra' in returned_resolved_obj:
resolved_obj['RA'] = float(returned_resolved_obj['ra'])
if 'dec' in returned_resolved_obj:
resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
if 'object_ids' in returned_resolved_obj:
resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
if 'object_type' in returned_resolved_obj:
resolved_obj['object_type'] = returned_resolved_obj['object_type']
resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
resolved_obj['message'] = f'{name} successfully resolved'
elif not returned_resolved_obj['success']:
logger.info(f"resolution of the object {name} unsuccessful")
resolved_obj['message'] = f'{name} could not be resolved'
else:
logger.warning("There seems to be some problem in completing the request for the resolution of the object"
f" \"{name}\" using the local resolver.\n"
f"The request lead to the error {res.text}, "
"this might be due to an error in the url or the service "
"requested is currently not available. The external resolver will be used.")
if sentry_dsn is not None:
sentry.capture_message(f'Failed to resolve object "{name}" using the local resolver. '
f'URL: {local_name_resolver_url_formatted} '
f'Status Code: {res.status_code} '
f'Response: {res.text}')
except (ConnectionError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e:
logger.warning(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
f'using the url: {local_name_resolver_url_formatted}. Exception details: {str(e)}')
if sentry_dsn is not None:
sentry.capture_message(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
f'URL: {local_name_resolver_url_formatted} '
f"Exception details: {str(e)}")
external_name_resolver_url_formatted = external_name_resolver_url.format(quoted_name)
try:
res = requests.get(external_name_resolver_url_formatted)
if res.status_code == 200:
root = ET.fromstring(res.text)
resolved_obj['name'] = name.replace('_', ' ')
if returned_resolved_obj['success']:
logger.info(f"object {name} successfully resolved")
if 'ra' in returned_resolved_obj:
resolved_obj['RA'] = float(returned_resolved_obj['ra'])
if 'dec' in returned_resolved_obj:
resolved_obj['DEC'] = float(returned_resolved_obj['dec'])
if 'object_ids' in returned_resolved_obj:
resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
if 'object_type' in returned_resolved_obj:
resolved_obj['object_type'] = returned_resolved_obj['object_type']
resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
resolved_obj['message'] = f'{name} successfully resolved'
elif not returned_resolved_obj['success']:
logger.info(f"resolution of the object {name} unsuccessful")
resolver_tag = root.find('.//Resolver')
if resolver_tag is not None:
ra_tag = resolver_tag.find('.//jradeg')
dec_tag = resolver_tag.find('.//jdedeg')
if ra_tag is None or dec_tag is None:
info_tag = root.find('.//INFO')
resolved_obj['message'] = f'{name} could not be resolved'
if info_tag is not None:
message_info = info_tag.text
resolved_obj['message'] += f': {message_info}'
else:
resolved_obj['RA'] = float(ra_tag.text)
resolved_obj['DEC'] = float(dec_tag.text)
resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)

try:
Simbad.add_votable_fields("otype")
result_table = Simbad.query_object(quoted_name)
object_type = str(result_table[0]['OTYPE']).strip()
resolved_obj['object_type'] = object_type
except Exception as e:
logger.warning(f"An exception occurred while using Simbad to query the object \"{name}\" "
f"while using the external resolver:\n{str(e)}")
resolved_obj['object_type'] = None
try:
object_ids_table = Simbad.query_objectids(name)
source_ids_list = object_ids_table['ID'].tolist()
resolved_obj['object_ids'] = source_ids_list
except Exception as e:
logger.warning(f"An exception occurred while using Simbad to query the object ids for the object \"{name}\" "
f"while using the external resolver:\n{str(e)}")
resolved_obj['object_ids'] = None
else:
warning_msg = ("There seems to be some problem in completing the request for the resolution of the object"
f" \"{name}\" using the external resolver.")
resolved_obj['message'] = f'{name} could not be resolved'
else:
logger.warning(f"there seems to be some problem in completing the request for the resolution of the object: {name}\n"
f"the request lead to the error {res.text}, "
"this might be due to an error in the url or the service "
"requested is currently not available, "
"please check your request and try to issue it again")
raise InternalError('issue when performing a request to the local resolver',
status_code=500,
payload={'drupal_helper_error_message': res.text})
info_tag = root.find('.//INFO')
if info_tag is not None:
warning_msg += (f"The request lead to the error {info_tag.text}, "
"this might be due to an error in the name of the object that ha been provided.")
resolved_obj['message'] += f': {info_tag.text}'
logger.warning(warning_msg)
if sentry_dsn is not None:
sentry.capture_message(f'Failed to resolve object "{name}" using the external resolver. '
f'URL: {external_name_resolver_url_formatted} '
f'Status Code: {res.status_code} '
f'Response: {res.text}'
f"Info returned from the resolver: {resolved_obj['message']}")
else:
logger.warning("There seems to be some problem in completing the request for the resolution of the object"
f" \"{name}\" using the external resolver.\n"
f"The request lead to the error {res.text}, "
"this might be due to an error in the url or the service "
"requested is currently not available. The object could not be resolved.")
if sentry_dsn is not None:
sentry.capture_message(f'Failed to resolve object "{name}" using the external resolver. '
f'URL: {external_name_resolver_url_formatted} '
f'Status Code: {res.status_code} '
f'Response: {res.text}')
resolved_obj['message'] = f'{name} could not be resolved: {res.text}'
except (ConnectionError,
requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e:
logger.warning(f'An exception occurred while trying to resolve the object "{name}" using the local resolver. '
f'using the url: {external_name_resolver_url_formatted}. Exception details: {str(e)}')
if sentry_dsn is not None:
sentry.capture_message(f'An exception occurred while trying to resolve the object "{name}" using the external resolver. '
f'URL: {external_name_resolver_url_formatted} '
f"Exception details: {str(e)}")
return resolved_obj


Expand Down
6 changes: 4 additions & 2 deletions cdci_data_analysis/config_dir/conf_env.yml.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,10 @@ dispatcher:
product_gallery_secret_key: PRODUCT_GALLERY_SECRET_KEY
# timezone used within the drupal configuration, these two values have to be always aligned
product_gallery_timezone: PRODUCT_GALLERY_SECRET_KEY
# url of the name resolver
name_resolver_url: NAME_RESOLVER_URL
# url of the local name resolver
local_name_resolver_url: NAME_RESOLVER_URL
# url of the external name resolver
external_name_resolver_url: NAME_RESOLVER_URL
# url of the online catalog for astrophysical entities
entities_portal_url: ENTITIES_PORTAL_URL
# url for the conversion of a given time, in UTC format, to the correspondent REVNUM
Expand Down
11 changes: 8 additions & 3 deletions cdci_data_analysis/configurer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,10 @@ def __init__(self, cfg_dict, origin=None):
disp_dict.get('product_gallery_options', {}).get('product_gallery_secret_key', None),
disp_dict.get('product_gallery_options', {}).get('product_gallery_timezone',
"Europe/Zurich"),
disp_dict.get('product_gallery_options', {}).get('name_resolver_url', 'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
disp_dict.get('product_gallery_options', {}).get('local_name_resolver_url',
'https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}'),
disp_dict.get('product_gallery_options', {}).get('external_name_resolver_url',
'http://cdsweb.u-strasbg.fr/cgi-bin/nph-sesame/-oxp/NSV?{}'),
disp_dict.get('product_gallery_options', {}).get('entities_portal_url', 'http://cdsportal.u-strasbg.fr/?target={}'),
disp_dict.get('product_gallery_options', {}).get('converttime_revnum_service_url', 'https://www.astro.unige.ch/mmoda/dispatch-data/gw/timesystem/api/v1.0/converttime/UTC/{}/REVNUM'),
disp_dict.get('renku_options', {}).get('renku_gitlab_repository_url', None),
Expand Down Expand Up @@ -338,7 +341,8 @@ def set_conf_dispatcher(self,
product_gallery_url,
product_gallery_secret_key,
product_gallery_timezone,
name_resolver_url,
local_name_resolver_url,
external_name_resolver_url,
entities_portal_url,
converttime_revnum_service_url,
renku_gitlab_repository_url,
Expand Down Expand Up @@ -389,7 +393,8 @@ def set_conf_dispatcher(self,
self.product_gallery_url = product_gallery_url
self.product_gallery_secret_key = product_gallery_secret_key
self.product_gallery_timezone = product_gallery_timezone
self.name_resolver_url = name_resolver_url
self.local_name_resolver_url = local_name_resolver_url
self.external_name_resolver_url = external_name_resolver_url
self.entities_portal_url = entities_portal_url
self.converttime_revnum_service_url = converttime_revnum_service_url
self.renku_gitlab_repository_url = renku_gitlab_repository_url
Expand Down
11 changes: 8 additions & 3 deletions cdci_data_analysis/flask_app/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -650,12 +650,17 @@ def resolve_name():

name = par_dic.get('name', None)

name_resolver_url = app_config.name_resolver_url
local_name_resolver_url = app_config.local_name_resolver_url
external_name_resolver_url = app_config.external_name_resolver_url
entities_portal_url = app_config.entities_portal_url

resolve_object = drupal_helper.resolve_name(name_resolver_url=name_resolver_url,
sentry_dsn = sentry.sentry_url

resolve_object = drupal_helper.resolve_name(local_name_resolver_url=local_name_resolver_url,
external_name_resolver_url=external_name_resolver_url,
entities_portal_url=entities_portal_url,
name=name)
name=name,
sentry_dsn=sentry_dsn)

return resolve_object

Expand Down
Loading

0 comments on commit 2d00c76

Please sign in to comment.