diff --git a/Changes.txt b/Changes.txt index dd80a0f..29a5f6f 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,6 +1,10 @@ -unreleased +v2.3.1 Wed Nov 15 2023 + New error 'SSLError' which is more explicit in case of SSL certificate chain issues + Allow setting a domain name (only used in test suite) + Allow setting sslcontext, for example to ignore SSL certificat errors (for debugging only) Batch example: Guess if input is coordinate pair, if so then do reverse geocoding Batch example: Give example of input file format + Batch example: Ship CA root certificates instead of relying on those of the operating system v2.3.0 Tue 04 Jul 2023 Batch example: Raise exception when API key fails (quota, missing API key) diff --git a/examples/batch.py b/examples/batch.py index b4a6375..65a1117 100755 --- a/examples/batch.py +++ b/examples/batch.py @@ -32,12 +32,21 @@ import sys import csv import re +import ssl import asyncio import traceback +import aiohttp import backoff +import certifi from tqdm import tqdm -from opencage.geocoder import OpenCageGeocode, AioHttpError +from opencage.geocoder import OpenCageGeocode, SSLError +# Use certificates from the certifi package instead of those of the operating system +# https://pypi.org/project/certifi/ +# https://docs.aiohttp.org/en/stable/client_advanced.html#ssl-control-for-tcp-sockets +sslcontext = ssl.create_default_context(cafile=certifi.where()) +# Alternatively set sslcontext=False to ignore certificate validation (not advised) +# or sslcontext=None to use those of the operating system @@ -49,11 +58,11 @@ FORWARD_OR_REVERSE = 'guess' # 'forward' (address -> coordinates) or 'reverse' (coordinates -> address) # With 'guess' the script checks if the address is two numbers and then # assumes reverse - +API_DOMAIN = 'api.opencagedata.com' MAX_ITEMS = 100 # How many lines to read from the input file. Set to 0 for unlimited NUM_WORKERS = 3 # For 10 requests per second try 2-5 REQUEST_TIMEOUT_SECONDS = 5 # For individual HTTP requests. Default is 1 -RETRY_MAX_TRIES = 10 # How often to retry if a HTTP request times out +RETRY_MAX_TRIES = 10 # How often to retry if a HTTP request times out RETRY_MAX_TIME = 60 # Limit in seconds for retries SHOW_PROGRESS = True # Show progress bar @@ -133,7 +142,7 @@ def backoff_hdlr(details): max_tries=RETRY_MAX_TRIES, on_backoff=backoff_hdlr) async def geocode_one_address(address, address_id): - async with OpenCageGeocode(API_KEY) as geocoder: + async with OpenCageGeocode(API_KEY, domain=API_DOMAIN, sslcontext=sslcontext) as geocoder: global FORWARD_OR_REVERSE try: if FORWARD_OR_REVERSE == 'reverse' or \ @@ -150,6 +159,9 @@ async def geocode_one_address(address, address_id): # countrycode, language, etc # see the full list: https://opencagedata.com/api#forward-opt geocoding_results = await geocoder.geocode_async(address, no_annotations=1) + except SSLError as exc: + sys.stderr.write(str(exc)) + except Exception as exc: traceback.print_exception(exc, file=sys.stderr) diff --git a/opencage/__init__.py b/opencage/__init__.py index 5c06498..078e440 100644 --- a/opencage/__init__.py +++ b/opencage/__init__.py @@ -2,4 +2,4 @@ __author__ = "OpenCage GmbH" __email__ = 'support@opencagedata.com' -__version__ = '2.3.0' +__version__ = '2.3.1' diff --git a/opencage/geocoder.py b/opencage/geocoder.py index 33d9dc6..5913f2d 100644 --- a/opencage/geocoder.py +++ b/opencage/geocoder.py @@ -14,6 +14,8 @@ except ImportError: AIOHTTP_AVAILABLE = False +DEFAULT_DOMAIN = 'api.opencagedata.com' + def backoff_max_time(): return int(os.environ.get('BACKOFF_MAX_TIME', '120')) @@ -102,6 +104,21 @@ class AioHttpError(OpenCageGeocodeError): """ +class SSLError(OpenCageGeocodeError): + + """ + Exception raised when SSL connection to OpenCage server fails. + """ + + def __unicode__(self): + """Convert exception to a string.""" + return ("SSL Certificate error connecting to OpenCage API. This is usually due to " + "outdated CA root certificates of the operating system. " + ) + + __str__ = __unicode__ + + class OpenCageGeocode: """ @@ -121,15 +138,18 @@ class OpenCageGeocode: """ - url = 'https://api.opencagedata.com/geocode/v1/json' - key = '' session = None - def __init__(self, key, protocol='https'): + def __init__(self, key, protocol='https', domain=DEFAULT_DOMAIN, sslcontext=None): """Constructor.""" self.key = key - if protocol and protocol == 'http': - self.url = self.url.replace('https://', 'http://') + + if protocol and protocol not in ('http', 'https'): + protocol = 'https' + self.url = protocol + '://' + domain + '/geocode/v1/json' + + # https://docs.aiohttp.org/en/stable/client_advanced.html#ssl-control-for-tcp-sockets + self.sslcontext = sslcontext def __enter__(self): self.session = requests.Session() @@ -167,7 +187,7 @@ def geocode(self, query, **kwargs): """ if self.session and isinstance(self.session, aiohttp.client.ClientSession): - raise AioHttpError("Cannot use `geocode` in an async context, use `gecode_async`.") + raise AioHttpError("Cannot use `geocode` in an async context, use `geocode_async`.") request = self._parse_request(query, kwargs) response = self._opencage_request(request) @@ -271,34 +291,38 @@ def _opencage_request(self, params): return response_json async def _opencage_async_request(self, params): - async with self.session.get(self.url, params=params) as response: - try: - response_json = await response.json() - except ValueError as excinfo: - raise UnknownError("Non-JSON result from server") from excinfo - - if response.status == 401: - raise NotAuthorizedError() - - if response.status == 403: - raise ForbiddenError() - - if response.status in (402, 429): - # Rate limit exceeded - print(response_json) - reset_time = datetime.utcfromtimestamp(response_json['rate']['reset']) - raise RateLimitExceededError( - reset_to=int(response_json['rate']['limit']), - reset_time=reset_time - ) - - if response.status == 500: - raise UnknownError("500 status code from API") - - if 'results' not in response_json: - raise UnknownError("JSON from API doesn't have a 'results' key") - - return response_json + try: + async with self.session.get(self.url, params=params, ssl=self.sslcontext) as response: + try: + response_json = await response.json() + except ValueError as excinfo: + raise UnknownError("Non-JSON result from server") from excinfo + + if response.status == 401: + raise NotAuthorizedError() + + if response.status == 403: + raise ForbiddenError() + + if response.status in (402, 429): + # Rate limit exceeded + reset_time = datetime.utcfromtimestamp(response_json['rate']['reset']) + raise RateLimitExceededError( + reset_to=int(response_json['rate']['limit']), + reset_time=reset_time + ) + + if response.status == 500: + raise UnknownError("500 status code from API") + + if 'results' not in response_json: + raise UnknownError("JSON from API doesn't have a 'results' key") + + return response_json + except aiohttp.ClientSSLError as exp: + raise SSLError() from exp + except aiohttp.client_exceptions.ClientConnectorCertificateError as exp: + raise SSLError() from exp def _parse_request(self, query, params): if not isinstance(query, str): diff --git a/setup.py b/setup.py index 789a381..cb7e4fa 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name="opencage", - version="2.3.0", + version="2.3.1", description="Wrapper module for the OpenCage Geocoder API", long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', diff --git a/test/fixtures/badssl-com-chain.pem b/test/fixtures/badssl-com-chain.pem new file mode 100644 index 0000000..8be3165 --- /dev/null +++ b/test/fixtures/badssl-com-chain.pem @@ -0,0 +1,90 @@ +-----BEGIN CERTIFICATE----- +MIIE8DCCA9igAwIBAgISA4mqZntfCH8MYyIVqUF1XZgpMA0GCSqGSIb3DQEBCwUA +MDIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQD +EwJSMzAeFw0yMzEwMTkxNTUwMjlaFw0yNDAxMTcxNTUwMjhaMBcxFTATBgNVBAMM +DCouYmFkc3NsLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAONj +dsqxZsR+pDzWX6GLCy6ImoAT60LNYvs9U6BIQ+fatIWbMELAFD6jY+IP25hrVEr1 +bgwRWmAAOnUc2qKXdtx6KXXO3cAJoCSHFNBDEZqzg/+exj+3emQH8dVZiYAS2Rpd +nL9uKc3xgDDb74p1m7J4JdMewHmebRUmMt0MbA0f8sxvhbv9wIXkgAZd6dKYPGzJ +KJlCoQifPiJ66JwYk8WVGEJH9m8LNDse388MscfsuwvAAh9tt2Fq6rmV9s21P6qf +JgjePl65e8fVjsEWBAvC/aMYvTUs7Gdqej0qByjESpt1LZClNomJDvIgqA9+5KsU +yCXigT6OiPjtZhdhgw0CAwEAAaOCAhkwggIVMA4GA1UdDwEB/wQEAwIFoDAdBgNV +HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E +FgQUqryJ2HM8bXniU+mPXLakkgUQobMwHwYDVR0jBBgwFoAUFC6zF7dYVsuuUAlA +5h+vnYsUwsYwVQYIKwYBBQUHAQEESTBHMCEGCCsGAQUFBzABhhVodHRwOi8vcjMu +by5sZW5jci5vcmcwIgYIKwYBBQUHMAKGFmh0dHA6Ly9yMy5pLmxlbmNyLm9yZy8w +IwYDVR0RBBwwGoIMKi5iYWRzc2wuY29tggpiYWRzc2wuY29tMBMGA1UdIAQMMAow +CAYGZ4EMAQIBMIIBAwYKKwYBBAHWeQIEAgSB9ASB8QDvAHUA2ra/az+1tiKfm8K7 +XGvocJFxbLtRhIU0vaQ9MEjX+6sAAAGLSNh8nwAABAMARjBEAiBkJnQowOqs+tDj +7qXXu0PlDCvgvtEemuw1OvInlaHSrAIgcCZV5dJmGVrS1voinEpAzScJejhGB0vb +G8dfKhJZD+wAdgA7U3d1Pi25gE6LMFsG/kA7Z9hPw/THvQANLXJv4frUFwAAAYtI +2HyZAAAEAwBHMEUCIQCJ+gamX0P/hGiIuu70hn8d0svHSOAMJs3D+eOjMVqsywIg +JXR/lAknUTRU+SyfySDoQ22bDSXfYWZGHLFgAkiRo48wDQYJKoZIhvcNAQELBQAD +ggEBAGE3PDg7p2N8aZyAyO0pGVb/ob9opu12g+diNIdRSjsKIE+TO3uClM2OxT0t +5GBz6Owbe010MQtqBKmX4Zm2LSLUm1kVhPh2ohWmA4hTyN3RG5W0IJ3red6VjrJY +URhZQoXQb0gonxMs+zC+4GQ7+yqzWA1UkrWrURjjJCuljyoWF9sE7qEweomSQWnV +v6bIF599/di1R2l5vcRq1DsQDgKaFY4IpKnvh3RhgO19YxlSS9ERRGBem3Aml9tb +Yac12RmyuxsEAr0v75YeL3pAuq/1Rd5OeKfkm+K06Px3LxwcF92RljXkH6T2U8VM +PEFKedHjYjAag3DUMqSuuGI+ONU= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAw +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAw +WhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3Mg +RW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cP +R5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdx +sxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8Zutm +NHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxg +Z3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG +/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYB +Af8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaA +FHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcw +AoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRw +Oi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQB +gt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6W +PTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wl +ikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQz +CkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BIm +lJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4 +avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2 +yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1O +yK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90Ids +hCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+ +HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6Zv +MldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqX +nLRbwHOoq7hHwg== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFYDCCBEigAwIBAgIQQAF3ITfU6UK47naqPGQKtzANBgkqhkiG9w0BAQsFADA/ +MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT +DkRTVCBSb290IENBIFgzMB4XDTIxMDEyMDE5MTQwM1oXDTI0MDkzMDE4MTQwM1ow +TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh +cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCt6CRz9BQ385ueK1coHIe+3LffOJCMbjzmV6B493XC +ov71am72AE8o295ohmxEk7axY/0UEmu/H9LqMZshftEzPLpI9d1537O4/xLxIZpL +wYqGcWlKZmZsj348cL+tKSIG8+TA5oCu4kuPt5l+lAOf00eXfJlII1PoOK5PCm+D +LtFJV4yAdLbaL9A4jXsDcCEbdfIwPPqPrt3aY6vrFk/CjhFLfs8L6P+1dy70sntK +4EwSJQxwjQMpoOFTJOwT2e4ZvxCzSow/iaNhUd6shweU9GNx7C7ib1uYgeGJXDR5 +bHbvO5BieebbpJovJsXQEOEO3tkQjhb7t/eo98flAgeYjzYIlefiN5YNNnWe+w5y +sR2bvAP5SQXYgd0FtCrWQemsAXaVCg/Y39W9Eh81LygXbNKYwagJZHduRze6zqxZ +Xmidf3LWicUGQSk+WT7dJvUkyRGnWqNMQB9GoZm1pzpRboY7nn1ypxIFeFntPlF4 +FQsDj43QLwWyPntKHEtzBRL8xurgUBN8Q5N0s8p0544fAQjQMNRbcTa0B7rBMDBc +SLeCO5imfWCKoqMpgsy6vYMEG6KDA0Gh1gXxG8K28Kh8hjtGqEgqiNx2mna/H2ql +PRmP6zjzZN7IKw0KKP/32+IVQtQi0Cdd4Xn+GOdwiK1O5tmLOsbdJ1Fu/7xk9TND +TwIDAQABo4IBRjCCAUIwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +SwYIKwYBBQUHAQEEPzA9MDsGCCsGAQUFBzAChi9odHRwOi8vYXBwcy5pZGVudHJ1 +c3QuY29tL3Jvb3RzL2RzdHJvb3RjYXgzLnA3YzAfBgNVHSMEGDAWgBTEp7Gkeyxx ++tvhS5B1/8QVYIWJEDBUBgNVHSAETTBLMAgGBmeBDAECATA/BgsrBgEEAYLfEwEB +ATAwMC4GCCsGAQUFBwIBFiJodHRwOi8vY3BzLnJvb3QteDEubGV0c2VuY3J5cHQu +b3JnMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9jcmwuaWRlbnRydXN0LmNvbS9E +U1RST09UQ0FYM0NSTC5jcmwwHQYDVR0OBBYEFHm0WeZ7tuXkAXOACIjIGlj26Ztu +MA0GCSqGSIb3DQEBCwUAA4IBAQAKcwBslm7/DlLQrt2M51oGrS+o44+/yQoDFVDC +5WxCu2+b9LRPwkSICHXM6webFGJueN7sJ7o5XPWioW5WlHAQU7G75K/QosMrAdSW +9MUgNTP52GE24HGNtLi1qoJFlcDyqSMo59ahy2cI2qBDLKobkx/J3vWraV0T9VuG +WCLKTVXkcGdtwlfFRjlBz4pYg1htmf5X6DYO8A4jqv2Il9DjXA6USbW1FzXSLr9O +he8Y4IWS6wY7bCkjCWDcRQJMEhg76fsO3txE+FiYruq9RUWhiF1myv4Q6W+CyBFC +Dfvp7OOGAN6dEOM4+qR9sdjoSYKEBpsr6GtPAQw4dy753ec5 +-----END CERTIFICATE----- diff --git a/test/test_async.py b/test/test_async.py index 503bea0..00ce1b6 100644 --- a/test/test_async.py +++ b/test/test_async.py @@ -39,4 +39,4 @@ async def test_using_non_async_method(): with pytest.raises(AioHttpError) as excinfo: await geocoder.geocode("Atlantis") - assert str(excinfo.value) == 'Cannot use `geocode` in an async context, use `gecode_async`.' + assert str(excinfo.value) == 'Cannot use `geocode` in an async context, use `geocode_async`.' diff --git a/test/test_error_ssl.py b/test/test_error_ssl.py new file mode 100644 index 0000000..4e9cfb6 --- /dev/null +++ b/test/test_error_ssl.py @@ -0,0 +1,27 @@ +# encoding: utf-8 + +import ssl +import pytest +from opencage.geocoder import OpenCageGeocode, SSLError + +# NOTE: Testing keys https://opencagedata.com/api#testingkeys + +# Connect to a host that has an invalid certificate +@pytest.mark.asyncio +async def test_sslerror(): + bad_domain = 'wrong.host.badssl.com' + async with OpenCageGeocode('6d0e711d72d74daeb2b0bfd2a5cdfdba', domain=bad_domain) as geocoder: + with pytest.raises(SSLError) as excinfo: + await geocoder.geocode_async("something") + assert str(excinfo.value).startswith('SSL Certificate error') + +# Connect to OpenCage API domain but use certificate of another domain +# This tests that sslcontext can be set. +@pytest.mark.asyncio +async def test_sslerror_wrong_certificate(): + sslcontext = ssl.create_default_context(cafile='test/fixtures/badssl-com-chain.pem') + + async with OpenCageGeocode('6d0e711d72d74daeb2b0bfd2a5cdfdba', sslcontext=sslcontext) as geocoder: + with pytest.raises(SSLError) as excinfo: + await geocoder.geocode_async("something") + assert str(excinfo.value).startswith('SSL Certificate error')