diff --git a/argo-egi-connectors.spec b/argo-egi-connectors.spec index 324e011d..e34f18e3 100644 --- a/argo-egi-connectors.spec +++ b/argo-egi-connectors.spec @@ -1,5 +1,5 @@ Name: argo-egi-connectors -Version: 1.7.1 +Version: 1.7.2 Release: 1%{?dist} Group: EGI/SA4 License: ASL 2.0 @@ -11,7 +11,8 @@ Obsoletes: ar-sync Prefix: %{_prefix} Requires: argo-ams-library Requires: avro -Requires: pyOpenSSL +Requires: python-requests +Requires: python2-ndg_httpsclient Source0: %{name}-%{version}.tar.gz BuildArch: noarch @@ -47,6 +48,8 @@ rm -rf $RPM_BUILD_ROOT %attr(0755,root,root) %dir %{_localstatedir}/log/argo-connectors/ %changelog +* Wed Feb 20 2019 Daniel Vrcic - 1.7.2-1%{?dist} +- ARGO-1674 Use requests library in connectors * Fri Nov 30 2018 Daniel Vrcic , Katarina Zailac - 1.7.1-1%{?dist} - ARGO-1428 ServiceGroup topology filtering - ARGO-1370 Optimize connectors queries to POEM diff --git a/modules/input.py b/modules/input.py index 1a3b56dc..7fe678e8 100644 --- a/modules/input.py +++ b/modules/input.py @@ -1,17 +1,11 @@ import base64 -import httplib import json -import os +import requests import socket import xml.dom.minidom from argo_egi_connectors.helpers import retry -from OpenSSL.SSL import TLSv1_METHOD, Context, Connection -from OpenSSL.SSL import VERIFY_PEER -from OpenSSL.SSL import WantReadError as SSLWantReadError -from ssl import SSLError -from time import sleep from xml.parsers.expat import ExpatError from urlparse import urlparse @@ -25,49 +19,45 @@ def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None): try: buf = None - if scheme.startswith('https'): - if eval(globopts['AuthenticationVerifyServerCert'.lower()]): - verify_cert(host, int(globopts['ConnectionTimeout'.lower()]), - globopts['AuthenticationCAPath'.lower()], - globopts['AuthenticationCAFile'.lower()]) - conn = httplib.HTTPSConnection(host, 443, - globopts['AuthenticationHostKey'.lower()], - globopts['AuthenticationHostCert'.lower()], - timeout=int(globopts['ConnectionTimeout'.lower()])) - else: - conn = httplib.HTTPConnection(host, 80, - timeout=int(globopts['ConnectionTimeout'.lower()])) - headers = {} if custauth and eval(custauth['AuthenticationUsePlainHttpAuth'.lower()]): userpass = base64.b64encode(custauth['AuthenticationHttpUser'.lower()] + ':' \ + custauth['AuthenticationHttpPass'.lower()]) headers={'Authorization': 'Basic ' + userpass} - conn.request('GET', url, headers=headers) - resp = conn.getresponse() + if scheme.startswith('https'): + response = requests.get('https://'+ host + url, headers=headers, + cert=(globopts['AuthenticationHostCert'.lower()], + globopts['AuthenticationHostKey'.lower()]), + verify=eval(globopts['AuthenticationVerifyServerCert'.lower()]), + timeout=int(globopts['ConnectionTimeout'.lower()])) + response.raise_for_status() + else: + response = requests.get('http://'+ host + url, headers=headers, + timeout=int(globopts['ConnectionTimeout'.lower()])) - if resp.status >= 300 and resp.status < 400: - headers = resp.getheaders() + + if response.status_code >= 300 and response.status_code < 400: + headers = response.headers location = filter(lambda h: 'location' in h[0], headers) if location: redir = urlparse(location[0][1]) else: - raise httplib.HTTPException('No Location header set for redirect') + raise requests.exceptions.RequestException('No Location header set for redirect') return connection(logger, msgprefix, globopts, scheme, redir.netloc, redir.path + '?' + redir.query, custauth=custauth) - elif resp.status == 200: - buf = resp.read() + elif response.status_code == 200: + buf = response.content if not buf: - raise httplib.HTTPException('Empty response') + raise requests.exceptions.RequestException('Empty response') else: - raise httplib.HTTPException('Response: %s %s' % (resp.status, resp.reason)) + raise requests.exceptions.RequestException('response: %s %s' % (response.status_code, response.reason)) return buf - except SSLError as e: + except requests.exceptions.SSLError as e: if (getattr(e, 'args', False) and type(e.args) == tuple and type(e.args[0]) == str and 'timed out' in e.args[0]): @@ -86,7 +76,7 @@ def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None): repr(e))) raise e - except httplib.HTTPException as e: + except requests.exceptions.RequestException as e: logger.warn('%sCustomer:%s Job:%s HTTP error %s - %s' % (msgprefix + ' ' if msgprefix else '', logger.customer, logger.job, scheme + '://' + host + url, @@ -100,9 +90,6 @@ def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None): repr(e))) return False - finally: - conn.close() - def parse_xml(logger, objname, globopts, buf, method): try: @@ -134,64 +121,3 @@ def parse_json(logger, objname, globopts, buf, method): else: return doc - - -def verify_cert(host, timeout, capath, cafile): - def verify_cert(host, ca, timeout): - server_ctx = Context(TLSv1_METHOD) - server_cert_chain = [] - - if os.path.isdir(ca): - server_ctx.load_verify_locations(None, ca) - else: - server_ctx.load_verify_locations(ca, None) - - def verify_cb(conn, cert, errnum, depth, ok): - server_cert_chain.append(cert) - return ok - server_ctx.set_verify(VERIFY_PEER, verify_cb) - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setblocking(1) - sock.settimeout(timeout) - sock.connect((host, 443)) - - server_conn = Connection(server_ctx, sock) - server_conn.set_connect_state() - - def iosock_try(): - ok = True - try: - server_conn.do_handshake() - sleep(0.5) - except SSLWantReadError as e: - ok = False - pass - except Exception as e: - raise e - return ok - - try: - while True: - if iosock_try(): - break - - server_subject = server_cert_chain[-1].get_subject() - if host != server_subject.CN: - raise SSLError('Server certificate CN does not match %s' % host) - - except SSLError as e: - raise e - finally: - server_conn.shutdown() - server_conn.close() - - return True - - try: - verify_cert(host, capath, timeout) - except SSLError: - verify_cert(host, cafile, timeout) - - - return True