Merge pull request #138 from ARGOeu/devel

Version 1.7.2

argo-egi-connectors.spec

@@ -1,5 +1,5 @@
 Name: argo-egi-connectors
-Version: 1.7.1
+Version: 1.7.2
 Release: 1%{?dist}
 Group: EGI/SA4
 License: ASL 2.0
@@ -11,7 +11,8 @@ Obsoletes: ar-sync
 Prefix: %{_prefix}
 Requires: argo-ams-library
 Requires: avro
-Requires: pyOpenSSL
+Requires: python-requests
+Requires: python2-ndg_httpsclient
 Source0: %{name}-%{version}.tar.gz
 
 BuildArch: noarch
@@ -47,6 +48,8 @@ rm -rf $RPM_BUILD_ROOT
 %attr(0755,root,root) %dir %{_localstatedir}/log/argo-connectors/
 
 %changelog
+* Wed Feb 20 2019 Daniel Vrcic <[email protected]> - 1.7.2-1%{?dist}
+- ARGO-1674 Use requests library in connectors
 * Fri Nov 30 2018 Daniel Vrcic <[email protected]>, Katarina Zailac <[email protected]> - 1.7.1-1%{?dist}
 - ARGO-1428 ServiceGroup topology filtering
 - ARGO-1370 Optimize connectors queries to POEM

modules/input.py

@@ -1,17 +1,11 @@
 import base64
-import httplib
 import json
-import os
+import requests
 import socket
 import xml.dom.minidom
 
 from argo_egi_connectors.helpers import retry
 
-from OpenSSL.SSL import TLSv1_METHOD, Context, Connection
-from OpenSSL.SSL import VERIFY_PEER
-from OpenSSL.SSL import WantReadError as SSLWantReadError
-from ssl import SSLError
-from time import sleep
 from xml.parsers.expat import ExpatError
 from urlparse import urlparse
 
@@ -25,49 +19,45 @@ def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None):
     try:
         buf = None
 
-        if scheme.startswith('https'):
-            if eval(globopts['AuthenticationVerifyServerCert'.lower()]):
-                verify_cert(host, int(globopts['ConnectionTimeout'.lower()]),
-                            globopts['AuthenticationCAPath'.lower()],
-                            globopts['AuthenticationCAFile'.lower()])
-            conn = httplib.HTTPSConnection(host, 443,
-                                           globopts['AuthenticationHostKey'.lower()],
-                                           globopts['AuthenticationHostCert'.lower()],
-                                           timeout=int(globopts['ConnectionTimeout'.lower()]))
-        else:
-            conn = httplib.HTTPConnection(host, 80,
-                                          timeout=int(globopts['ConnectionTimeout'.lower()]))
-
         headers = {}
         if custauth and eval(custauth['AuthenticationUsePlainHttpAuth'.lower()]):
             userpass = base64.b64encode(custauth['AuthenticationHttpUser'.lower()] + ':' \
                                         + custauth['AuthenticationHttpPass'.lower()])
             headers={'Authorization': 'Basic ' + userpass}
 
-        conn.request('GET', url, headers=headers)
-        resp = conn.getresponse()
+        if scheme.startswith('https'):
+            response = requests.get('https://'+ host + url, headers=headers,
+                                    cert=(globopts['AuthenticationHostCert'.lower()],
+                                          globopts['AuthenticationHostKey'.lower()]),
+                                    verify=eval(globopts['AuthenticationVerifyServerCert'.lower()]),
+                                    timeout=int(globopts['ConnectionTimeout'.lower()]))
+            response.raise_for_status()
+        else:
+            response = requests.get('http://'+ host + url, headers=headers,
+                                    timeout=int(globopts['ConnectionTimeout'.lower()]))
 
-        if resp.status >= 300 and resp.status < 400:
-            headers = resp.getheaders()
+
+        if response.status_code >= 300 and response.status_code < 400:
+            headers = response.headers
             location = filter(lambda h: 'location' in h[0], headers)
             if location:
                 redir = urlparse(location[0][1])
             else:
-                raise httplib.HTTPException('No Location header set for redirect')
+                raise requests.exceptions.RequestException('No Location header set for redirect')
 
             return connection(logger, msgprefix, globopts, scheme, redir.netloc, redir.path + '?' + redir.query, custauth=custauth)
 
-        elif resp.status == 200:
-            buf = resp.read()
+        elif response.status_code == 200:
+            buf = response.content
             if not buf:
-                raise httplib.HTTPException('Empty response')
+                raise requests.exceptions.RequestException('Empty response')
 
         else:
-            raise httplib.HTTPException('Response: %s %s' % (resp.status, resp.reason))
+            raise requests.exceptions.RequestException('response: %s %s' % (response.status_code, response.reason))
 
         return buf
 
-    except SSLError as e:
+    except requests.exceptions.SSLError as e:
         if (getattr(e, 'args', False) and type(e.args) == tuple
             and type(e.args[0]) == str
             and 'timed out' in e.args[0]):
@@ -86,7 +76,7 @@ def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None):
                                                                        repr(e)))
         raise e
 
-    except httplib.HTTPException as e:
+    except requests.exceptions.RequestException as e:
         logger.warn('%sCustomer:%s Job:%s HTTP error %s - %s' % (msgprefix + ' ' if msgprefix else '',
                                                                  logger.customer, logger.job,
                                                                  scheme + '://' + host + url,
@@ -100,9 +90,6 @@ def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None):
                                                                 repr(e)))
         return False
 
-    finally:
-        conn.close()
-
 
 def parse_xml(logger, objname, globopts, buf, method):
     try:
@@ -134,64 +121,3 @@ def parse_json(logger, objname, globopts, buf, method):
 
     else:
         return doc
-
-
-def verify_cert(host, timeout, capath, cafile):
-    def verify_cert(host, ca, timeout):
-        server_ctx = Context(TLSv1_METHOD)
-        server_cert_chain = []
-
-        if os.path.isdir(ca):
-            server_ctx.load_verify_locations(None, ca)
-        else:
-            server_ctx.load_verify_locations(ca, None)
-
-        def verify_cb(conn, cert, errnum, depth, ok):
-            server_cert_chain.append(cert)
-            return ok
-        server_ctx.set_verify(VERIFY_PEER, verify_cb)
-
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.setblocking(1)
-        sock.settimeout(timeout)
-        sock.connect((host, 443))
-
-        server_conn = Connection(server_ctx, sock)
-        server_conn.set_connect_state()
-
-        def iosock_try():
-            ok = True
-            try:
-                server_conn.do_handshake()
-                sleep(0.5)
-            except SSLWantReadError as e:
-                ok = False
-                pass
-            except Exception as e:
-                raise e
-            return ok
-
-        try:
-            while True:
-                if iosock_try():
-                    break
-
-            server_subject = server_cert_chain[-1].get_subject()
-            if host != server_subject.CN:
-                raise SSLError('Server certificate CN does not match %s' % host)
-
-        except SSLError as e:
-            raise e
-        finally:
-            server_conn.shutdown()
-            server_conn.close()
-
-        return True
-
-    try:
-        verify_cert(host, capath, timeout)
-    except SSLError:
-        verify_cert(host, cafile, timeout)
-
-
-    return True