From 13a16cadeeb6c080850861dce7b90cd80fd6f3f3 Mon Sep 17 00:00:00 2001
From: nours <nours@staff.42.fr>
Date: Thu, 5 Dec 2013 02:10:02 +0100
Subject: [PATCH 1/5] Basic mongodb implem

---
 blueprint/io/http.py                   |   2 +-
 blueprint/io/server/__init__.py        |  11 +-
 blueprint/io/server/backend_mongodb.py | 144 +++++++++++++++++++++++++
 3 files changed, 148 insertions(+), 9 deletions(-)
 create mode 100644 blueprint/io/server/backend_mongodb.py

diff --git a/blueprint/io/http.py b/blueprint/io/http.py
index b75e1ca..e3572ec 100644
--- a/blueprint/io/http.py
+++ b/blueprint/io/http.py
@@ -11,7 +11,7 @@ def _connect(server=None):
         server = cfg.get('io', 'server')
     url = urlparse.urlparse(server)
     if -1 == url.netloc.find(':'):
-        port = url.port or 443 if 'https' == url.scheme else 80
+        port = url.port
     else:
         port = None
     if 'https' == url.scheme:
diff --git a/blueprint/io/server/__init__.py b/blueprint/io/server/__init__.py
index a7b6194..849e0cb 100644
--- a/blueprint/io/server/__init__.py
+++ b/blueprint/io/server/__init__.py
@@ -9,7 +9,7 @@
 
 from blueprint import Blueprint
 from blueprint import cfg
-import backend
+import backend_mongodb as backend
 import librato
 import statsd
 
@@ -101,12 +101,7 @@ def validate_content_length():
 
 @app.route('/secret', methods=['GET'])
 def secret():
-    while 1:
-        s = base64.urlsafe_b64encode(os.urandom(48))
-        try:
-            iter(backend.list(s)).next()
-        except StopIteration:
-            break
+    s = base64.urlsafe_b64encode(os.urandom(48))
     return MeteredResponse(response='{0}\n'.format(s),
                            status=201,
                            content_type='text/plain')
@@ -274,4 +269,4 @@ def user_data(secret, name):
 
 
 if '__main__' == __name__:
-    app.run(host='0.0.0.0', debug=True)
+    app.run(host='0.0.0.0', debug=True, threaded=True)
diff --git a/blueprint/io/server/backend_mongodb.py b/blueprint/io/server/backend_mongodb.py
new file mode 100644
index 0000000..a6bb4bc
--- /dev/null
+++ b/blueprint/io/server/backend_mongodb.py
@@ -0,0 +1,144 @@
+from pymongo import MongoClient
+import httplib
+import socket
+
+from blueprint import cfg
+import librato
+import statsd
+
+#pip install pymongo
+
+address = cfg.get('mongodb', 'address')
+port = cfg.get('mongodb', 'port')
+database = cfg.get('mongodb', 'database')
+collection = cfg.get('mongodb', 'collection')
+
+client = MongoClient(address, int(port))
+db = client[database]
+collection = db[collection]
+
+class StoredObject:
+	key = None
+	tarball = None
+	def __init__(self, key, tarball):
+		self.key = key
+		self.tarball = tarball
+
+def delete(key):
+    """
+    Remove an object from S3.  DELETE requests are free but this function
+    still makes one billable request to account for freed storage.
+    """
+    content_length = head(key)
+    if content_length is None:
+        return None
+    librato.count('blueprint-io-server.requests.delete')
+    statsd.increment('blueprint-io-server.requests.delete')
+    collection.delete({"key" : key})
+
+
+def delete_blueprint(secret, name):
+    return delete(key_for_blueprint(secret, name))
+
+
+def delete_tarball(secret, name, sha):
+    return delete(key_for_tarball(secret, name, sha))
+
+
+def get(key):
+    """
+    Fetch an object from S3.  This function makes one billable request.
+    """
+    librato.count('blueprint-io-server.requests.get')
+    statsd.increment('blueprint-io-server.requests.get')
+    k = collection.find_one({"key" : key})
+    if k is None:
+    	return False
+    return k.tarball
+
+
+def get_blueprint(secret, name):
+    return get(key_for_blueprint(secret, name))
+
+
+def get_tarball(secret, name, sha):
+    return get(key_for_tarball(secret, name, sha))
+
+
+def head(key):
+    """
+    Make a HEAD request for an object in S3.  This is needed to find the
+    object's length so it can be accounted.  This function makes one
+    billable request and anticipates another.
+    """
+    librato.count('blueprint-io-server.requests.head')
+    statsd.increment('blueprint-io-server.requests.head')
+    k = collection.find_one({"key" : key})
+    if k is None:
+      	return None
+    return len(k.tarball)
+
+
+def head_blueprint(secret, name):
+    return head(key_for_blueprint(secret, name))
+
+
+def head_tarball(secret, name, sha):
+    return head(key_for_tarball(secret, name, sha))
+
+
+def key_for_blueprint(secret, name):
+    return '{0}/{1}/{2}'.format(secret,
+                                name,
+                                'blueprint.json')
+
+
+def key_for_tarball(secret, name, sha):
+    return '{0}/{1}/{2}.tar'.format(secret,
+                                    name,
+                                    sha)
+
+
+def list(key):
+    """
+    List objects in S3 whose keys begin with the given prefix.  This
+    function makes at least one billable request.
+    """
+    librato.count('blueprint-io-server.requests.list')
+    statsd.increment('blueprint-io-server.requests.list')
+    result = collection.find({"key" : '^%s' % (key)})
+    return list(result)
+
+
+def put(key, data):
+    """
+    Store an object in S3.  This function makes one billable request.
+    """
+    librato.count('blueprint-io-server.requests.put')
+    statsd.increment('blueprint-io-server.requests.put')
+    # TODO librato.something('blueprint-io-server.storage', len(data))
+    statsd.update('blueprint-io-server.storage', len(data))
+    element = StoredObject(key, data)
+    collection.insert(element.__dict__)
+    return True
+
+
+
+def put_blueprint(secret, name, data):
+    return put(key_for_blueprint(secret, name), data)
+
+
+def put_tarball(secret, name, sha, data):
+    return put(key_for_tarball(secret, name, sha), data)
+
+
+def url_for(key):
+    return ''
+
+
+def url_for_blueprint(secret, name):
+    return url_for(key_for_blueprint(secret, name))
+
+
+def url_for_tarball(secret, name, sha):
+    return url_for(key_for_tarball(secret, name, sha))

From 610ea0de8d18a4550ebb6c56999636d0e928d580 Mon Sep 17 00:00:00 2001
From: nours <nours@staff.42.fr>
Date: Thu, 5 Dec 2013 17:04:31 +0100
Subject: [PATCH 2/5] Reversed changes I made to HTTP server

---
 blueprint/io/http.py            |  2 +-
 blueprint/io/server/__init__.py | 17 ++++++++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/blueprint/io/http.py b/blueprint/io/http.py
index e3572ec..b75e1ca 100644
--- a/blueprint/io/http.py
+++ b/blueprint/io/http.py
@@ -11,7 +11,7 @@ def _connect(server=None):
         server = cfg.get('io', 'server')
     url = urlparse.urlparse(server)
     if -1 == url.netloc.find(':'):
-        port = url.port
+        port = url.port or 443 if 'https' == url.scheme else 80
     else:
         port = None
     if 'https' == url.scheme:
diff --git a/blueprint/io/server/__init__.py b/blueprint/io/server/__init__.py
index 849e0cb..84d21b2 100644
--- a/blueprint/io/server/__init__.py
+++ b/blueprint/io/server/__init__.py
@@ -9,10 +9,16 @@
 
 from blueprint import Blueprint
 from blueprint import cfg
-import backend_mongodb as backend
+
 import librato
 import statsd
 
+backend = cfg.get('server', 'backend')
+if backend == "mongodb":
+    import backend_mongodb as backend
+else:
+    import backend
+
 
 app = Flask(__name__)
 
@@ -101,7 +107,12 @@ def validate_content_length():
 
 @app.route('/secret', methods=['GET'])
 def secret():
-    s = base64.urlsafe_b64encode(os.urandom(48))
+    while 1:
+        s = base64.urlsafe_b64encode(os.urandom(48))
+        try:
+            iter(backend.list(s)).next()
+        except StopIteration:
+            break
     return MeteredResponse(response='{0}\n'.format(s),
                            status=201,
                            content_type='text/plain')
@@ -269,4 +280,4 @@ def user_data(secret, name):
 
 
 if '__main__' == __name__:
-    app.run(host='0.0.0.0', debug=True, threaded=True)
+    app.run(host='0.0.0.0', debug=True)

From c9ac7e970eb3a78608098a43d6b957c5ef4f5a8b Mon Sep 17 00:00:00 2001
From: nours <nours@staff.42.fr>
Date: Thu, 5 Dec 2013 18:27:05 +0100
Subject: [PATCH 3/5] Working push and pull for mongodb

---
 blueprint/io/__init__.py               |  1 -
 blueprint/io/server/__init__.py        |  7 ++++++-
 blueprint/io/server/backend_mongodb.py | 25 ++++++++++++-------------
 3 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/blueprint/io/__init__.py b/blueprint/io/__init__.py
index 899d797..55adb4b 100644
--- a/blueprint/io/__init__.py
+++ b/blueprint/io/__init__.py
@@ -14,7 +14,6 @@ def pull(server, secret, name):
     r = http.get('/{0}/{1}'.format(secret, name), server=server)
     if 200 == r.status:
         b = Blueprint.load(r, name)
-
         for filename in b.sources.itervalues():
             logging.info('fetching source tarballs - this may take a while')
             r = http.get('/{0}/{1}/{2}'.format(secret, name, filename),
diff --git a/blueprint/io/server/__init__.py b/blueprint/io/server/__init__.py
index 84d21b2..f35efa6 100644
--- a/blueprint/io/server/__init__.py
+++ b/blueprint/io/server/__init__.py
@@ -22,7 +22,6 @@
 
 app = Flask(__name__)
 
-
 def _blueprint(secret, name):
     """
     Fetch a blueprint from S3 and turn it into a real Blueprint object.
@@ -120,6 +119,12 @@ def secret():
 
 browser_pattern = re.compile(r'Chrome|Gecko|Microsoft|Mozilla|Safari|WebKit')
 
+@app.route('/<secret>/<name>/blueprint.json', methods=['GET'])
+def get_json(secret, name):
+    validate_secret(secret)
+    validate_name(name)
+
+    return backend.get(backend.key_for_blueprint(secret, name))
 
 @app.route('/<secret>/<name>', methods=['GET'])
 def get_blueprint(secret, name):
diff --git a/blueprint/io/server/backend_mongodb.py b/blueprint/io/server/backend_mongodb.py
index a6bb4bc..376dcb2 100644
--- a/blueprint/io/server/backend_mongodb.py
+++ b/blueprint/io/server/backend_mongodb.py
@@ -10,8 +10,10 @@
 
 address = cfg.get('mongodb', 'address')
 port = cfg.get('mongodb', 'port')
+protocol = 'https' if cfg.getboolean('server', 'use_https') else 'http'
 database = cfg.get('mongodb', 'database')
 collection = cfg.get('mongodb', 'collection')
+url = cfg.get('server', 'url')
 
 client = MongoClient(address, int(port))
 db = client[database]
@@ -26,8 +28,7 @@ def __init__(self, key, tarball):
 
 def delete(key):
     """
-    Remove an object from S3.  DELETE requests are free but this function
-    still makes one billable request to account for freed storage.
+    Remove an object from MongoDB.
     """
     content_length = head(key)
     if content_length is None:
@@ -47,14 +48,14 @@ def delete_tarball(secret, name, sha):
 
 def get(key):
     """
-    Fetch an object from S3.  This function makes one billable request.
+    Fetch an object from MongoDB.
     """
     librato.count('blueprint-io-server.requests.get')
     statsd.increment('blueprint-io-server.requests.get')
     k = collection.find_one({"key" : key})
     if k is None:
     	return False
-    return k.tarball
+    return k['tarball']
 
 
 def get_blueprint(secret, name):
@@ -67,16 +68,15 @@ def get_tarball(secret, name, sha):
 
 def head(key):
     """
-    Make a HEAD request for an object in S3.  This is needed to find the
-    object's length so it can be accounted.  This function makes one
-    billable request and anticipates another.
+    Make a HEAD request for an object in MongoDB. 
+    This returns the size of the tarball.
     """
     librato.count('blueprint-io-server.requests.head')
     statsd.increment('blueprint-io-server.requests.head')
     k = collection.find_one({"key" : key})
     if k is None:
       	return None
-    return len(k.tarball)
+    return len(k['tarball'])
 
 
 def head_blueprint(secret, name):
@@ -101,18 +101,17 @@ def key_for_tarball(secret, name, sha):
 
 def list(key):
     """
-    List objects in S3 whose keys begin with the given prefix.  This
-    function makes at least one billable request.
+    List objects in MongoDB whose key begins with the given prefix
     """
     librato.count('blueprint-io-server.requests.list')
     statsd.increment('blueprint-io-server.requests.list')
     result = collection.find({"key" : '^%s' % (key)})
-    return list(result)
+    return result
 
 
 def put(key, data):
     """
-    Store an object in S3.  This function makes one billable request.
+    Store an object in MongoDB.
     """
     librato.count('blueprint-io-server.requests.put')
     statsd.increment('blueprint-io-server.requests.put')
@@ -133,7 +132,7 @@ def put_tarball(secret, name, sha, data):
 
 
 def url_for(key):
-    return ''
+    return '{0}://{1}/{2}'.format(protocol, url, key)
 
 
 def url_for_blueprint(secret, name):

From 33cb289e5bc1a098007f42257616dcfbbb49573f Mon Sep 17 00:00:00 2001
From: nours <nours@staff.42.fr>
Date: Thu, 5 Dec 2013 18:39:02 +0100
Subject: [PATCH 4/5] Better error handling

---
 blueprint/io/server/backend_mongodb.py | 36 ++++++++++++++++++--------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/blueprint/io/server/backend_mongodb.py b/blueprint/io/server/backend_mongodb.py
index 376dcb2..349d12e 100644
--- a/blueprint/io/server/backend_mongodb.py
+++ b/blueprint/io/server/backend_mongodb.py
@@ -13,7 +13,7 @@
 protocol = 'https' if cfg.getboolean('server', 'use_https') else 'http'
 database = cfg.get('mongodb', 'database')
 collection = cfg.get('mongodb', 'collection')
-url = cfg.get('server', 'url')
+url = cfg.get('server', 'address')
 
 client = MongoClient(address, int(port))
 db = client[database]
@@ -35,8 +35,10 @@ def delete(key):
         return None
     librato.count('blueprint-io-server.requests.delete')
     statsd.increment('blueprint-io-server.requests.delete')
-    collection.delete({"key" : key})
-
+    try:
+        collection.delete({"key" : key})
+    except:
+        return False
 
 def delete_blueprint(secret, name):
     return delete(key_for_blueprint(secret, name))
@@ -52,9 +54,12 @@ def get(key):
     """
     librato.count('blueprint-io-server.requests.get')
     statsd.increment('blueprint-io-server.requests.get')
-    k = collection.find_one({"key" : key})
-    if k is None:
-    	return False
+    try:
+        k = collection.find_one({"key" : key})
+        if k is None:
+    	   return None
+    except:
+        return False
     return k['tarball']
 
 
@@ -73,9 +78,12 @@ def head(key):
     """
     librato.count('blueprint-io-server.requests.head')
     statsd.increment('blueprint-io-server.requests.head')
-    k = collection.find_one({"key" : key})
-    if k is None:
-      	return None
+    try:
+        k = collection.find_one({"key" : key})
+        if k is None:
+            return None
+    except:
+        return False
     return len(k['tarball'])
 
 
@@ -105,7 +113,10 @@ def list(key):
     """
     librato.count('blueprint-io-server.requests.list')
     statsd.increment('blueprint-io-server.requests.list')
-    result = collection.find({"key" : '^%s' % (key)})
+    try:
+        result = collection.find({"key" : '^%s' % (key)})
+    except:
+        return False
     return result
 
 
@@ -118,7 +129,10 @@ def put(key, data):
     # TODO librato.something('blueprint-io-server.storage', len(data))
     statsd.update('blueprint-io-server.storage', len(data))
     element = StoredObject(key, data)
-    collection.insert(element.__dict__)
+    try:
+        collection.insert(element.__dict__)
+    except:
+        return False
     return True
 
 

From 162ed7e293baf05757429c608343de259c6a964a Mon Sep 17 00:00:00 2001
From: nours <nours@staff.42.fr>
Date: Thu, 5 Dec 2013 18:58:04 +0100
Subject: [PATCH 5/5] We now handle auth to MongoDB

---
 blueprint/io/server/backend_mongodb.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/blueprint/io/server/backend_mongodb.py b/blueprint/io/server/backend_mongodb.py
index 349d12e..59b1d22 100644
--- a/blueprint/io/server/backend_mongodb.py
+++ b/blueprint/io/server/backend_mongodb.py
@@ -6,17 +6,26 @@
 import librato
 import statsd
 
-#pip install pymongo
+username = None
+password = None
 
 address = cfg.get('mongodb', 'address')
 port = cfg.get('mongodb', 'port')
-protocol = 'https' if cfg.getboolean('server', 'use_https') else 'http'
 database = cfg.get('mongodb', 'database')
 collection = cfg.get('mongodb', 'collection')
 url = cfg.get('server', 'address')
+protocol = 'https' if cfg.getboolean('server', 'use_https') else 'http'
+try:
+    username = cfg.get('mongodb', 'user')
+    password = cfg.get('mongodb', 'password')
+except:
+    pass
 
 client = MongoClient(address, int(port))
 db = client[database]
+if username is not None:
+    if password is not None:
+        db.authenticate(username, password)
 collection = db[collection]
 
 class StoredObject: