MongoDB support for storage

blueprint/io/__init__.py

@@ -14,7 +14,6 @@ def pull(server, secret, name):
     r = http.get('/{0}/{1}'.format(secret, name), server=server)
     if 200 == r.status:
         b = Blueprint.load(r, name)
-
         for filename in b.sources.itervalues():
             logging.info('fetching source tarballs - this may take a while')
             r = http.get('/{0}/{1}/{2}'.format(secret, name, filename),

blueprint/io/server/__init__.py

@@ -9,13 +9,18 @@
 
 from blueprint import Blueprint
 from blueprint import cfg
-import backend
+
 import librato
 import statsd
 
+backend = cfg.get('server', 'backend')
+if backend == "mongodb":
+    import backend_mongodb as backend
+else:
+    import backend
 
-app = Flask(__name__)
 
+app = Flask(__name__)
 
 def _blueprint(secret, name):
     """
@@ -114,6 +119,12 @@ def secret():
 
 browser_pattern = re.compile(r'Chrome|Gecko|Microsoft|Mozilla|Safari|WebKit')
 
+@app.route('/<secret>/<name>/blueprint.json', methods=['GET'])
+def get_json(secret, name):
+    validate_secret(secret)
+    validate_name(name)
+
+    return backend.get(backend.key_for_blueprint(secret, name))
 
 @app.route('/<secret>/<name>', methods=['GET'])
 def get_blueprint(secret, name):

blueprint/io/server/backend_mongodb.py

@@ -0,0 +1,166 @@
+from pymongo import MongoClient
+import httplib
+import socket
+
+from blueprint import cfg
+import librato
+import statsd
+
+username = None
+password = None
+
+address = cfg.get('mongodb', 'address')
+port = cfg.get('mongodb', 'port')
+database = cfg.get('mongodb', 'database')
+collection = cfg.get('mongodb', 'collection')
+url = cfg.get('server', 'address')
+protocol = 'https' if cfg.getboolean('server', 'use_https') else 'http'
+try:
+    username = cfg.get('mongodb', 'user')
+    password = cfg.get('mongodb', 'password')
+except:
+    pass
+
+client = MongoClient(address, int(port))
+db = client[database]
+if username is not None:
+    if password is not None:
+        db.authenticate(username, password)
+collection = db[collection]
+
+class StoredObject:
+	key = None
+	tarball = None
+	def __init__(self, key, tarball):
+		self.key = key
+		self.tarball = tarball
+
+def delete(key):
+    """
+    Remove an object from MongoDB.
+    """
+    content_length = head(key)
+    if content_length is None:
+        return None
+    librato.count('blueprint-io-server.requests.delete')
+    statsd.increment('blueprint-io-server.requests.delete')
+    try:
+        collection.delete({"key" : key})
+    except:
+        return False
+
+def delete_blueprint(secret, name):
+    return delete(key_for_blueprint(secret, name))
+
+
+def delete_tarball(secret, name, sha):
+    return delete(key_for_tarball(secret, name, sha))
+
+
+def get(key):
+    """
+    Fetch an object from MongoDB.
+    """
+    librato.count('blueprint-io-server.requests.get')
+    statsd.increment('blueprint-io-server.requests.get')
+    try:
+        k = collection.find_one({"key" : key})
+        if k is None:
+    	   return None
+    except:
+        return False
+    return k['tarball']
+
+
+def get_blueprint(secret, name):
+    return get(key_for_blueprint(secret, name))
+
+
+def get_tarball(secret, name, sha):
+    return get(key_for_tarball(secret, name, sha))
+
+
+def head(key):
+    """
+    Make a HEAD request for an object in MongoDB. 
+    This returns the size of the tarball.
+    """
+    librato.count('blueprint-io-server.requests.head')
+    statsd.increment('blueprint-io-server.requests.head')
+    try:
+        k = collection.find_one({"key" : key})
+        if k is None:
+            return None
+    except:
+        return False
+    return len(k['tarball'])
+
+
+def head_blueprint(secret, name):
+    return head(key_for_blueprint(secret, name))
+
+
+def head_tarball(secret, name, sha):
+    return head(key_for_tarball(secret, name, sha))
+
+
+def key_for_blueprint(secret, name):
+    return '{0}/{1}/{2}'.format(secret,
+                                name,
+                                'blueprint.json')
+
+
+def key_for_tarball(secret, name, sha):
+    return '{0}/{1}/{2}.tar'.format(secret,
+                                    name,
+                                    sha)
+
+
+def list(key):
+    """
+    List objects in MongoDB whose key begins with the given prefix
+    """
+    librato.count('blueprint-io-server.requests.list')
+    statsd.increment('blueprint-io-server.requests.list')
+    try:
+        result = collection.find({"key" : '^%s' % (key)})
+    except:
+        return False
+    return result
+
+
+def put(key, data):
+    """
+    Store an object in MongoDB.
+    """
+    librato.count('blueprint-io-server.requests.put')
+    statsd.increment('blueprint-io-server.requests.put')
+    # TODO librato.something('blueprint-io-server.storage', len(data))
+    statsd.update('blueprint-io-server.storage', len(data))
+    element = StoredObject(key, data)
+    try:
+        collection.insert(element.__dict__)
+    except:
+        return False
+    return True
+
+
+
+def put_blueprint(secret, name, data):
+    return put(key_for_blueprint(secret, name), data)
+
+
+def put_tarball(secret, name, sha, data):
+    return put(key_for_tarball(secret, name, sha), data)
+
+
+def url_for(key):
+    return '{0}://{1}/{2}'.format(protocol, url, key)
+
+
+def url_for_blueprint(secret, name):
+    return url_for(key_for_blueprint(secret, name))
+
+
+def url_for_tarball(secret, name, sha):
+    return url_for(key_for_tarball(secret, name, sha))