cache: Add an exception type and make sure cache errors are non fatal

If the cache fails for whatever reason then carry on without it. We
generally don't want cache errors to be fatal but they are useful to
know about so we just print the exceptions which will get picked up by
the monitoring.

getter/cache.py

@@ -6,51 +6,66 @@
 DATABASE_NAME = "cache_datagetter.db"
 CACHE_DIR = "cache_dir"
 
+class DatagetterCacheError(Exception):
+    pass
+
 
 def setup_database():
-    con = sqlite3.connect(DATABASE_NAME)
-    cur = con.cursor()
-    cur.execute(
-        """CREATE TABLE IF NOT EXISTS cache
-        (original_file_name TEXT NOT NULL UNIQUE,
-         hash TEXT NOT NULL UNIQUE,
-         json_file TEXT NOT NULL UNIQUE);"""
-    )
-    con.commit()
-    con.close()
+    try:
+        con = sqlite3.connect(DATABASE_NAME)
+        cur = con.cursor()
+        cur.execute(
+            """CREATE TABLE IF NOT EXISTS cache
+            (original_file_name TEXT NOT NULL UNIQUE,
+            hash TEXT NOT NULL UNIQUE,
+            json_file TEXT NOT NULL UNIQUE);"""
+        )
+        con.commit()
+        con.close()
+    except Exception as e:
+        raise DatagetterCacheError(e)
 
 
 def setup_cache_dir():
-    os.makedirs(CACHE_DIR, exist_ok=True)
+    try:
+        os.makedirs(CACHE_DIR, exist_ok=True)
+    except Exception as e:
+        raise DatagetterCacheError(e)
 
 
 def hash_file(original_file_path):
-    file_hash = hashlib.sha1()
+    try:
+        file_hash = hashlib.sha1()
 
-    with open(original_file_path, "rb") as fp:
-        while True:
-            data = fp.read(6000)
-            if not data:
-                break
+        with open(original_file_path, "rb") as fp:
+            while True:
+                data = fp.read(6000)
+                if not data:
+                    break
 
-            file_hash.update(data)
+                file_hash.update(data)
+    except Exception as e:
+        raise DatagetterCacheError(e)
 
     return file_hash.hexdigest()
 
 
 def get_file(file_hash_str):
-    con = sqlite3.connect(DATABASE_NAME)
-    cur = con.cursor()
-    cur.execute("SELECT json_file FROM cache WHERE hash = ?", (file_hash_str,))
-    row = cur.fetchone()
-    con.close()
-
-    # We haven't already converted the file
-    if not row:
-        return False
-
-    # We have converted the file before so copy from the CACHE_DIR
-    return os.path.join(CACHE_DIR, row[0])
+    try:
+        con = sqlite3.connect(DATABASE_NAME)
+        cur = con.cursor()
+        cur.execute("SELECT json_file FROM cache WHERE hash = ?", (file_hash_str,))
+        row = cur.fetchone()
+        con.close()
+
+        # We haven't already converted the file
+        if not row:
+            return False
+
+        # We have converted the file before so copy from the CACHE_DIR
+        return os.path.join(CACHE_DIR, row[0])
+    except Exception as e:
+        raise DatagetterCacheError(e)
 
 
 def update_cache(json_file_name, file_hash_str, file_identifier, file_type):

getter/get.py

@@ -211,18 +211,22 @@ def fetch_and_convert(args, dataset, schema_path, package_schema):
                     print("Running convert on %s to %s" % (original_file_path,
                                                            json_file_name))
 
-                    # Hash the file
-                    file_hash_str = cache.hash_file(original_file_path)
-
-                    # Check if we have already converted the file
-                    cached_file_path = cache.get_file(file_hash_str)
-
-                    # We have converted the file before so copy from the CACHE_DIR
-                    if cached_file_path:
-                        try:
-                            shutil.copy(cached_file_path, json_file_name)
-                            print("Cache hit")
-                        except FileNotFoundError:
+                    try:
+                        # Hash the file
+                        file_hash_str = cache.hash_file(original_file_path)
+                        # Check if we have already converted the file
+                        cached_file_path = cache.get_file(file_hash_str)
+
+                        # We have converted the file before so copy from the CACHE_DIR
+                        if cached_file_path:
+                            try:
+                                shutil.copy(cached_file_path, json_file_name)
+                                print("Cache hit")
+                            except (FileNotFoundError, PermissionError):
+                                cached_file_path = False
+                    except cache.DatagetterCacheError as e:
+                            print(e)
+                            print("Continuing without cache")
                             cached_file_path = False
 
                     if not cached_file_path:
@@ -233,12 +237,15 @@ def fetch_and_convert(args, dataset, schema_path, package_schema):
                             schema_path,
                             args.schema_branch
                             )
-
-                        cache.update_cache(
-                                json_file_name,
-                                file_hash_str,
-                                dataset['identifier'],
-                                file_type)
+                        try:
+                            cache.update_cache(
+                                    json_file_name,
+                                    file_hash_str,
+                                    dataset['identifier'],
+                                    file_type)
+                        except cache.DatagetterCacheError as e:
+                            print(e)
+                            print("Continuing without cache")
 
                 except KeyboardInterrupt:
                     raise
@@ -309,8 +316,12 @@ def file_cache_schema(schema_branch):
 
 def get(args):
 
-    cache.setup_database()
-    cache.setup_cache_dir()
+    try:
+        cache.setup_database()
+        cache.setup_cache_dir()
+    except cache.DatagetterCacheError as e:
+        print(e)
+        print("Continuing without cache")
 
     if not args.download:
         mkdirs(args.data_dir, True)