black style

config.py

@@ -10,7 +10,7 @@
 
 config = Dynaconf(
     envvar_prefix=False,
-    settings_files=['config.yaml'],
+    settings_files=["config.yaml"],
 )
 
 
@@ -19,9 +19,11 @@ class ConfigError(Exception):
 
 
 def validate_config(config):
-    """ Validate config - make sure values are consistent """
+    """Validate config - make sure values are consistent"""
 
-    if not (config.mergin.username and config.mergin.password and config.mergin.project_name):
+    if not (
+        config.mergin.username and config.mergin.password and config.mergin.project_name
+    ):
         raise ConfigError("Config error: Incorrect mergin settings")
 
     if config.driver not in ["local", "minio"]:
@@ -30,10 +32,15 @@ def validate_config(config):
     if config.operation_mode not in ["move", "copy"]:
         raise ConfigError("Config error: Unsupported operation mode")
 
-    if config.driver == 'local' and not config.local.dest:
+    if config.driver == "local" and not config.local.dest:
         raise ConfigError("Config error: Incorrect Local driver settings")
 
-    if config.driver == 'minio' and not (config.minio.endpoint and config.minio.access_key and config.minio.secret_key and config.minio.bucket):
+    if config.driver == "minio" and not (
+        config.minio.endpoint
+        and config.minio.access_key
+        and config.minio.secret_key
+        and config.minio.bucket
+    ):
         raise ConfigError("Config error: Incorrect MinIO driver settings")
 
     if not (config.allowed_extensions and len(config.allowed_extensions)):
@@ -43,5 +50,8 @@ def validate_config(config):
         raise ConfigError("Config error: References list can not be empty")
 
     for ref in config.references:
-        if not all(hasattr(ref, attr) for attr in ['file', 'table', 'local_path_column', 'driver_path_column']):
+        if not all(
+            hasattr(ref, attr)
+            for attr in ["file", "table", "local_path_column", "driver_path_column"]
+        ):
             raise ConfigError("Config error: Incorrect media reference settings")

drivers.py

@@ -22,12 +22,12 @@ def __init__(self, config):
         self.config = config
 
     def upload_file(self, src, obj_path):
-        """ Copy object to destination and return path """
+        """Copy object to destination and return path"""
         raise NotImplementedError
 
 
 class LocalDriver(Driver):
-    """ Driver to work with local drive, for testing purpose mainly """
+    """Driver to work with local drive, for testing purpose mainly"""
 
     def __init__(self, config):
         super(LocalDriver, self).__init__(config)
@@ -52,7 +52,7 @@ def upload_file(self, src, obj_path):
 
 
 class MinioDriver(Driver):
-    """ Driver to handle connection to minio-like server """
+    """Driver to handle connection to minio-like server"""
 
     def __init__(self, config):
         super(MinioDriver, self).__init__(config)
@@ -63,7 +63,7 @@ def __init__(self, config):
                 access_key=config.minio.access_key,
                 secret_key=config.minio.secret_key,
                 secure=config.as_bool("minio.secure"),
-                region=config.minio.region
+                region=config.minio.region,
             )
             self.bucket = config.minio.bucket
             bucket_found = self.client.bucket_exists(self.bucket)
@@ -77,7 +77,7 @@ def __init__(self, config):
 
             # construct base url for bucket
             scheme = "https://" if config.as_bool("minio.secure") else "http://"
-            self.base_url = scheme + config.minio.endpoint + '/' + self.bucket
+            self.base_url = scheme + config.minio.endpoint + "/" + self.bucket
         except S3Error as e:
             raise DriverError("MinIO driver init error: " + str(e))
 
@@ -86,14 +86,14 @@ def upload_file(self, src, obj_path):
             obj_path = f"{self.bucket_subpath}/{obj_path}"
         try:
             res = self.client.fput_object(self.bucket, obj_path, src)
-            dest = self.base_url + '/' + res.object_name
+            dest = self.base_url + "/" + res.object_name
         except S3Error as e:
             raise DriverError("MinIO driver error: " + str(e))
         return dest
 
 
 def create_driver(config):
-    """ Create driver object based on type defined in config """
+    """Create driver object based on type defined in config"""
     driver = None
     if config.driver == "local":
         driver = LocalDriver(config)

media_sync.py

@@ -21,58 +21,79 @@ class MediaSyncError(Exception):
 
 def _quote_identifier(identifier):
     """Quote identifiers"""
-    return "\"" + identifier + "\""
+    return '"' + identifier + '"'
 
 
 def _get_project_version():
-    """ Returns the current version of the project """
+    """Returns the current version of the project"""
     mp = MerginProject(config.project_working_dir)
     return mp.version()
 
 
 def _check_has_working_dir():
     if not os.path.exists(config.project_working_dir):
-        raise MediaSyncError("The project working directory does not exist: " + config.project_working_dir)
+        raise MediaSyncError(
+            "The project working directory does not exist: "
+            + config.project_working_dir
+        )
 
-    if not os.path.exists(os.path.join(config.project_working_dir, '.mergin')):
-        raise MediaSyncError("The project working directory does not seem to contain Mergin project: " + config.project_working_dir)
+    if not os.path.exists(os.path.join(config.project_working_dir, ".mergin")):
+        raise MediaSyncError(
+            "The project working directory does not seem to contain Mergin project: "
+            + config.project_working_dir
+        )
 
 
 def _check_pending_changes():
-    """ Check working directory was not modified manually - this is probably uncommitted change from last attempt"""
+    """Check working directory was not modified manually - this is probably uncommitted change from last attempt"""
     mp = MerginProject(config.project_working_dir)
     status_push = mp.get_push_changes()
-    if status_push['added'] or status_push['updated'] or status_push['removed']:
+    if status_push["added"] or status_push["updated"] or status_push["removed"]:
         raise MediaSyncError(
-            "There are pending changes in the local directory - please review and push manually! " + str(status_push))
+            "There are pending changes in the local directory - please review and push manually! "
+            + str(status_push)
+        )
 
 
 def _get_media_sync_files(files):
-    """ Return files relevant to media sync from project files """
+    """Return files relevant to media sync from project files"""
     allowed_extensions = config.allowed_extensions
-    files_to_upload = [f for f in files if os.path.splitext(f["path"])[1].lstrip('.') in allowed_extensions]
+    files_to_upload = [
+        f
+        for f in files
+        if os.path.splitext(f["path"])[1].lstrip(".") in allowed_extensions
+    ]
     # filter out files which are not under particular directory in mergin project
     if "base_path" in config and config.base_path:
-        filtered_files = [f for f in files_to_upload if f["path"].startswith(config.base_path)]
+        filtered_files = [
+            f for f in files_to_upload if f["path"].startswith(config.base_path)
+        ]
         files_to_upload = filtered_files
     return files_to_upload
 
 
 def create_mergin_client():
-    """ Create instance of MerginClient"""
+    """Create instance of MerginClient"""
     try:
-        return MerginClient(config.mergin.url, login=config.mergin.username, password=config.mergin.password, plugin_version=f"media-sync/{__version__}")
+        return MerginClient(
+            config.mergin.url,
+            login=config.mergin.username,
+            password=config.mergin.password,
+            plugin_version=f"media-sync/{__version__}",
+        )
     except LoginError as e:
         # this could be auth failure, but could be also server problem (e.g. worker crash)
-        raise MediaSyncError(f"Unable to log in to Mergin: {str(e)} \n\n" +
-                          "Have you specified correct credentials in configuration file?")
+        raise MediaSyncError(
+            f"Unable to log in to Mergin: {str(e)} \n\n"
+            + "Have you specified correct credentials in configuration file?"
+        )
     except ClientError as e:
         # this could be e.g. DNS error
         raise MediaSyncError("Mergin client error: " + str(e))
 
 
 def mc_download(mc):
-    """ Clone mergin project to local dir
+    """Clone mergin project to local dir
     :param mc: mergin client instance
     :return: list(dict) list of project files metadata
     """
@@ -89,7 +110,7 @@ def mc_download(mc):
 
 
 def mc_pull(mc):
-    """ Pull latest version to synchronize with local dir
+    """Pull latest version to synchronize with local dir
     :param mc: mergin client instance
     :return: list(dict) list of project files metadata
     """
@@ -120,33 +141,46 @@ def mc_pull(mc):
         raise MediaSyncError("Mergin client error on pull: " + str(e))
 
     print("Pulled new version from Mergin: " + _get_project_version())
-    files_to_upload = _get_media_sync_files(status_pull["added"]+status_pull["updated"])
+    files_to_upload = _get_media_sync_files(
+        status_pull["added"] + status_pull["updated"]
+    )
     return files_to_upload
 
 
 def _update_references(files):
-    """ Update references to media files in reference table """
+    """Update references to media files in reference table"""
     for ref in config.references:
-        reference_config = [ref.file, ref.table, ref.local_path_column, ref.driver_path_column]
+        reference_config = [
+            ref.file,
+            ref.table,
+            ref.local_path_column,
+            ref.driver_path_column,
+        ]
         if not all(reference_config):
             return
 
         print("Updating references ...")
         try:
-            gpkg_conn = sqlite3.connect(os.path.join(config.project_working_dir, ref.file))
+            gpkg_conn = sqlite3.connect(
+                os.path.join(config.project_working_dir, ref.file)
+            )
             gpkg_conn.enable_load_extension(True)
             gpkg_cur = gpkg_conn.cursor()
             gpkg_cur.execute('SELECT load_extension("mod_spatialite")')
             for file_path, dest in files.items():
                 # remove reference to the local path only in the move mode
                 if config.operation_mode == "move":
-                    sql = f"UPDATE {_quote_identifier(ref.table)} " \
-                          f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column, {_quote_identifier(ref.local_path_column)}=Null " \
-                          f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path"
+                    sql = (
+                        f"UPDATE {_quote_identifier(ref.table)} "
+                        f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column, {_quote_identifier(ref.local_path_column)}=Null "
+                        f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path"
+                    )
                 elif config.operation_mode == "copy":
-                    sql = f"UPDATE {_quote_identifier(ref.table)} " \
-                          f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column " \
-                          f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path"
+                    sql = (
+                        f"UPDATE {_quote_identifier(ref.table)} "
+                        f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column "
+                        f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path"
+                    )
                 gpkg_cur.execute(sql, {"dest_column": dest, "file_path": file_path})
             gpkg_conn.commit()
             gpkg_conn.close()
@@ -172,7 +206,7 @@ def media_sync_push(mc, driver, files):
             size = os.path.getsize(src) / 1024 / 1024  # file size in MB
             print(f"Uploading {file['path']} of size {size:.2f} MB")
             dest = driver.upload_file(src, file["path"])
-            migrated_files[file['path']] = dest
+            migrated_files[file["path"]] = dest
         except DriverError as e:
             print(f"Failed to upload {file['path']}: " + str(e))
             continue
@@ -191,7 +225,9 @@ def media_sync_push(mc, driver, files):
         mp = MerginProject(config.project_working_dir)
         status_push = mp.get_push_changes()
         if status_push["added"]:
-            raise MediaSyncError("There are changes to be added - it should never happen")
+            raise MediaSyncError(
+                "There are changes to be added - it should never happen"
+            )
         if status_push["updated"] or status_push["removed"]:
             mc.push_project(config.project_working_dir)
             version = _get_project_version()
@@ -238,5 +274,5 @@ def main():
         print("Error: " + str(err))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

media_sync_daemon.py

@@ -10,7 +10,13 @@
 import os
 import time
 from drivers import DriverError, create_driver
-from media_sync import create_mergin_client, mc_download, media_sync_push, mc_pull, MediaSyncError
+from media_sync import (
+    create_mergin_client,
+    mc_download,
+    media_sync_push,
+    mc_pull,
+    MediaSyncError,
+)
 from config import config, validate_config, ConfigError
 from version import __version__
 
@@ -53,7 +59,9 @@ def main():
             media_sync_push(mc, driver, files_to_sync)
 
             # check mergin client token expiration
-            delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc)
+            delta = mc._auth_session["expire"] - datetime.datetime.now(
+                datetime.timezone.utc
+            )
             if delta.total_seconds() < 3600:
                 mc = create_mergin_client()
 
@@ -64,5 +72,5 @@ def main():
         time.sleep(sleep_time)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

test/conftest.py

@@ -44,7 +44,7 @@ def setup_config():
             "MINIO__BUCKET": "",
             "MINIO__BUCKET_SUBPATH": "",
             "MINIO__SECURE": False,
-            "MINIO__REGION": ""
+            "MINIO__REGION": "",
         }
     )