From edfb35817b0d9d17819752cb7426d770e5b73974 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 20 Mar 2024 15:44:05 +0100 Subject: [PATCH] black style --- config.py | 22 ++- drivers.py | 14 +- media_sync.py | 96 ++++++++---- media_sync_daemon.py | 14 +- test/conftest.py | 2 +- test/test_config.py | 89 +++++++---- test/test_sync.py | 362 ++++++++++++++++++++++++++++++------------- version.py | 2 +- 8 files changed, 411 insertions(+), 190 deletions(-) diff --git a/config.py b/config.py index 43d5a45..83a49e9 100644 --- a/config.py +++ b/config.py @@ -10,7 +10,7 @@ config = Dynaconf( envvar_prefix=False, - settings_files=['config.yaml'], + settings_files=["config.yaml"], ) @@ -19,9 +19,11 @@ class ConfigError(Exception): def validate_config(config): - """ Validate config - make sure values are consistent """ + """Validate config - make sure values are consistent""" - if not (config.mergin.username and config.mergin.password and config.mergin.project_name): + if not ( + config.mergin.username and config.mergin.password and config.mergin.project_name + ): raise ConfigError("Config error: Incorrect mergin settings") if config.driver not in ["local", "minio"]: @@ -30,10 +32,15 @@ def validate_config(config): if config.operation_mode not in ["move", "copy"]: raise ConfigError("Config error: Unsupported operation mode") - if config.driver == 'local' and not config.local.dest: + if config.driver == "local" and not config.local.dest: raise ConfigError("Config error: Incorrect Local driver settings") - if config.driver == 'minio' and not (config.minio.endpoint and config.minio.access_key and config.minio.secret_key and config.minio.bucket): + if config.driver == "minio" and not ( + config.minio.endpoint + and config.minio.access_key + and config.minio.secret_key + and config.minio.bucket + ): raise ConfigError("Config error: Incorrect MinIO driver settings") if not (config.allowed_extensions and len(config.allowed_extensions)): @@ -43,5 +50,8 @@ def validate_config(config): raise ConfigError("Config error: References list can not be empty") for ref in config.references: - if not all(hasattr(ref, attr) for attr in ['file', 'table', 'local_path_column', 'driver_path_column']): + if not all( + hasattr(ref, attr) + for attr in ["file", "table", "local_path_column", "driver_path_column"] + ): raise ConfigError("Config error: Incorrect media reference settings") diff --git a/drivers.py b/drivers.py index a3936b2..b08d1a4 100644 --- a/drivers.py +++ b/drivers.py @@ -22,12 +22,12 @@ def __init__(self, config): self.config = config def upload_file(self, src, obj_path): - """ Copy object to destination and return path """ + """Copy object to destination and return path""" raise NotImplementedError class LocalDriver(Driver): - """ Driver to work with local drive, for testing purpose mainly """ + """Driver to work with local drive, for testing purpose mainly""" def __init__(self, config): super(LocalDriver, self).__init__(config) @@ -52,7 +52,7 @@ def upload_file(self, src, obj_path): class MinioDriver(Driver): - """ Driver to handle connection to minio-like server """ + """Driver to handle connection to minio-like server""" def __init__(self, config): super(MinioDriver, self).__init__(config) @@ -63,7 +63,7 @@ def __init__(self, config): access_key=config.minio.access_key, secret_key=config.minio.secret_key, secure=config.as_bool("minio.secure"), - region=config.minio.region + region=config.minio.region, ) self.bucket = config.minio.bucket bucket_found = self.client.bucket_exists(self.bucket) @@ -77,7 +77,7 @@ def __init__(self, config): # construct base url for bucket scheme = "https://" if config.as_bool("minio.secure") else "http://" - self.base_url = scheme + config.minio.endpoint + '/' + self.bucket + self.base_url = scheme + config.minio.endpoint + "/" + self.bucket except S3Error as e: raise DriverError("MinIO driver init error: " + str(e)) @@ -86,14 +86,14 @@ def upload_file(self, src, obj_path): obj_path = f"{self.bucket_subpath}/{obj_path}" try: res = self.client.fput_object(self.bucket, obj_path, src) - dest = self.base_url + '/' + res.object_name + dest = self.base_url + "/" + res.object_name except S3Error as e: raise DriverError("MinIO driver error: " + str(e)) return dest def create_driver(config): - """ Create driver object based on type defined in config """ + """Create driver object based on type defined in config""" driver = None if config.driver == "local": driver = LocalDriver(config) diff --git a/media_sync.py b/media_sync.py index 07305fe..f3a6ec8 100644 --- a/media_sync.py +++ b/media_sync.py @@ -21,58 +21,79 @@ class MediaSyncError(Exception): def _quote_identifier(identifier): """Quote identifiers""" - return "\"" + identifier + "\"" + return '"' + identifier + '"' def _get_project_version(): - """ Returns the current version of the project """ + """Returns the current version of the project""" mp = MerginProject(config.project_working_dir) return mp.version() def _check_has_working_dir(): if not os.path.exists(config.project_working_dir): - raise MediaSyncError("The project working directory does not exist: " + config.project_working_dir) + raise MediaSyncError( + "The project working directory does not exist: " + + config.project_working_dir + ) - if not os.path.exists(os.path.join(config.project_working_dir, '.mergin')): - raise MediaSyncError("The project working directory does not seem to contain Mergin project: " + config.project_working_dir) + if not os.path.exists(os.path.join(config.project_working_dir, ".mergin")): + raise MediaSyncError( + "The project working directory does not seem to contain Mergin project: " + + config.project_working_dir + ) def _check_pending_changes(): - """ Check working directory was not modified manually - this is probably uncommitted change from last attempt""" + """Check working directory was not modified manually - this is probably uncommitted change from last attempt""" mp = MerginProject(config.project_working_dir) status_push = mp.get_push_changes() - if status_push['added'] or status_push['updated'] or status_push['removed']: + if status_push["added"] or status_push["updated"] or status_push["removed"]: raise MediaSyncError( - "There are pending changes in the local directory - please review and push manually! " + str(status_push)) + "There are pending changes in the local directory - please review and push manually! " + + str(status_push) + ) def _get_media_sync_files(files): - """ Return files relevant to media sync from project files """ + """Return files relevant to media sync from project files""" allowed_extensions = config.allowed_extensions - files_to_upload = [f for f in files if os.path.splitext(f["path"])[1].lstrip('.') in allowed_extensions] + files_to_upload = [ + f + for f in files + if os.path.splitext(f["path"])[1].lstrip(".") in allowed_extensions + ] # filter out files which are not under particular directory in mergin project if "base_path" in config and config.base_path: - filtered_files = [f for f in files_to_upload if f["path"].startswith(config.base_path)] + filtered_files = [ + f for f in files_to_upload if f["path"].startswith(config.base_path) + ] files_to_upload = filtered_files return files_to_upload def create_mergin_client(): - """ Create instance of MerginClient""" + """Create instance of MerginClient""" try: - return MerginClient(config.mergin.url, login=config.mergin.username, password=config.mergin.password, plugin_version=f"media-sync/{__version__}") + return MerginClient( + config.mergin.url, + login=config.mergin.username, + password=config.mergin.password, + plugin_version=f"media-sync/{__version__}", + ) except LoginError as e: # this could be auth failure, but could be also server problem (e.g. worker crash) - raise MediaSyncError(f"Unable to log in to Mergin: {str(e)} \n\n" + - "Have you specified correct credentials in configuration file?") + raise MediaSyncError( + f"Unable to log in to Mergin: {str(e)} \n\n" + + "Have you specified correct credentials in configuration file?" + ) except ClientError as e: # this could be e.g. DNS error raise MediaSyncError("Mergin client error: " + str(e)) def mc_download(mc): - """ Clone mergin project to local dir + """Clone mergin project to local dir :param mc: mergin client instance :return: list(dict) list of project files metadata """ @@ -89,7 +110,7 @@ def mc_download(mc): def mc_pull(mc): - """ Pull latest version to synchronize with local dir + """Pull latest version to synchronize with local dir :param mc: mergin client instance :return: list(dict) list of project files metadata """ @@ -120,33 +141,46 @@ def mc_pull(mc): raise MediaSyncError("Mergin client error on pull: " + str(e)) print("Pulled new version from Mergin: " + _get_project_version()) - files_to_upload = _get_media_sync_files(status_pull["added"]+status_pull["updated"]) + files_to_upload = _get_media_sync_files( + status_pull["added"] + status_pull["updated"] + ) return files_to_upload def _update_references(files): - """ Update references to media files in reference table """ + """Update references to media files in reference table""" for ref in config.references: - reference_config = [ref.file, ref.table, ref.local_path_column, ref.driver_path_column] + reference_config = [ + ref.file, + ref.table, + ref.local_path_column, + ref.driver_path_column, + ] if not all(reference_config): return print("Updating references ...") try: - gpkg_conn = sqlite3.connect(os.path.join(config.project_working_dir, ref.file)) + gpkg_conn = sqlite3.connect( + os.path.join(config.project_working_dir, ref.file) + ) gpkg_conn.enable_load_extension(True) gpkg_cur = gpkg_conn.cursor() gpkg_cur.execute('SELECT load_extension("mod_spatialite")') for file_path, dest in files.items(): # remove reference to the local path only in the move mode if config.operation_mode == "move": - sql = f"UPDATE {_quote_identifier(ref.table)} " \ - f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column, {_quote_identifier(ref.local_path_column)}=Null " \ - f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path" + sql = ( + f"UPDATE {_quote_identifier(ref.table)} " + f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column, {_quote_identifier(ref.local_path_column)}=Null " + f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path" + ) elif config.operation_mode == "copy": - sql = f"UPDATE {_quote_identifier(ref.table)} " \ - f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column " \ - f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path" + sql = ( + f"UPDATE {_quote_identifier(ref.table)} " + f"SET {_quote_identifier(ref.driver_path_column)}=:dest_column " + f"WHERE {_quote_identifier(ref.local_path_column)}=:file_path" + ) gpkg_cur.execute(sql, {"dest_column": dest, "file_path": file_path}) gpkg_conn.commit() gpkg_conn.close() @@ -172,7 +206,7 @@ def media_sync_push(mc, driver, files): size = os.path.getsize(src) / 1024 / 1024 # file size in MB print(f"Uploading {file['path']} of size {size:.2f} MB") dest = driver.upload_file(src, file["path"]) - migrated_files[file['path']] = dest + migrated_files[file["path"]] = dest except DriverError as e: print(f"Failed to upload {file['path']}: " + str(e)) continue @@ -191,7 +225,9 @@ def media_sync_push(mc, driver, files): mp = MerginProject(config.project_working_dir) status_push = mp.get_push_changes() if status_push["added"]: - raise MediaSyncError("There are changes to be added - it should never happen") + raise MediaSyncError( + "There are changes to be added - it should never happen" + ) if status_push["updated"] or status_push["removed"]: mc.push_project(config.project_working_dir) version = _get_project_version() @@ -238,5 +274,5 @@ def main(): print("Error: " + str(err)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/media_sync_daemon.py b/media_sync_daemon.py index b66909d..b745cdc 100644 --- a/media_sync_daemon.py +++ b/media_sync_daemon.py @@ -10,7 +10,13 @@ import os import time from drivers import DriverError, create_driver -from media_sync import create_mergin_client, mc_download, media_sync_push, mc_pull, MediaSyncError +from media_sync import ( + create_mergin_client, + mc_download, + media_sync_push, + mc_pull, + MediaSyncError, +) from config import config, validate_config, ConfigError from version import __version__ @@ -53,7 +59,9 @@ def main(): media_sync_push(mc, driver, files_to_sync) # check mergin client token expiration - delta = mc._auth_session['expire'] - datetime.datetime.now(datetime.timezone.utc) + delta = mc._auth_session["expire"] - datetime.datetime.now( + datetime.timezone.utc + ) if delta.total_seconds() < 3600: mc = create_mergin_client() @@ -64,5 +72,5 @@ def main(): time.sleep(sleep_time) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/test/conftest.py b/test/conftest.py index b6a58ac..5f4acb6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -44,7 +44,7 @@ def setup_config(): "MINIO__BUCKET": "", "MINIO__BUCKET_SUBPATH": "", "MINIO__SECURE": False, - "MINIO__REGION": "" + "MINIO__REGION": "", } ) diff --git a/test/test_config.py b/test/test_config.py index b3ed3c2..be28b62 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -9,27 +9,43 @@ from config import config, ConfigError, validate_config -from .conftest import SERVER_URL, API_USER, USER_PWD, MINIO_URL, MINIO_SECRET_KEY, MINIO_ACCESS_KEY +from .conftest import ( + SERVER_URL, + API_USER, + USER_PWD, + MINIO_URL, + MINIO_SECRET_KEY, + MINIO_ACCESS_KEY, +) def _reset_config(): - """ helper to reset config settings to ensure valid config """ - config.update({ - 'ALLOWED_EXTENSIONS': ["png"], - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'MERGIN__PROJECT_NAME': "test/mediasync", - 'PROJECT_WORKING_DIR': "/tmp/working_project", - 'OPERATION_MODE': "copy", - 'DRIVER': "minio", - 'MINIO__ENDPOINT': MINIO_URL, - 'MINIO__ACCESS_KEY': MINIO_ACCESS_KEY, - 'MINIO__SECRET_KEY': MINIO_SECRET_KEY, - 'MINIO__BUCKET': 'test', - 'REFERENCES': [{"file": "survey.gpkg", "table": "table", "local_path_column": "local_path_column", "driver_path_column": "driver_path_column"}], - "BASE_PATH": "" - }) + """helper to reset config settings to ensure valid config""" + config.update( + { + "ALLOWED_EXTENSIONS": ["png"], + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "MERGIN__PROJECT_NAME": "test/mediasync", + "PROJECT_WORKING_DIR": "/tmp/working_project", + "OPERATION_MODE": "copy", + "DRIVER": "minio", + "MINIO__ENDPOINT": MINIO_URL, + "MINIO__ACCESS_KEY": MINIO_ACCESS_KEY, + "MINIO__SECRET_KEY": MINIO_SECRET_KEY, + "MINIO__BUCKET": "test", + "REFERENCES": [ + { + "file": "survey.gpkg", + "table": "table", + "local_path_column": "local_path_column", + "driver_path_column": "driver_path_column", + } + ], + "BASE_PATH": "", + } + ) def test_config(): @@ -38,41 +54,50 @@ def test_config(): validate_config(config) with pytest.raises(ConfigError, match="Config error: Incorrect mergin settings"): - config.update({'MERGIN__USERNAME': None}) + config.update({"MERGIN__USERNAME": None}) validate_config(config) _reset_config() with pytest.raises(ConfigError, match="Config error: Unsupported driver"): - config.update({'DRIVER': None}) + config.update({"DRIVER": None}) validate_config(config) _reset_config() - with pytest.raises(ConfigError, match="Config error: Incorrect Local driver settings"): - config.update({'DRIVER': "local", "LOCAL__DEST": None}) + with pytest.raises( + ConfigError, match="Config error: Incorrect Local driver settings" + ): + config.update({"DRIVER": "local", "LOCAL__DEST": None}) validate_config(config) _reset_config() - with pytest.raises(ConfigError, match="Config error: Incorrect MinIO driver settings"): - config.update({'DRIVER': "minio", "MINIO__ENDPOINT": None}) + with pytest.raises( + ConfigError, match="Config error: Incorrect MinIO driver settings" + ): + config.update({"DRIVER": "minio", "MINIO__ENDPOINT": None}) validate_config(config) _reset_config() - with pytest.raises(ConfigError, match="Config error: Allowed extensions can not be empty"): - config.update({'ALLOWED_EXTENSIONS': []}) + with pytest.raises( + ConfigError, match="Config error: Allowed extensions can not be empty" + ): + config.update({"ALLOWED_EXTENSIONS": []}) validate_config(config) _reset_config() - with pytest.raises(ConfigError, match="Config error: References list can not be empty"): - config.update({'REFERENCES': []}) + with pytest.raises( + ConfigError, match="Config error: References list can not be empty" + ): + config.update({"REFERENCES": []}) validate_config(config) _reset_config() - with pytest.raises(ConfigError, match="Config error: Incorrect media reference settings"): - config.update({'REFERENCES': [{"file": "survey.gpkg"}]}) + with pytest.raises( + ConfigError, match="Config error: Incorrect media reference settings" + ): + config.update({"REFERENCES": [{"file": "survey.gpkg"}]}) validate_config(config) _reset_config() with pytest.raises(ConfigError, match="Config error: Unsupported operation mode"): - config.update({'OPERATION_MODE': ""}) + config.update({"OPERATION_MODE": ""}) validate_config(config) - diff --git a/test/test_sync.py b/test/test_sync.py index 9146f64..c021940 100644 --- a/test/test_sync.py +++ b/test/test_sync.py @@ -12,13 +12,32 @@ import sqlite3 from drivers import MinioDriver, LocalDriver -from media_sync import main, config, mc_pull, media_sync_push, mc_download, MediaSyncError - -from .conftest import API_USER, WORKSPACE, TMP_DIR, USER_PWD, USER_PWD, SERVER_URL,MINIO_URL,MINIO_ACCESS_KEY,MINIO_SECRET_KEY, cleanup, prepare_mergin_project +from media_sync import ( + main, + config, + mc_pull, + media_sync_push, + mc_download, + MediaSyncError, +) + +from .conftest import ( + API_USER, + WORKSPACE, + TMP_DIR, + USER_PWD, + USER_PWD, + SERVER_URL, + MINIO_URL, + MINIO_ACCESS_KEY, + MINIO_SECRET_KEY, + cleanup, + prepare_mergin_project, +) def test_sync(mc): - """ Test media sync starting from fresh project and download and then following scenarios + """Test media sync starting from fresh project and download and then following scenarios - simple full copy of all images without reference -> mergin server is not aware of any changes - copy only .jpg images -> .png file is ignored @@ -28,25 +47,38 @@ def test_sync(mc): """ project_name = "mediasync_test" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + '_work') # working dir for mediasync - driver_dir = os.path.join(TMP_DIR, project_name + '_driver') # destination dir for 'local' driver + work_project_dir = os.path.join( + TMP_DIR, project_name + "_work" + ) # working dir for mediasync + driver_dir = os.path.join( + TMP_DIR, project_name + "_driver" + ) # destination dir for 'local' driver cleanup(mc, full_project_name, [work_project_dir, driver_dir]) - prepare_mergin_project(mc, full_project_name) + prepare_mergin_project(mc, full_project_name) # patch config to fit testing purposes - config.update({ - 'ALLOWED_EXTENSIONS': ["png","jpg"], - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'MERGIN__PROJECT_NAME': full_project_name, - 'PROJECT_WORKING_DIR': work_project_dir, - 'DRIVER': "local", - 'LOCAL__DEST': driver_dir, - 'OPERATION_MODE': "copy", - 'REFERENCES': [{"file": None, "table": None, "local_path_column": None, "driver_path_column": None}], - }) + config.update( + { + "ALLOWED_EXTENSIONS": ["png", "jpg"], + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "MERGIN__PROJECT_NAME": full_project_name, + "PROJECT_WORKING_DIR": work_project_dir, + "DRIVER": "local", + "LOCAL__DEST": driver_dir, + "OPERATION_MODE": "copy", + "REFERENCES": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ) main() # check synced images @@ -74,9 +106,18 @@ def test_sync(mc): shutil.rmtree(driver_dir) # enable references updates - config.update({ - 'REFERENCES': [{"file": "survey.gpkg", "table": "notes", "local_path_column": "photo", "driver_path_column": "ext_url"}] - }) + config.update( + { + "REFERENCES": [ + { + "file": "survey.gpkg", + "table": "notes", + "local_path_column": "photo", + "driver_path_column": "ext_url", + } + ] + } + ) main() # check synced images @@ -86,7 +127,9 @@ def test_sync(mc): # check references have been changed and pushed project_info = mc.project_info(full_project_name) assert project_info["version"] == "v2" - gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, config.references[0].file)) + gpkg_conn = sqlite3.connect( + os.path.join(work_project_dir, config.references[0].file) + ) gpkg_cur = gpkg_conn.cursor() sql = f"SELECT count(*) FROM {config.references[0].table} WHERE {config.references[0].driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) @@ -95,9 +138,7 @@ def test_sync(mc): shutil.rmtree(driver_dir) # change to 'move' mode - config.update({ - 'OPERATION_MODE': "move" - }) + config.update({"OPERATION_MODE": "move"}) main() # check synced images @@ -107,9 +148,13 @@ def test_sync(mc): assert not os.path.exists(os.path.join(work_project_dir, "images", "img2.jpg")) project_info = mc.project_info(full_project_name) assert project_info["version"] == "v3" - moved_file = next((f for f in project_info["files"] if f["path"] == "images/img2.jpg"), None) + moved_file = next( + (f for f in project_info["files"] if f["path"] == "images/img2.jpg"), None + ) assert not moved_file - gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, config.references[0].file)) + gpkg_conn = sqlite3.connect( + os.path.join(work_project_dir, config.references[0].file) + ) gpkg_cur = gpkg_conn.cursor() sql = f"SELECT count(*) FROM {config.references[0].table} WHERE {config.references[0].local_path_column} is Null AND {config.references[0].driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) @@ -128,38 +173,49 @@ def test_sync(mc): def test_pull_and_sync(mc): - """ Test media sync if mergin project is ahead of locally downloaded one """ + """Test media sync if mergin project is ahead of locally downloaded one""" project_name = "mediasync_test_pull" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + '_work') - driver_dir = os.path.join(TMP_DIR, project_name + '_driver') + work_project_dir = os.path.join(TMP_DIR, project_name + "_work") + driver_dir = os.path.join(TMP_DIR, project_name + "_driver") cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) - config.update({ - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'MERGIN__PROJECT_NAME': full_project_name, - 'PROJECT_WORKING_DIR': work_project_dir, - 'DRIVER': "local", - 'LOCAL__DEST': driver_dir, - 'OPERATION_MODE': "copy", - "ALLOWED_EXTENSIONS": ["png","jpg"], - "BASE_PATH": None, - 'REFERENCES': [{"file": None, "table": None, "local_path_column": None, "driver_path_column": None}], - }) + config.update( + { + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "MERGIN__PROJECT_NAME": full_project_name, + "PROJECT_WORKING_DIR": work_project_dir, + "DRIVER": "local", + "LOCAL__DEST": driver_dir, + "OPERATION_MODE": "copy", + "ALLOWED_EXTENSIONS": ["png", "jpg"], + "BASE_PATH": None, + "REFERENCES": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ) # initial run main() # let's update project on server - create new .png file and modify reference .gpkg file - project_dir = os.path.join(TMP_DIR, project_name + '_create') + project_dir = os.path.join(TMP_DIR, project_name + "_create") if os.path.exists(project_dir): shutil.rmtree(project_dir) shutil.copytree(work_project_dir, project_dir) - shutil.copyfile(os.path.join(project_dir, 'img1.png'), os.path.join(project_dir, 'img_new.png')) - gpkg_conn = sqlite3.connect(os.path.join(project_dir, 'survey.gpkg')) + shutil.copyfile( + os.path.join(project_dir, "img1.png"), os.path.join(project_dir, "img_new.png") + ) + gpkg_conn = sqlite3.connect(os.path.join(project_dir, "survey.gpkg")) gpkg_conn.enable_load_extension(True) gpkg_cur = gpkg_conn.cursor() gpkg_cur.execute('SELECT load_extension("mod_spatialite")') @@ -177,34 +233,45 @@ def test_pull_and_sync(mc): def test_minio_backend(mc): - """ Test media sync connected to minio backend (needs local service running) """ + """Test media sync connected to minio backend (needs local service running)""" project_name = "mediasync_test_minio" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + '_work') + work_project_dir = os.path.join(TMP_DIR, project_name + "_work") cleanup(mc, full_project_name, [work_project_dir]) prepare_mergin_project(mc, full_project_name) # patch config to fit testing purposes - config.update({ - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'MERGIN__PROJECT_NAME': full_project_name, - 'PROJECT_WORKING_DIR': work_project_dir, - 'OPERATION_MODE': "copy", - 'REFERENCES': [{"file": None, "table": None, "local_path_column": None, "driver_path_column": None}], - 'DRIVER': "minio", - 'MINIO__ENDPOINT': MINIO_URL, - 'MINIO__ACCESS_KEY': MINIO_ACCESS_KEY, - 'MINIO__SECRET_KEY': MINIO_SECRET_KEY, - 'MINIO__BUCKET': 'test' - }) + config.update( + { + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "MERGIN__PROJECT_NAME": full_project_name, + "PROJECT_WORKING_DIR": work_project_dir, + "OPERATION_MODE": "copy", + "REFERENCES": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + "DRIVER": "minio", + "MINIO__ENDPOINT": MINIO_URL, + "MINIO__ACCESS_KEY": MINIO_ACCESS_KEY, + "MINIO__SECRET_KEY": MINIO_SECRET_KEY, + "MINIO__BUCKET": "test", + } + ) main() # check synced images driver = MinioDriver(config) - minio_objects = [o.object_name for o in driver.client.list_objects('test', recursive=True)] + minio_objects = [ + o.object_name for o in driver.client.list_objects("test", recursive=True) + ] assert "img1.png" in minio_objects assert "images/img2.jpg" in minio_objects @@ -223,55 +290,86 @@ def test_minio_backend(mc): "MERGIN__PROJECT_NAME": full_project_name, "PROJECT_WORKING_DIR": work_project_dir, "OPERATION_MODE": "copy", - "REFERENCES": [{"file": None, "table": None, "local_path_column": None, "driver_path_column": None}], + "REFERENCES": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], "DRIVER": "minio", "MINIO__ENDPOINT": MINIO_URL, "MINIO__ACCESS_KEY": MINIO_ACCESS_KEY, "MINIO__SECRET_KEY": MINIO_SECRET_KEY, "MINIO__BUCKET": "test1", - "MINIO__BUCKET_SUBPATH": "subPath" + "MINIO__BUCKET_SUBPATH": "subPath", } ) main() # check synced images driver = MinioDriver(config) - minio_objects = [o.object_name for o in driver.client.list_objects("test1", recursive=True)] + minio_objects = [ + o.object_name for o in driver.client.list_objects("test1", recursive=True) + ] assert "subPath/img1.png" in minio_objects assert "subPath/images/img2.jpg" in minio_objects def test_sync_failures(mc): - """ Test common sync failures """ + """Test common sync failures""" project_name = "mediasync_fail" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + '_work') # working dir for mediasync - driver_dir = os.path.join(TMP_DIR, project_name + '_driver') # destination dir for 'local' driver + work_project_dir = os.path.join( + TMP_DIR, project_name + "_work" + ) # working dir for mediasync + driver_dir = os.path.join( + TMP_DIR, project_name + "_driver" + ) # destination dir for 'local' driver cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) - - config.update({ - 'ALLOWED_EXTENSIONS': ["png","jpg"], - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'MERGIN__PROJECT_NAME': full_project_name, - 'PROJECT_WORKING_DIR': work_project_dir, - 'DRIVER': "local", - 'LOCAL__DEST': driver_dir, - 'OPERATION_MODE': "copy", - 'REFERENCES': [{"file": None, "table": None, "local_path_column": None, "driver_path_column": None}], - }) + + config.update( + { + "ALLOWED_EXTENSIONS": ["png", "jpg"], + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "MERGIN__PROJECT_NAME": full_project_name, + "PROJECT_WORKING_DIR": work_project_dir, + "DRIVER": "local", + "LOCAL__DEST": driver_dir, + "OPERATION_MODE": "copy", + "REFERENCES": [ + { + "file": None, + "table": None, + "local_path_column": None, + "driver_path_column": None, + } + ], + } + ) driver = LocalDriver(config) files_to_sync = mc_download(mc) # "remove" .mergin hidden dir to mimic broken working directory - os.rename(os.path.join(work_project_dir, '.mergin'), os.path.join(work_project_dir, '.hidden')) + os.rename( + os.path.join(work_project_dir, ".mergin"), + os.path.join(work_project_dir, ".hidden"), + ) with pytest.raises(MediaSyncError) as exc: media_sync_push(mc, driver, files_to_sync) - assert "The project working directory does not seem to contain Mergin project" in exc.value.args[0] - os.rename(os.path.join(work_project_dir, '.hidden'), os.path.join(work_project_dir, '.mergin')) + assert ( + "The project working directory does not seem to contain Mergin project" + in exc.value.args[0] + ) + os.rename( + os.path.join(work_project_dir, ".hidden"), + os.path.join(work_project_dir, ".mergin"), + ) # "remove" working project os.rename(work_project_dir, work_project_dir + "_renamed") @@ -281,18 +379,39 @@ def test_sync_failures(mc): os.rename(work_project_dir + "_renamed", work_project_dir) # incorrect gpkg details for reference table - config.update({ - 'REFERENCES': [{"file": "survey.gpkg", "table": "notes_error", "local_path_column": "photo", "driver_path_column": "ext_url"}], - }) + config.update( + { + "REFERENCES": [ + { + "file": "survey.gpkg", + "table": "notes_error", + "local_path_column": "photo", + "driver_path_column": "ext_url", + } + ], + } + ) with pytest.raises(MediaSyncError) as exc: media_sync_push(mc, driver, files_to_sync) assert "SQLITE error" in exc.value.args[0] - config.update({ - 'REFERENCES': [{"file": "survey.gpkg", "table": "notes", "local_path_column": "photo", "driver_path_column": "ext_url"}], - }) + config.update( + { + "REFERENCES": [ + { + "file": "survey.gpkg", + "table": "notes", + "local_path_column": "photo", + "driver_path_column": "ext_url", + } + ], + } + ) # introduce some unknown local file - shutil.copyfile(os.path.join(work_project_dir, "survey.gpkg"), os.path.join(work_project_dir, "new.gpkg")) + shutil.copyfile( + os.path.join(work_project_dir, "survey.gpkg"), + os.path.join(work_project_dir, "new.gpkg"), + ) with pytest.raises(MediaSyncError) as exc: media_sync_push(mc, driver, files_to_sync) assert "There are changes to be added - it should never happen" in exc.value.args[0] @@ -306,29 +425,48 @@ def test_sync_failures(mc): assert not os.path.exists(os.path.join(driver_dir, "img1.png")) assert os.path.exists(os.path.join(driver_dir, "images", "img2.jpg")) + def test_multiple_tables(mc): project_name = "mediasync_test_multiple" full_project_name = WORKSPACE + "/" + project_name - work_project_dir = os.path.join(TMP_DIR, project_name + '_work') # working dir for mediasync - driver_dir = os.path.join(TMP_DIR, project_name + '_driver') # destination dir for 'local' driver + work_project_dir = os.path.join( + TMP_DIR, project_name + "_work" + ) # working dir for mediasync + driver_dir = os.path.join( + TMP_DIR, project_name + "_driver" + ) # destination dir for 'local' driver cleanup(mc, full_project_name, [work_project_dir, driver_dir]) prepare_mergin_project(mc, full_project_name) # patch config to fit testing purposes - config.update({ - 'ALLOWED_EXTENSIONS': ["png"], - 'MERGIN__USERNAME': API_USER, - 'MERGIN__PASSWORD': USER_PWD, - 'MERGIN__URL': SERVER_URL, - 'MERGIN__PROJECT_NAME': full_project_name, - 'PROJECT_WORKING_DIR': work_project_dir, - 'DRIVER': "local", - 'LOCAL__DEST': driver_dir, - 'OPERATION_MODE': "copy", - 'REFERENCES': [{"file": "survey.gpkg", "table": "notes", "local_path_column": "photo", "driver_path_column": "ext_url"}, - {"file": "survey.gpkg", "table": "photos", "local_path_column": "123_photo", "driver_path_column": "ext_url"}] - }) + config.update( + { + "ALLOWED_EXTENSIONS": ["png"], + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "MERGIN__PROJECT_NAME": full_project_name, + "PROJECT_WORKING_DIR": work_project_dir, + "DRIVER": "local", + "LOCAL__DEST": driver_dir, + "OPERATION_MODE": "copy", + "REFERENCES": [ + { + "file": "survey.gpkg", + "table": "notes", + "local_path_column": "photo", + "driver_path_column": "ext_url", + }, + { + "file": "survey.gpkg", + "table": "photos", + "local_path_column": "123_photo", + "driver_path_column": "ext_url", + }, + ], + } + ) main() # check synced images @@ -340,13 +478,17 @@ def test_multiple_tables(mc): copied_file = os.path.join(driver_dir, "img1.png") # check that both tables were updated - gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, config.references[0].file)) + gpkg_conn = sqlite3.connect( + os.path.join(work_project_dir, config.references[0].file) + ) gpkg_cur = gpkg_conn.cursor() sql = f"SELECT count(*) FROM {config.references[0].table} WHERE {config.references[0].driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) assert gpkg_cur.fetchone()[0] == 1 # check that both tables were updated - gpkg_conn = sqlite3.connect(os.path.join(work_project_dir, config.references[1].file)) + gpkg_conn = sqlite3.connect( + os.path.join(work_project_dir, config.references[1].file) + ) gpkg_cur = gpkg_conn.cursor() sql = f"SELECT count(*) FROM {config.references[1].table} WHERE {config.references[1].driver_path_column}='{copied_file}'" gpkg_cur.execute(sql) diff --git a/version.py b/version.py index 0404d81..493f741 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -__version__ = '0.3.0' +__version__ = "0.3.0"