diff --git a/.github/workflows/autotests.yml b/.github/workflows/autotests.yml index 0505896..8997abd 100644 --- a/.github/workflows/autotests.yml +++ b/.github/workflows/autotests.yml @@ -7,6 +7,10 @@ env: TEST_API_USERNAME2: test_plugin2 TEST_API_PASSWORD2: ${{ secrets.MERGINTEST_API_PASSWORD2 }} +concurrency: + group: ci-${{github.ref}}-autotests + cancel-in-progress: true + jobs: tests: runs-on: ubuntu-latest diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 1d86658..7e4bf94 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -1,6 +1,6 @@ name: Code Style -on: [push, pull_request] +on: [push] jobs: code_style_python: diff --git a/mergin/cli.py b/mergin/cli.py index b5691c8..58cf31b 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -31,10 +31,8 @@ download_project_finalize, download_project_is_running, ) -from mergin.client_pull import pull_project_async, pull_project_is_running, pull_project_finalize, \ - pull_project_cancel -from mergin.client_push import push_project_async, push_project_is_running, push_project_finalize, \ - push_project_cancel +from mergin.client_pull import pull_project_async, pull_project_is_running, pull_project_finalize, pull_project_cancel +from mergin.client_push import push_project_async, push_project_is_running, push_project_finalize, push_project_cancel from pygeodiff import GeoDiff @@ -143,7 +141,9 @@ def _print_unhandled_exception(): click.echo(line) -@click.group(epilog=f"Copyright (C) 2019-2021 Lutra Consulting\n\n(mergin-py-client v{__version__} / pygeodiff v{GeoDiff().version()})") +@click.group( + epilog=f"Copyright (C) 2019-2021 Lutra Consulting\n\n(mergin-py-client v{__version__} / pygeodiff v{GeoDiff().version()})" +) @click.option( "--url", envvar="MERGIN_URL", @@ -183,9 +183,12 @@ def login(ctx): @cli.command() @click.argument("project") @click.option("--public", is_flag=True, default=False, help="Public project, visible to everyone") -@click.option("--from-dir", default=None, - help="Content of the directory will be uploaded to the newly created project. " - "The directory will get assigned to the project.") +@click.option( + "--from-dir", + default=None, + help="Content of the directory will be uploaded to the newly created project. " + "The directory will get assigned to the project.", +) @click.pass_context def create(ctx, project, public, from_dir): """Create a new project on Mergin Maps server. `project` needs to be a combination of namespace/project.""" @@ -234,29 +237,24 @@ def create(ctx, project, public, from_dir): @click.option( "--order_params", help="optional attributes for sorting the list. " - "It should be a comma separated attribute names " - "with _asc or _desc appended for sorting direction. " - "For example: \"namespace_asc,disk_usage_desc\". " - "Available attrs: namespace, name, created, updated, disk_usage, creator", + "It should be a comma separated attribute names " + "with _asc or _desc appended for sorting direction. " + 'For example: "namespace_asc,disk_usage_desc". ' + "Available attrs: namespace, name, created, updated, disk_usage, creator", ) @click.pass_context def list_projects(ctx, flag, name, namespace, order_params): """List projects on the server.""" filter_str = "(filter flag={})".format(flag) if flag is not None else "(all public)" - + click.echo("List of projects {}:".format(filter_str)) mc = ctx.obj["client"] if mc is None: return - projects_list = mc.projects_list( - flag=flag, - name=name, - namespace=namespace, - order_params=order_params - ) - + projects_list = mc.projects_list(flag=flag, name=name, namespace=namespace, order_params=order_params) + click.echo("Fetched {} projects .".format(len(projects_list))) for project in projects_list: full_name = "{} / {}".format(project["namespace"], project["name"]) @@ -401,9 +399,12 @@ def status(ctx): return if mc.has_unfinished_pull(os.getcwd()): - click.secho("The previous pull has not finished completely: status " - "of some files may be reported incorrectly. Use " - "resolve_unfinished_pull command to try to fix that.", fg="yellow") + click.secho( + "The previous pull has not finished completely: status " + "of some files may be reported incorrectly. Use " + "resolve_unfinished_pull command to try to fix that.", + fg="yellow", + ) click.secho("### Server changes:", fg="magenta") pretty_diff(pull_changes) @@ -526,7 +527,7 @@ def show_file_history(ctx, path): @click.argument("version") @click.pass_context def show_file_changeset(ctx, path, version): - """ Displays information about project changes.""" + """Displays information about project changes.""" mc = ctx.obj["client"] if mc is None: return diff --git a/mergin/client.py b/mergin/client.py index 8a80834..4a497f7 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -42,7 +42,7 @@ def decode_token_data(token): if not token.startswith(token_prefix): raise TokenError(f"Token doesn't start with 'Bearer .': {token}") try: - data = token[len(token_prefix):].split('.')[0] + data = token[len(token_prefix) :].split(".")[0] # add proper base64 padding" data += "=" * (-len(data) % 4) decoded = zlib.decompress(base64.urlsafe_b64decode(data)) @@ -64,13 +64,14 @@ class MerginClient: of the server should be provided. Expected keys: "url", "port", "user", "password". Currently, only HTTP proxies are supported. """ + def __init__(self, url=None, auth_token=None, login=None, password=None, plugin_version=None, proxy_config=None): self.url = url if url is not None else MerginClient.default_url() self._auth_params = None self._auth_session = None self._user_info = None self.client_version = "Python-client/" + __version__ - if plugin_version is not None: # this could be e.g. "Plugin/2020.1 QGIS/3.14" + if plugin_version is not None: # this could be e.g. "Plugin/2020.1 QGIS/3.14" self.client_version += " " + plugin_version self.setup_logging() if auth_token: @@ -90,7 +91,9 @@ def __init__(self, url=None, auth_token=None, login=None, password=None, plugin_ if proxy_config["user"] and proxy_config["password"]: handlers.append( urllib.request.ProxyHandler( - {"https": f"{proxy_config['user']}:{proxy_config['password']}@{proxy_url}:{proxy_config['port']}"} + { + "https": f"{proxy_config['user']}:{proxy_config['password']}@{proxy_url}:{proxy_config['port']}" + } ) ) handlers.append(urllib.request.HTTPBasicAuthHandler()) @@ -105,7 +108,7 @@ def __init__(self, url=None, auth_token=None, login=None, password=None, plugin_ if os.path.exists(default_capath): self.opener = urllib.request.build_opener(*handlers, urllib.request.HTTPSHandler()) else: - cafile = os.path.join(this_dir, 'cert.pem') + cafile = os.path.join(this_dir, "cert.pem") if not os.path.exists(cafile): raise Exception("missing " + cafile) ctx = ssl.SSLContext() @@ -120,22 +123,19 @@ def __init__(self, url=None, auth_token=None, login=None, password=None, plugin_ raise ClientError("Unable to log in: password provided but no username/email") if login and password: - self._auth_params = { - "login": login, - "password": password - } + self._auth_params = {"login": login, "password": password} if not self._auth_session: self.login(login, password) def setup_logging(self): """Setup Mergin Maps client logging.""" - client_log_file = os.environ.get('MERGIN_CLIENT_LOG', None) - self.log = logging.getLogger('mergin.client') + client_log_file = os.environ.get("MERGIN_CLIENT_LOG", None) + self.log = logging.getLogger("mergin.client") self.log.setLevel(logging.DEBUG) # log everything (it would otherwise log just warnings+errors) if not self.log.handlers: if client_log_file: log_handler = logging.FileHandler(client_log_file) - log_handler.setFormatter(logging.Formatter('%(asctime)s %(message)s')) + log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) self.log.addHandler(log_handler) else: # no Mergin Maps log path in the environment - create a null handler that does nothing @@ -144,15 +144,16 @@ def setup_logging(self): @staticmethod def default_url(): - """ Returns URL of the public instance of Mergin Maps""" + """Returns URL of the public instance of Mergin Maps""" return "https://app.merginmaps.com" def user_agent_info(self): - """ Returns string as it is sent as User-Agent http header to the server """ + """Returns string as it is sent as User-Agent http header to the server""" system_version = "Unknown" if platform.system() == "Linux": try: from pip._vendor import distro + system_version = distro.linux_distribution()[0] except ModuleNotFoundError: # pip may not be installed... system_version = "Linux" @@ -164,6 +165,7 @@ def user_agent_info(self): def _check_token(f): """Wrapper for creating/renewing authorization token.""" + def wrapper(self, *args): if self._auth_params: if self._auth_session: @@ -175,8 +177,9 @@ def wrapper(self, *args): else: # Create a new authorization token self.log.info(f"No token - login user: {self._auth_params['login']}") - self.login(self._auth_params['login'], self._auth_params['password']) + self.login(self._auth_params["login"], self._auth_params["password"]) return f(self, *args) + return wrapper @_check_token @@ -219,9 +222,9 @@ def is_server_compatible(self): """ resp = self.get("/ping") data = json.load(resp) - if 'endpoints' not in data: + if "endpoints" not in data: return False - endpoints = data['endpoints'] + endpoints = data["endpoints"] client_endpoints = { "data_sync": { @@ -230,21 +233,19 @@ def is_server_compatible(self): "/project/push/cancel/{transaction_id}", "/project/push/finish/{transaction_id}", "/project/push/{namespace}/{project_name}", - # "/project/push/chunk/{transaction_id}/{chunk_id}" # issue in server - ] + # "/project/push/chunk/{transaction_id}/{chunk_id}" # issue in server + ], }, "project": { "DELETE": ["/project/{namespace}/{project_name}"], "GET": [ "/project", "/project/{namespace}/{project_name}", - "/project/version/{namespace}/{project_name}" + "/project/version/{namespace}/{project_name}", ], - "POST": ["/project/{namespace}"] + "POST": ["/project/{namespace}"], }, - "user": { - "POST": ["/auth/login"] - } + "user": {"POST": ["/auth/login"]}, } for k, v in client_endpoints.items(): @@ -268,10 +269,7 @@ def login(self, login, password): :param password: User's password :type password: String """ - params = { - "login": login, - "password": password - } + params = {"login": login, "password": password} self._auth_session = None self.log.info(f"Going to log in user {login}") try: @@ -295,16 +293,14 @@ def login(self, login, password): raise ClientError("failure reason: " + str(e.reason)) self._auth_session = { "token": "Bearer %s" % session["token"], - "expire": dateutil.parser.parse(session["expire"]) - } - self._user_info = { - "username": data["username"] + "expire": dateutil.parser.parse(session["expire"]), } + self._user_info = {"username": data["username"]} self.log.info(f"User {data['username']} successfully logged in.") return session def username(self): - """ Returns user name used in this session or None if not authenticated """ + """Returns user name used in this session or None if not authenticated""" if not self._user_info: return None # not authenticated return self._user_info["username"] @@ -343,10 +339,7 @@ def create_project(self, project_name, is_public=False, namespace=None): if not self._user_info: raise Exception("Authentication required") - params = { - "name": project_name, - "public": is_public - } + params = {"name": project_name, "public": is_public} if namespace is None: namespace = self.username() try: @@ -359,8 +352,8 @@ def create_project_and_push(self, project_name, directory, is_public=False, name """ Convenience method to create project and push the initial version right after that. """ - if os.path.exists(os.path.join(directory, '.mergin')): - raise ClientError('Directory is already assigned to a Mergin Maps project (contains .mergin sub-dir)') + if os.path.exists(os.path.join(directory, ".mergin")): + raise ClientError("Directory is already assigned to a Mergin Maps project (contains .mergin sub-dir)") if namespace is None: namespace = self.username() @@ -372,8 +365,9 @@ def create_project_and_push(self, project_name, directory, is_public=False, name if mp.inspect_files(): self.push_project(directory) - def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, flag=None, name=None, - namespace=None, order_params=None): + def paginated_projects_list( + self, page=1, per_page=50, tags=None, user=None, flag=None, name=None, namespace=None, order_params=None + ): """ Find all available Mergin Maps projects. @@ -426,9 +420,9 @@ def paginated_projects_list(self, page=1, per_page=50, tags=None, user=None, fla def projects_list(self, tags=None, user=None, flag=None, name=None, namespace=None, order_params=None): """ - Find all available Mergin Maps projects. - - Calls paginated_projects_list for all pages. Can take significant time to fetch all pages. + Find all available Mergin Maps projects. + + Calls paginated_projects_list for all pages. Can take significant time to fetch all pages. :param tags: Filter projects by tags ('valid_qgis', 'mappin_use', input_use') :type tags: List @@ -459,12 +453,12 @@ def projects_list(self, tags=None, user=None, flag=None, name=None, namespace=No resp = self.paginated_projects_list( page=page_i, per_page=50, - tags=tags, - user=user, - flag=flag, + tags=tags, + user=user, + flag=flag, name=name, - namespace=namespace, - order_params=order_params + namespace=namespace, + order_params=order_params, ) fetched_projects += len(resp["projects"]) count = resp["count"] @@ -487,10 +481,10 @@ def project_info(self, project_path, since=None, version=None): :type version: String :rtype: Dict """ - params = {'since': since} if since else {} + params = {"since": since} if since else {} # since and version are mutually exclusive if version: - params = {'version': version} + params = {"version": version} resp = self.get("/v1/project/{}".format(project_path), params) return json.load(resp) @@ -515,26 +509,18 @@ def project_versions(self, project_path, since=None, to=None): start_page = math.ceil(num_since / per_page) if not num_to: # let's get first page and count - params = { - "page": start_page, - "per_page": per_page, - "descending": False - } + params = {"page": start_page, "per_page": per_page, "descending": False} resp = self.get("/v1/project/versions/paginated/{}".format(project_path), params) resp_json = json.load(resp) versions = resp_json["versions"] - num_to = resp_json['count'] + num_to = resp_json["count"] latest_version = int_version(versions[-1]["name"]) if latest_version < num_to: versions += self.project_versions(project_path, f"v{latest_version+1}", f"v{num_to}") else: end_page = math.ceil(num_to / per_page) - for page in range(start_page, end_page+1): - params = { - "page": page, - "per_page": per_page, - "descending": False - } + for page in range(start_page, end_page + 1): + params = {"page": page, "per_page": per_page, "descending": False} resp = self.get("/v1/project/versions/paginated/{}".format(project_path), params) versions += json.load(resp)["versions"] @@ -571,7 +557,7 @@ def enough_storage_available(self, data): return True, free_space def user_info(self): - resp = self.get('/v1/user/' + self.username()) + resp = self.get("/v1/user/" + self.username()) return json.load(resp) def set_project_access(self, project_path, access): @@ -586,7 +572,7 @@ def set_project_access(self, project_path, access): params = {"access": access} path = "/v1/project/%s" % project_path url = urllib.parse.urljoin(self.url, urllib.parse.quote(path)) - json_headers = {'Content-Type': 'application/json'} + json_headers = {"Content-Type": "application/json"} try: request = urllib.request.Request(url, data=json.dumps(params).encode(), headers=json_headers, method="PUT") self._do_request(request) @@ -605,7 +591,7 @@ def add_user_permissions_to_project(self, project_path, usernames, permission_le raise ClientError("Unsupported permission level") project_info = self.project_info(project_path) - access = project_info.get('access') + access = project_info.get("access") for name in usernames: if permission_level == "owner": access.get("ownersnames").append(name) @@ -617,12 +603,12 @@ def add_user_permissions_to_project(self, project_path, usernames, permission_le def remove_user_permissions_from_project(self, project_path, usernames): """ - Removes specified users from project - :param project_path: project full name (/) - :param usernames: list of usernames to be granted specified permission level - """ + Removes specified users from project + :param project_path: project full name (/) + :param usernames: list of usernames to be granted specified permission level + """ project_info = self.project_info(project_path) - access = project_info.get('access') + access = project_info.get("access") for name in usernames: if name in access.get("ownersnames"): access.get("ownersnames").remove(name) @@ -634,14 +620,14 @@ def remove_user_permissions_from_project(self, project_path, usernames): def project_user_permissions(self, project_path): """ - Returns permissions for project - :param project_path: project full name (/) - :return dict("owners": list(usernames), - "writers": list(usernames), - "readers": list(usernames)) - """ + Returns permissions for project + :param project_path: project full name (/) + :return dict("owners": list(usernames), + "writers": list(usernames), + "readers": list(usernames)) + """ project_info = self.project_info(project_path) - access = project_info.get('access') + access = project_info.get("access") result = {} result["owners"] = access.get("ownersnames") result["writers"] = access.get("writersnames") @@ -657,7 +643,7 @@ def push_project(self, directory): """ job = push_project_async(self, directory) if job is None: - return # there is nothing to push (or we only deleted some files) + return # there is nothing to push (or we only deleted some files) push_project_wait(job) push_project_finalize(job) @@ -670,7 +656,7 @@ def pull_project(self, directory): """ job = pull_project_async(self, directory) if job is None: - return # project is up to date + return # project is up to date pull_project_wait(job) return pull_project_finalize(job) @@ -687,10 +673,10 @@ def clone_project(self, source_project_path, cloned_project_name, cloned_project """ path = "/v1/project/clone/%s" % source_project_path url = urllib.parse.urljoin(self.url, urllib.parse.quote(path)) - json_headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} + json_headers = {"Content-Type": "application/json", "Accept": "application/json"} data = { - 'namespace': cloned_project_namespace if cloned_project_namespace else self.username(), - 'project': cloned_project_name + "namespace": cloned_project_namespace if cloned_project_namespace else self.username(), + "project": cloned_project_name, } request = urllib.request.Request(url, data=json.dumps(data).encode(), headers=json_headers, method="POST") @@ -730,25 +716,25 @@ def project_status(self, directory): return pull_changes, push_changes, push_changes_summary def project_version_info(self, project_path, version): - """ Returns JSON with detailed information about a single project version""" - params = {'version_id': version} + """Returns JSON with detailed information about a single project version""" + params = {"version_id": version} resp = self.get("/v1/project/version/{}".format(project_path), params) return json.load(resp) def project_file_history_info(self, project_path, file_path): - """ Returns JSON with full history of a single file within a project """ - params = {'path': file_path} + """Returns JSON with full history of a single file within a project""" + params = {"path": file_path} resp = self.get("/v1/resource/history/{}".format(project_path), params) return json.load(resp) def project_file_changeset_info(self, project_path, file_path, version): - """ Returns JSON with changeset details of a particular version of a file within a project """ - params = {'path': file_path} + """Returns JSON with changeset details of a particular version of a file within a project""" + params = {"path": file_path} resp = self.get("/v1/resource/changesets/{}/{}".format(project_path, version), params) return json.load(resp) def get_projects_by_names(self, projects): - """ Returns JSON with projects' info for list of required projects. + """Returns JSON with projects' info for list of required projects. The schema of the returned information is the same as the response from projects_list(). This is useful when we have a couple of Mergin Maps projects available locally and we want to @@ -779,7 +765,7 @@ def download_file(self, project_dir, file_path, output_filename, version=None): download_file_finalize(job) def get_file_diff(self, project_dir, file_path, output_diff, version_from, version_to): - """ Create concatenated diff for project file diffs between versions version_from and version_to. + """Create concatenated diff for project file diffs between versions version_from and version_to. :param project_dir: project local directory :type project_dir: String @@ -815,7 +801,7 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi shutil.copy(diffs[0], output_diff) def download_file_diffs(self, project_dir, file_path, versions): - """ Download file diffs for specified versions if they are not present + """Download file diffs for specified versions if they are not present in the cache. :param project_dir: project local directory diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 025dfbe..728b800 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -41,47 +41,49 @@ class DownloadJob: Used for downloading whole projects but also single files. """ - def __init__(self, project_path, total_size, version, update_tasks, download_queue_items, directory, mp, project_info): + def __init__( + self, project_path, total_size, version, update_tasks, download_queue_items, directory, mp, project_info + ): self.project_path = project_path - self.total_size = total_size # size of data to download (in bytes) + self.total_size = total_size # size of data to download (in bytes) self.transferred_size = 0 self.version = version self.update_tasks = update_tasks self.download_queue_items = download_queue_items - self.directory = directory # project's directory - self.mp = mp # MerginProject instance + self.directory = directory # project's directory + self.mp = mp # MerginProject instance self.is_cancelled = False - self.project_info = project_info # parsed JSON with project info returned from the server + self.project_info = project_info # parsed JSON with project info returned from the server def dump(self): print("--- JOB ---", self.total_size, "bytes") for task in self.update_tasks: print("- {} ... {}".format(task.file_path, len(task.download_queue_items))) - print ("--") + print("--") for item in self.download_queue_items: print("- {} {} {} {}".format(item.file_path, item.version, item.part_index, item.size)) print("--- END ---") def _download_items(file, directory, diff_only=False): - """ Returns an array of download queue items """ + """Returns an array of download queue items""" - file_dir = os.path.dirname(os.path.normpath(os.path.join(directory, file['path']))) - basename = os.path.basename(file['diff']['path']) if diff_only else os.path.basename(file['path']) - file_size = file['diff']['size'] if diff_only else file['size'] + file_dir = os.path.dirname(os.path.normpath(os.path.join(directory, file["path"]))) + basename = os.path.basename(file["diff"]["path"]) if diff_only else os.path.basename(file["path"]) + file_size = file["diff"]["size"] if diff_only else file["size"] chunks = math.ceil(file_size / CHUNK_SIZE) items = [] for part_index in range(chunks): download_file_path = os.path.join(file_dir, basename + ".{}".format(part_index)) size = min(CHUNK_SIZE, file_size - part_index * CHUNK_SIZE) - items.append(DownloadQueueItem(file['path'], size, file['version'], diff_only, part_index, download_file_path)) + items.append(DownloadQueueItem(file["path"], size, file["version"], diff_only, part_index, download_file_path)) return items def _do_download(item, mc, mp, project_path, job): - """ runs in worker thread """ + """runs in worker thread""" if job.is_cancelled: return @@ -110,7 +112,7 @@ def download_project_async(mc, project_path, directory, project_version=None): Using that object it is possible to watch progress or cancel the ongoing work. """ - if '/' not in project_path: + if "/" not in project_path: raise ClientError("Project name needs to be fully qualified, e.g. /") if os.path.exists(directory): raise ClientError("Project directory already exists") @@ -132,17 +134,17 @@ def download_project_async(mc, project_path, directory, project_version=None): _cleanup_failed_download(directory, mp) raise - version = project_info['version'] if project_info['version'] else 'v0' + version = project_info["version"] if project_info["version"] else "v0" mp.log.info(f"got project info. version {version}") # prepare download update_tasks = [] # stuff to do at the end of download - for file in project_info['files']: - file['version'] = version + for file in project_info["files"]: + file["version"] = version items = _download_items(file, directory) is_latest_version = project_version == latest_proj_info["version"] - update_tasks.append(UpdateTask(file['path'], items, latest_version=is_latest_version)) + update_tasks.append(UpdateTask(file["path"], items, latest_version=is_latest_version)) # make a single list of items to download total_size = 0 @@ -167,7 +169,7 @@ def download_project_async(mc, project_path, directory, project_version=None): def download_project_wait(job): - """ blocks until all download tasks are finished """ + """blocks until all download tasks are finished""" concurrent.futures.wait(job.futures) @@ -215,11 +217,7 @@ def download_project_finalize(job): # final update of project metadata # TODO: why not exact copy of project info JSON ? - job.mp.metadata = { - "name": job.project_path, - "version": job.version, - "files": job.project_info["files"] - } + job.mp.metadata = {"name": job.project_path, "version": job.version, "files": job.project_info["files"]} def download_project_cancel(job): @@ -248,7 +246,7 @@ def __init__(self, file_path, download_queue_items, destination_file=None, lates self.latest_version = latest_version def apply(self, directory, mp): - """ assemble downloaded chunks into a single file """ + """assemble downloaded chunks into a single file""" if self.destination_file is None: basename = os.path.basename(self.file_path) @@ -272,56 +270,72 @@ def apply(self, directory, mp): class DownloadQueueItem: - """ a piece of data from a project that should be downloaded - it can be either a chunk or it can be a diff """ + """a piece of data from a project that should be downloaded - it can be either a chunk or it can be a diff""" def __init__(self, file_path, size, version, diff_only, part_index, download_file_path): - self.file_path = file_path # relative path to the file within project - self.size = size # size of the item in bytes - self.version = version # version of the file ("v123") - self.diff_only = diff_only # whether downloading diff or full version - self.part_index = part_index # index of the chunk - self.download_file_path = download_file_path # full path to a temporary file which will receive the content + self.file_path = file_path # relative path to the file within project + self.size = size # size of the item in bytes + self.version = version # version of the file ("v123") + self.diff_only = diff_only # whether downloading diff or full version + self.part_index = part_index # index of the chunk + self.download_file_path = download_file_path # full path to a temporary file which will receive the content def __repr__(self): return "".format( - self.file_path, self.version, self.diff_only, self.part_index, self.size, self.download_file_path) + self.file_path, self.version, self.diff_only, self.part_index, self.size, self.download_file_path + ) def download_blocking(self, mc, mp, project_path): - """ Starts download and only returns once the file has been fully downloaded and saved """ + """Starts download and only returns once the file has been fully downloaded and saved""" - mp.log.debug(f"Downloading {self.file_path} version={self.version} diff={self.diff_only} part={self.part_index}") + mp.log.debug( + f"Downloading {self.file_path} version={self.version} diff={self.diff_only} part={self.part_index}" + ) start = self.part_index * (1 + CHUNK_SIZE) - resp = mc.get("/v1/project/raw/{}".format(project_path), data={ - "file": self.file_path, - "version": self.version, - "diff": self.diff_only - }, headers={ - "Range": "bytes={}-{}".format(start, start + CHUNK_SIZE) - } + resp = mc.get( + "/v1/project/raw/{}".format(project_path), + data={"file": self.file_path, "version": self.version, "diff": self.diff_only}, + headers={"Range": "bytes={}-{}".format(start, start + CHUNK_SIZE)}, ) if resp.status in [200, 206]: mp.log.debug(f"Download finished: {self.file_path}") save_to_file(resp, self.download_file_path) else: mp.log.error(f"Download failed: {self.file_path}") - raise ClientError('Failed to download part {} of file {}'.format(self.part_index, self.file_path)) + raise ClientError("Failed to download part {} of file {}".format(self.part_index, self.file_path)) class PullJob: - def __init__(self, project_path, pull_changes, total_size, version, files_to_merge, download_queue_items, - temp_dir, mp, project_info, basefiles_to_patch, user_name): + def __init__( + self, + project_path, + pull_changes, + total_size, + version, + files_to_merge, + download_queue_items, + temp_dir, + mp, + project_info, + basefiles_to_patch, + user_name, + ): self.project_path = project_path - self.pull_changes = pull_changes # dictionary with changes (dict[str, list[dict]] - keys: "added", "updated", ...) - self.total_size = total_size # size of data to download (in bytes) + self.pull_changes = ( + pull_changes # dictionary with changes (dict[str, list[dict]] - keys: "added", "updated", ...) + ) + self.total_size = total_size # size of data to download (in bytes) self.transferred_size = 0 self.version = version - self.files_to_merge = files_to_merge # list of FileToMerge instances + self.files_to_merge = files_to_merge # list of FileToMerge instances self.download_queue_items = download_queue_items - self.temp_dir = temp_dir # full path to temporary directory where we store downloaded files - self.mp = mp # MerginProject instance + self.temp_dir = temp_dir # full path to temporary directory where we store downloaded files + self.mp = mp # MerginProject instance self.is_cancelled = False - self.project_info = project_info # parsed JSON with project info returned from the server - self.basefiles_to_patch = basefiles_to_patch # list of tuples (relative path within project, list of diff files in temp dir to apply) + self.project_info = project_info # parsed JSON with project info returned from the server + self.basefiles_to_patch = ( + basefiles_to_patch # list of tuples (relative path within project, list of diff files in temp dir to apply) + ) self.user_name = user_name def dump(self): @@ -375,68 +389,67 @@ def pull_project_async(mc, directory): # we either download a versioned file using diffs (strongly preferred), # but if we don't have history with diffs (e.g. uploaded without diffs) # then we just download the whole file - _pulling_file_with_diffs = lambda f: 'diffs' in f and len(f['diffs']) != 0 + _pulling_file_with_diffs = lambda f: "diffs" in f and len(f["diffs"]) != 0 - temp_dir = mp.fpath_meta(f'fetch_{local_version}-{server_version}') + temp_dir = mp.fpath_meta(f"fetch_{local_version}-{server_version}") os.makedirs(temp_dir, exist_ok=True) pull_changes = mp.get_pull_changes(server_info["files"]) mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes)) fetch_files = [] for f in pull_changes["added"]: - f['version'] = server_version + f["version"] = server_version fetch_files.append(f) # extend fetch files download list with various version of diff files (if needed) for f in pull_changes["updated"]: if _pulling_file_with_diffs(f): - for diff in f['diffs']: + for diff in f["diffs"]: diff_file = copy.deepcopy(f) - for k, v in f['history'].items(): - if 'diff' not in v: + for k, v in f["history"].items(): + if "diff" not in v: continue - if diff == v['diff']['path']: - diff_file['version'] = k - diff_file['diff'] = v['diff'] + if diff == v["diff"]["path"]: + diff_file["version"] = k + diff_file["diff"] = v["diff"] fetch_files.append(diff_file) else: - f['version'] = server_version + f["version"] = server_version fetch_files.append(f) - files_to_merge = [] # list of FileToMerge instances + files_to_merge = [] # list of FileToMerge instances for file in fetch_files: diff_only = _pulling_file_with_diffs(file) items = _download_items(file, temp_dir, diff_only) # figure out destination path for the file - file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))) - basename = os.path.basename(file['diff']['path']) if diff_only else os.path.basename(file['path']) + file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir, file["path"]))) + basename = os.path.basename(file["diff"]["path"]) if diff_only else os.path.basename(file["path"]) dest_file_path = os.path.join(file_dir, basename) os.makedirs(file_dir, exist_ok=True) - files_to_merge.append( FileToMerge(dest_file_path, items) ) - + files_to_merge.append(FileToMerge(dest_file_path, items)) # make sure we can update geodiff reference files (aka. basefiles) with diffs or # download their full versions so we have them up-to-date for applying changes basefiles_to_patch = [] # list of tuples (relative path within project, list of diff files in temp dir to apply) - for file in pull_changes['updated']: + for file in pull_changes["updated"]: if not _pulling_file_with_diffs(file): continue # this is only for diffable files (e.g. geopackages) - basefile = mp.fpath_meta(file['path']) + basefile = mp.fpath_meta(file["path"]) if not os.path.exists(basefile): # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. - file_path = file['path'] + file_path = file["path"] mp.log.info(f"missing base file for {file_path} -> going to download it (version {server_version})") - file['version'] = server_version + file["version"] = server_version items = _download_items(file, temp_dir, diff_only=False) dest_file_path = mp.fpath(file["path"], temp_dir) - #dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path'])) - files_to_merge.append( FileToMerge(dest_file_path, items) ) + # dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path'])) + files_to_merge.append(FileToMerge(dest_file_path, items)) continue - basefiles_to_patch.append( (file['path'], file['diffs']) ) + basefiles_to_patch.append((file["path"], file["diffs"])) # make a single list of items to download total_size = 0 @@ -448,7 +461,19 @@ def pull_project_async(mc, directory): mp.log.info(f"will download {len(download_list)} chunks, total size {total_size}") - job = PullJob(project_path, pull_changes, total_size, server_version, files_to_merge, download_list, temp_dir, mp, server_info, basefiles_to_patch, mc.username()) + job = PullJob( + project_path, + pull_changes, + total_size, + server_version, + files_to_merge, + download_list, + temp_dir, + mp, + server_info, + basefiles_to_patch, + mc.username(), + ) # start download job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) @@ -461,7 +486,7 @@ def pull_project_async(mc, directory): def pull_project_wait(job): - """ blocks until all download tasks are finished """ + """blocks until all download tasks are finished""" concurrent.futures.wait(job.futures) @@ -502,15 +527,16 @@ class FileToMerge: to the temporary file containing its data. Calling merge() will create the destination file and remove the temporary files of the chunks """ + def __init__(self, dest_file, downloaded_items, size_check=True): - self.dest_file = dest_file # full path to the destination file to be created - self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged + self.dest_file = dest_file # full path to the destination file to be created + self.downloaded_items = downloaded_items # list of pieces of the destination file to be merged self.size_check = size_check # whether we want to do merged file size check def merge(self): - with open(self.dest_file, 'wb') as final: + with open(self.dest_file, "wb") as final: for item in self.downloaded_items: - with open(item.download_file_path, 'rb') as chunk: + with open(item.download_file_path, "rb") as chunk: shutil.copyfileobj(chunk, final) os.remove(item.download_file_path) @@ -519,7 +545,7 @@ def merge(self): expected_size = sum(item.size for item in self.downloaded_items) if os.path.getsize(self.dest_file) != expected_size: os.remove(self.dest_file) - raise ClientError('Download of file {} failed. Please try it again.'.format(self.dest_file)) + raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file)) def pull_project_finalize(job): @@ -582,15 +608,15 @@ def pull_project_finalize(job): raise ClientError("Failed to apply pull changes: " + str(e)) job.mp.metadata = { - 'name': job.project_path, - 'version': job.version if job.version else "v0", # for new projects server version is "" - 'files': job.project_info['files'] + "name": job.project_path, + "version": job.version if job.version else "v0", # for new projects server version is "" + "files": job.project_info["files"], } if job.mp.has_unfinished_pull(): job.mp.log.info("--- failed to complete pull -- project left in the unfinished pull state") else: - job.mp.log.info("--- pull finished -- at version " + job.mp.metadata['version']) + job.mp.log.info("--- pull finished -- at version " + job.mp.metadata["version"]) shutil.rmtree(job.temp_dir) return conflicts @@ -622,12 +648,12 @@ def download_file_async(mc, project_dir, file_path, output_file, version): # it is necessary to pass actual version. if version is None: version = latest_proj_info["version"] - for file in project_info['files']: + for file in project_info["files"]: if file["path"] == file_path: - file['version'] = version + file["version"] = version items = _download_items(file, temp_dir) is_latest_version = version == latest_proj_info["version"] - task = UpdateTask(file['path'], items, output_file, latest_version=is_latest_version) + task = UpdateTask(file["path"], items, output_file, latest_version=is_latest_version) download_list.extend(task.download_queue_items) for item in task.download_queue_items: total_size += item.size @@ -640,9 +666,7 @@ def download_file_async(mc, project_dir, file_path, output_file, version): raise ClientError(warn) mp.log.info(f"will download file {file_path} in {len(download_list)} chunks, total size {total_size}") - job = DownloadJob( - project_path, total_size, version, update_tasks, download_list, temp_dir, mp, project_info - ) + job = DownloadJob(project_path, total_size, version, update_tasks, download_list, temp_dir, mp, project_info) job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) job.futures = [] for item in download_list: @@ -712,8 +736,8 @@ def download_diffs_async(mc, project_directory, file_path, versions): if "diff" not in version_data: continue # skip if there is no diff in history diff_data = copy.deepcopy(version_data) - diff_data['version'] = version - diff_data['diff'] = version_data['diff'] + diff_data["version"] = version + diff_data["diff"] = version_data["diff"] fetch_files.append(diff_data) files_to_merge = [] # list of FileToMerge instances @@ -721,7 +745,7 @@ def download_diffs_async(mc, project_directory, file_path, versions): total_size = 0 for file in fetch_files: items = _download_items(file, mp.cache_dir, diff_only=True) - dest_file_path = mp.fpath_cache(file['diff']['path'], version=file["version"]) + dest_file_path = mp.fpath_cache(file["diff"]["path"], version=file["version"]) if os.path.exists(dest_file_path): continue files_to_merge.append(FileToMerge(dest_file_path, items)) @@ -731,8 +755,19 @@ def download_diffs_async(mc, project_directory, file_path, versions): mp.log.info(f"will download {len(download_list)} chunks, total size {total_size}") - job = PullJob(project_path, None, total_size, None, files_to_merge, download_list, mp.cache_dir, mp, - server_info, {}, mc.username()) + job = PullJob( + project_path, + None, + total_size, + None, + files_to_merge, + download_list, + mp.cache_dir, + mp, + server_info, + {}, + mc.username(), + ) # start download job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) @@ -745,7 +780,7 @@ def download_diffs_async(mc, project_directory, file_path, versions): def download_diffs_finalize(job): - """ To be called after download_diffs_async + """To be called after download_diffs_async Returns: diffs: list of downloaded diffs (their actual locations on disk) diff --git a/mergin/client_push.py b/mergin/client_push.py index 3a22757..11ae3ba 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -20,22 +20,22 @@ class UploadJob: - """ Keeps all the important data about a pending upload job """ + """Keeps all the important data about a pending upload job""" def __init__(self, project_path, changes, transaction_id, mp, mc, tmp_dir): - self.project_path = project_path # full project name ("username/projectname") - self.changes = changes # dictionary of local changes to the project - self.transaction_id = transaction_id # ID of the transaction assigned by the server - self.total_size = 0 # size of data to upload (in bytes) - self.transferred_size = 0 # size of data already uploaded (in bytes) - self.upload_queue_items = [] # list of items to upload in the background - self.mp = mp # MerginProject instance - self.mc = mc # MerginClient instance - self.tmp_dir = tmp_dir # TemporaryDirectory instance for any temp file we need - self.is_cancelled = False # whether upload has been cancelled - self.executor = None # ThreadPoolExecutor that manages background upload tasks - self.futures = [] # list of futures submitted to the executor - self.server_resp = None # server response when transaction is finished + self.project_path = project_path # full project name ("username/projectname") + self.changes = changes # dictionary of local changes to the project + self.transaction_id = transaction_id # ID of the transaction assigned by the server + self.total_size = 0 # size of data to upload (in bytes) + self.transferred_size = 0 # size of data already uploaded (in bytes) + self.upload_queue_items = [] # list of items to upload in the background + self.mp = mp # MerginProject instance + self.mc = mc # MerginClient instance + self.tmp_dir = tmp_dir # TemporaryDirectory instance for any temp file we need + self.is_cancelled = False # whether upload has been cancelled + self.executor = None # ThreadPoolExecutor that manages background upload tasks + self.futures = [] # list of futures submitted to the executor + self.server_resp = None # server response when transaction is finished def dump(self): print("--- JOB ---", self.total_size, "bytes") @@ -45,18 +45,18 @@ def dump(self): class UploadQueueItem: - """ A single chunk of data that needs to be uploaded """ + """A single chunk of data that needs to be uploaded""" def __init__(self, file_path, size, transaction_id, chunk_id, chunk_index): - self.file_path = file_path # full path to the file - self.size = size # size of the chunk in bytes - self.chunk_id = chunk_id # ID of the chunk within transaction - self.chunk_index = chunk_index # index (starting from zero) of the chunk within the file + self.file_path = file_path # full path to the file + self.size = size # size of the chunk in bytes + self.chunk_id = chunk_id # ID of the chunk within transaction + self.chunk_index = chunk_index # index (starting from zero) of the chunk within the file self.transaction_id = transaction_id # ID of the transaction def upload_blocking(self, mc, mp): - with open(self.file_path, 'rb') as file_handle: + with open(self.file_path, "rb") as file_handle: file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE) data = file_handle.read(UPLOAD_CHUNK_SIZE) @@ -69,7 +69,7 @@ def upload_blocking(self, mc, mp): resp = mc.post("/v1/project/push/chunk/{}/{}".format(self.transaction_id, self.chunk_id), data, headers) resp_dict = json.load(resp) mp.log.debug(f"Upload finished: {self.file_path}") - if not (resp_dict['size'] == len(data) and resp_dict['checksum'] == checksum.hexdigest()): + if not (resp_dict["size"] == len(data) and resp_dict["checksum"] == checksum.hexdigest()): try: mc.post("/v1/project/push/cancel/{}".format(self.transaction_id)) except ClientError: @@ -78,7 +78,7 @@ def upload_blocking(self, mc, mp): def push_project_async(mc, directory): - """ Starts push of a project and returns pending upload job """ + """Starts push of a project and returns pending upload job""" mp = MerginProject(directory) if mp.has_unfinished_pull(): @@ -110,8 +110,10 @@ def push_project_async(mc, directory): if local_version != server_version: mp.log.error(f"--- push {project_path} - not up to date (local {local_version} vs server {server_version})") - raise ClientError("There is a new version of the project on the server. Please update your local copy." + - f"\n\nLocal version: {local_version}\nServer version: {server_version}") + raise ClientError( + "There is a new version of the project on the server. Please update your local copy." + + f"\n\nLocal version: {local_version}\nServer version: {server_version}" + ) changes = mp.get_push_changes() mp.log.debug("push changes:\n" + pprint.pformat(changes)) @@ -135,7 +137,7 @@ def push_project_async(mc, directory): if username == project_path.split("/")[0]: enough_free_space, freespace = mc.enough_storage_available(changes) if not enough_free_space: - freespace = int(freespace/(1024*1024)) + freespace = int(freespace / (1024 * 1024)) mp.log.error(f"--- push {project_path} - not enough space") raise ClientError("Storage limit has been reached. Only " + str(freespace) + "MB left") @@ -144,22 +146,19 @@ def push_project_async(mc, directory): return # drop internal info from being sent to server - for item in changes['updated']: - item.pop('origin_checksum', None) - data = { - "version": local_version, - "changes": changes - } + for item in changes["updated"]: + item.pop("origin_checksum", None) + data = {"version": local_version, "changes": changes} try: - resp = mc.post(f'/v1/project/push/{project_path}', data, {"Content-Type": "application/json"}) + resp = mc.post(f"/v1/project/push/{project_path}", data, {"Content-Type": "application/json"}) except ClientError as err: mp.log.error("Error starting transaction: " + str(err)) mp.log.info("--- push aborted") raise server_resp = json.load(resp) - upload_files = data['changes']["added"] + data['changes']["updated"] + upload_files = data["changes"]["added"] + data["changes"]["updated"] transaction_id = server_resp["transaction"] if upload_files else None job = UploadJob(project_path, changes, transaction_id, mp, mc, tmp_dir) @@ -168,7 +167,7 @@ def push_project_async(mc, directory): mp.log.info("not uploading any files") job.server_resp = server_resp push_project_finalize(job) - return None # all done - no pending job + return None # all done - no pending job mp.log.info(f"got transaction ID {transaction_id}") @@ -176,18 +175,18 @@ def push_project_async(mc, directory): total_size = 0 # prepare file chunks for upload for file in upload_files: - if 'diff' in file: + if "diff" in file: # versioned file - uploading diff - file_location = mp.fpath_meta(file['diff']['path']) - file_size = file['diff']['size'] + file_location = mp.fpath_meta(file["diff"]["path"]) + file_size = file["diff"]["size"] elif "upload_file" in file: # versioned file - uploading full (a temporary copy) file_location = file["upload_file"] file_size = file["size"] else: # non-versioned file - file_location = mp.fpath(file['path']) - file_size = file['size'] + file_location = mp.fpath(file["path"]) + file_size = file["size"] for chunk_index, chunk_id in enumerate(file["chunks"]): size = min(UPLOAD_CHUNK_SIZE, file_size - chunk_index * UPLOAD_CHUNK_SIZE) @@ -210,7 +209,7 @@ def push_project_async(mc, directory): def push_project_wait(job): - """ blocks until all upload tasks are finished """ + """blocks until all upload tasks are finished""" concurrent.futures.wait(job.futures) @@ -255,7 +254,9 @@ def push_project_finalize(job): raise future.exception() if job.transferred_size != job.total_size: - error_msg = "Transferred size ({}) and expected total size ({}) do not match!".format(job.transferred_size, job.total_size) + error_msg = "Transferred size ({}) and expected total size ({}) do not match!".format( + job.transferred_size, job.total_size + ) job.mp.log.error("--- push finish failed! " + error_msg) raise ClientError("Upload error: " + error_msg) @@ -280,9 +281,9 @@ def push_project_finalize(job): raise err job.mp.metadata = { - 'name': job.project_path, - 'version': job.server_resp['version'], - 'files': job.server_resp["files"] + "name": job.project_path, + "version": job.server_resp["version"], + "files": job.server_resp["files"], } try: job.mp.apply_push_changes(job.changes) @@ -291,9 +292,9 @@ def push_project_finalize(job): job.mp.log.info("--- push aborted") raise ClientError("Failed to apply push changes: " + str(e)) - job.tmp_dir.cleanup() # delete our temporary dir and all its content + job.tmp_dir.cleanup() # delete our temporary dir and all its content - job.mp.log.info("--- push finished - new project version " + job.server_resp['version']) + job.mp.log.info("--- push finished - new project version " + job.server_resp["version"]) def push_project_cancel(job): @@ -317,7 +318,7 @@ def push_project_cancel(job): def _do_upload(item, job): - """ runs in worker thread """ + """runs in worker thread""" if job.is_cancelled: return diff --git a/mergin/common.py b/mergin/common.py index 3c6e4e6..bc75870 100644 --- a/mergin/common.py +++ b/mergin/common.py @@ -1,4 +1,3 @@ - import os @@ -28,9 +27,10 @@ class InvalidProject(Exception): from dateutil.tz import tzlocal except ImportError: # this is to import all dependencies shipped with package (e.g. to use in qgis-plugin) - deps_dir = os.path.join(this_dir, 'deps') + deps_dir = os.path.join(this_dir, "deps") if os.path.exists(deps_dir): import sys + for f in os.listdir(os.path.join(deps_dir)): sys.path.append(os.path.join(deps_dir, f)) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 410a7be..d1687fd 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -1,4 +1,3 @@ - import json import logging import math @@ -19,7 +18,7 @@ do_sqlite_checkpoint, unique_path_name, conflicted_copy_file_name, - edit_conflict_file_name + edit_conflict_file_name, ) @@ -36,23 +35,24 @@ class MerginProject: - """ Base class for Mergin Maps local projects. + """Base class for Mergin Maps local projects. Linked to existing local directory, with project metadata (mergin.json) and backups located in .mergin directory. """ + def __init__(self, directory): self.dir = os.path.abspath(directory) if not os.path.exists(self.dir): - raise InvalidProject('Project directory does not exist') + raise InvalidProject("Project directory does not exist") - self.meta_dir = os.path.join(self.dir, '.mergin') + self.meta_dir = os.path.join(self.dir, ".mergin") if not os.path.exists(self.meta_dir): os.mkdir(self.meta_dir) # location for files from unfinished pull - self.unfinished_pull_dir = os.path.join(self.meta_dir, 'unfinished_pull') + self.unfinished_pull_dir = os.path.join(self.meta_dir, "unfinished_pull") - self.cache_dir = os.path.join(self.meta_dir, '.cache') + self.cache_dir = os.path.join(self.meta_dir, ".cache") if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) @@ -75,18 +75,19 @@ def _logger_callback(level, text_bytes): self.log.warning("GEODIFF: " + text) else: self.log.info("GEODIFF: " + text) + self.geodiff.set_logger_callback(_logger_callback) self.geodiff.set_maximum_logger_level(pygeodiff.GeoDiff.LevelDebug) def setup_logging(self, logger_name): """Setup logging into project directory's .mergin/client-log.txt file.""" - self.log = logging.getLogger('mergin.project.' + logger_name) + self.log = logging.getLogger("mergin.project." + logger_name) self.log.setLevel(logging.DEBUG) # log everything (it would otherwise log just warnings+errors) if not self.log.handlers: # we only need to set the handler once # (otherwise we would get things logged multiple times as loggers are cached) - log_handler = logging.FileHandler(os.path.join(self.meta_dir, "client-log.txt"), encoding = 'utf-8') - log_handler.setFormatter(logging.Formatter('%(asctime)s %(message)s')) + log_handler = logging.FileHandler(os.path.join(self.meta_dir, "client-log.txt"), encoding="utf-8") + log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s")) self.log.addHandler(log_handler) def remove_logging_handler(self): @@ -116,15 +117,15 @@ def fpath(self, file, other_dir=None): return abs_path def fpath_meta(self, file): - """ Helper function to get absolute path of file in meta dir. """ + """Helper function to get absolute path of file in meta dir.""" return self.fpath(file, self.meta_dir) def fpath_unfinished_pull(self, file): - """ Helper function to get absolute path of file in unfinished_pull dir. """ + """Helper function to get absolute path of file in unfinished_pull dir.""" return self.fpath(file, self.unfinished_pull_dir) def fpath_cache(self, file, version=None): - """ Helper function to get absolute path of file in cache dir. + """Helper function to get absolute path of file in cache dir. It can be either in root cache directory (.mergin/.cache/) or in some version's subfolder """ if version: @@ -133,25 +134,25 @@ def fpath_cache(self, file, version=None): @property def metadata(self): - if not os.path.exists(self.fpath_meta('mergin.json')): - raise InvalidProject('Project metadata has not been created yet') - with open(self.fpath_meta('mergin.json'), 'r') as file: + if not os.path.exists(self.fpath_meta("mergin.json")): + raise InvalidProject("Project metadata has not been created yet") + with open(self.fpath_meta("mergin.json"), "r") as file: return json.load(file) @metadata.setter def metadata(self, data): - with open(self.fpath_meta('mergin.json'), 'w') as file: + with open(self.fpath_meta("mergin.json"), "w") as file: json.dump(data, file, indent=2) def is_versioned_file(self, file): - """ Check if file is compatible with geodiff lib and hence suitable for versioning. + """Check if file is compatible with geodiff lib and hence suitable for versioning. :param file: file path :type file: str :returns: if file is compatible with geodiff lib :rtype: bool """ - diff_extensions = ['.gpkg', '.sqlite'] + diff_extensions = [".gpkg", ".sqlite"] f_extension = os.path.splitext(file)[1] return f_extension in diff_extensions @@ -165,9 +166,9 @@ def is_gpkg_open(self, path): :rtype: bool """ f_extension = os.path.splitext(path)[1] - if f_extension != '.gpkg': + if f_extension != ".gpkg": return False - if os.path.exists(f'{path}-wal'): + if os.path.exists(f"{path}-wal"): return True return False @@ -180,8 +181,8 @@ def ignore_file(self, file): :returns: whether file should be ignored :rtype: bool """ - ignore_ext = re.compile(r'({})$'.format('|'.join(re.escape(x) for x in ['-shm', '-wal', '~', 'pyc', 'swap']))) - ignore_files = ['.DS_Store', '.directory'] + ignore_ext = re.compile(r"({})$".format("|".join(re.escape(x) for x in ["-shm", "-wal", "~", "pyc", "swap"]))) + ignore_files = [".DS_Store", ".directory"] name, ext = os.path.splitext(file) if ext and ignore_ext.search(ext): return True @@ -198,20 +199,22 @@ def inspect_files(self): """ files_meta = [] for root, dirs, files in os.walk(self.dir, topdown=True): - dirs[:] = [d for d in dirs if d not in ['.mergin']] + dirs[:] = [d for d in dirs if d not in [".mergin"]] for file in files: if self.ignore_file(file): continue abs_path = os.path.abspath(os.path.join(root, file)) rel_path = os.path.relpath(abs_path, start=self.dir) - proj_path = '/'.join(rel_path.split(os.path.sep)) # we need posix path - files_meta.append({ - "path": proj_path, - "checksum": generate_checksum(abs_path), - "size": os.path.getsize(abs_path), - "mtime": datetime.fromtimestamp(os.path.getmtime(abs_path), tzlocal()) - }) + proj_path = "/".join(rel_path.split(os.path.sep)) # we need posix path + files_meta.append( + { + "path": proj_path, + "checksum": generate_checksum(abs_path), + "size": os.path.getsize(abs_path), + "mtime": datetime.fromtimestamp(os.path.getmtime(abs_path), tzlocal()), + } + ) return files_meta def compare_file_sets(self, origin, current): @@ -247,12 +250,7 @@ def compare_file_sets(self, origin, current): f["origin_checksum"] = origin_map[path]["checksum"] updated.append(f) - return { - "renamed": [], - "added": added, - "removed": removed, - "updated": updated - } + return {"renamed": [], "added": added, "removed": removed, "updated": updated} def get_pull_changes(self, server_files): """ @@ -272,11 +270,11 @@ def get_pull_changes(self, server_files): """ # first let's have a look at the added/updated/removed files - changes = self.compare_file_sets(self.metadata['files'], server_files) + changes = self.compare_file_sets(self.metadata["files"], server_files) # then let's inspect our versioned files (geopackages) if there are any relevant changes not_updated = [] - for file in changes['updated']: + for file in changes["updated"]: if not self.is_versioned_file(file["path"]): continue @@ -286,28 +284,28 @@ def get_pull_changes(self, server_files): # get sorted list of the history (they may not be sorted or using lexical sorting - "v10", "v11", "v5", "v6", ...) history_list = [] - for version_str, version_info in file['history'].items(): - history_list.append( (int_version(version_str), version_info) ) + for version_str, version_info in file["history"].items(): + history_list.append((int_version(version_str), version_info)) history_list = sorted(history_list, key=lambda item: item[0]) # sort tuples based on version numbers # need to track geodiff file history to see if there were any changes for version, version_info in history_list: - if version <= int_version(self.metadata['version']): + if version <= int_version(self.metadata["version"]): continue # ignore history of no interest is_updated = True - if 'diff' in version_info: - diffs.append(version_info['diff']['path']) - diffs_size += version_info['diff']['size'] + if "diff" in version_info: + diffs.append(version_info["diff"]["path"]) + diffs_size += version_info["diff"]["size"] else: diffs = [] break # we found force update in history, does not make sense to download diffs if is_updated: - file['diffs'] = diffs + file["diffs"] = diffs else: not_updated.append(file) - changes['updated'] = [f for f in changes['updated'] if f not in not_updated] + changes["updated"] = [f for f in changes["updated"] if f not in not_updated] return changes def get_push_changes(self): @@ -323,18 +321,18 @@ def get_push_changes(self): :returns: changes metadata for files to be pushed to server :rtype: dict """ - changes = self.compare_file_sets(self.metadata['files'], self.inspect_files()) + changes = self.compare_file_sets(self.metadata["files"], self.inspect_files()) # do checkpoint to push changes from wal file to gpkg - for file in changes['added'] + changes['updated']: + for file in changes["added"] + changes["updated"]: size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log) if size and checksum: file["size"] = size file["checksum"] = checksum - file['chunks'] = [str(uuid.uuid4()) for i in range(math.ceil(file["size"] / UPLOAD_CHUNK_SIZE))] + file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(file["size"] / UPLOAD_CHUNK_SIZE))] # need to check for for real changes in geodiff files using geodiff tool (comparing checksum is not enough) not_updated = [] - for file in changes['updated']: + for file in changes["updated"]: path = file["path"] if not self.is_versioned_file(path): continue @@ -343,20 +341,20 @@ def get_push_changes(self): current_file = self.fpath(path) origin_file = self.fpath_meta(path) diff_id = str(uuid.uuid4()) - diff_name = path + '-diff-' + diff_id + diff_name = path + "-diff-" + diff_id diff_file = self.fpath_meta(diff_name) try: self.geodiff.create_changeset(origin_file, current_file, diff_file) if self.geodiff.has_changes(diff_file): diff_size = os.path.getsize(diff_file) - file['checksum'] = file['origin_checksum'] # need to match basefile on server - file['chunks'] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] - file['mtime'] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) - file['diff'] = { + file["checksum"] = file["origin_checksum"] # need to match basefile on server + file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))] + file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal()) + file["diff"] = { "path": diff_name, "checksum": generate_checksum(diff_file), "size": diff_size, - 'mtime': datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()) + "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()), } else: not_updated.append(file) @@ -366,7 +364,7 @@ def get_push_changes(self): # we will need to do full upload of the file pass - changes['updated'] = [f for f in changes['updated'] if f not in not_updated] + changes["updated"] = [f for f in changes["updated"] if f not in not_updated] return changes def copy_versioned_file_for_upload(self, f, tmp_dir): @@ -394,7 +392,7 @@ def get_list_of_push_changes(self, push_changes): result_file = self.fpath("change_list" + str(idx), self.meta_dir) try: self.geodiff.list_changes_summary(changeset, result_file) - with open(result_file, 'r') as f: + with open(result_file, "r") as f: change = f.read() changes[file["path"]] = json.loads(change) os.remove(result_file) @@ -422,22 +420,22 @@ def apply_pull_changes(self, changes, temp_dir, user_name): local_changes = self.get_push_changes() modified = {} for f in local_changes["added"] + local_changes["updated"]: - modified.update({f['path']: f}) + modified.update({f["path"]: f}) local_files_map = {} for f in self.inspect_files(): - local_files_map.update({f['path']: f}) + local_files_map.update({f["path"]: f}) for k, v in changes.items(): for item in v: - path = item['path'] + path = item["path"] src = self.fpath(path, temp_dir) dest = self.fpath(path) basefile = self.fpath_meta(path) # special care is needed for geodiff files # 'src' here is server version of file and 'dest' is locally modified - if self.is_versioned_file(path) and k == 'updated': + if self.is_versioned_file(path) and k == "updated": if path in modified: conflict = self.update_with_rebase(path, src, dest, basefile, temp_dir, user_name) if conflict: @@ -448,11 +446,11 @@ def apply_pull_changes(self, changes, temp_dir, user_name): self.update_without_rebase(path, src, dest, basefile, temp_dir) else: # backup if needed - if path in modified and item['checksum'] != local_files_map[path]['checksum']: + if path in modified and item["checksum"] != local_files_map[path]["checksum"]: conflict = self.create_conflicted_copy(path, user_name) conflicts.append(conflict) - if k == 'removed': + if k == "removed": if os.path.exists(dest): os.remove(dest) else: @@ -492,15 +490,15 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): """ self.log.info("updating file with rebase: " + path) - server_diff = self.fpath(f'{path}-server_diff', temp_dir) # diff between server file and local basefile - local_diff = self.fpath(f'{path}-local_diff', temp_dir) + server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile + local_diff = self.fpath(f"{path}-local_diff", temp_dir) # temporary backup of file pulled from server for recovery - f_server_backup = self.fpath(f'{path}-server_backup', temp_dir) + f_server_backup = self.fpath(f"{path}-server_backup", temp_dir) self.geodiff.make_copy_sqlite(src, f_server_backup) # create temp backup (ideally with geodiff) of locally modified file if needed later - f_conflict_file = self.fpath(f'{path}-local_backup', temp_dir) + f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir) try: self.geodiff.create_changeset(basefile, dest, local_diff) @@ -514,7 +512,8 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): # they will be stored in a JSON file - if there are no conflicts, the file # won't even be created rebase_conflicts = unique_path_name( - edit_conflict_file_name(self.fpath(path), user_name, int_version(self.metadata['version']))) + edit_conflict_file_name(self.fpath(path), user_name, int_version(self.metadata["version"])) + ) # try to do rebase magic try: @@ -538,7 +537,7 @@ def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name): f_server_unfinished = self.fpath_unfinished_pull(path) self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished) - return '' + return "" def update_without_rebase(self, path, src, dest, basefile, temp_dir): """ @@ -562,7 +561,7 @@ def update_without_rebase(self, path, src, dest, basefile, temp_dir): """ self.log.info("updating file without rebase: " + path) try: - server_diff = self.fpath(f'{path}-server_diff', temp_dir) # diff between server file and local basefile + server_diff = self.fpath(f"{path}-server_diff", temp_dir) # diff between server file and local basefile # TODO: it could happen that basefile does not exist. # It was either never created (e.g. when pushing without geodiff) # or it was deleted by mistake(?) by the user. We should detect that @@ -587,16 +586,16 @@ def apply_push_changes(self, changes): """ for k, v in changes.items(): for item in v: - path = item['path'] + path = item["path"] if not self.is_versioned_file(path): continue basefile = self.fpath_meta(path) - if k == 'removed': + if k == "removed": os.remove(basefile) - elif k == 'added': + elif k == "added": self.geodiff.make_copy_sqlite(self.fpath(path), basefile) - elif k == 'updated': + elif k == "updated": # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file) if "diff" not in item: self.log.info("updating basefile (copy) for: " + path) @@ -604,7 +603,7 @@ def apply_push_changes(self, changes): else: self.log.info("updating basefile (diff) for: " + path) # better to apply diff to previous basefile to avoid issues with geodiff tmp files - changeset = self.fpath_meta(item['diff']['path']) + changeset = self.fpath_meta(item["diff"]["path"]) patch_error = self.apply_diffs(basefile, [changeset]) if patch_error: # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server @@ -626,7 +625,9 @@ def create_conflicted_copy(self, file, user_name): if not os.path.exists(src): return - backup_path = unique_path_name(conflicted_copy_file_name(self.fpath(file), user_name, int_version(self.metadata['version']))) + backup_path = unique_path_name( + conflicted_copy_file_name(self.fpath(file), user_name, int_version(self.metadata["version"])) + ) if self.is_versioned_file(file): self.geodiff.make_copy_sqlite(src, backup_path) @@ -660,7 +661,7 @@ def apply_diffs(self, basefile, diffs): return error def has_unfinished_pull(self): - """ Check if there is an unfinished pull for this project. + """Check if there is an unfinished pull for this project. Unfinished pull means that a previous pull_project() call has failed in the final stage due to some files being in read-only diff --git a/mergin/report.py b/mergin/report.py index 24ee717..9c3a6a1 100644 --- a/mergin/report.py +++ b/mergin/report.py @@ -10,8 +10,13 @@ from .utils import int_version try: - from qgis.core import QgsGeometry, QgsDistanceArea, QgsCoordinateReferenceSystem, QgsCoordinateTransformContext, \ - QgsWkbTypes + from qgis.core import ( + QgsGeometry, + QgsDistanceArea, + QgsCoordinateReferenceSystem, + QgsCoordinateTransformContext, + QgsWkbTypes, + ) has_qgis = True except ImportError: @@ -22,7 +27,7 @@ # in geodiff lib (MIT licence) # ideally there should be pygeodiff api for it def parse_gpkgb_header_size(gpkg_wkb): - """ Parse header of geopackage wkb and return its size """ + """Parse header of geopackage wkb and return its size""" # some constants no_envelope_header_size = 8 flag_byte_pos = 3 @@ -58,7 +63,7 @@ def qgs_geom_from_wkb(geom): class ChangesetReportEntry: - """ Derivative of geodiff ChangesetEntry suitable for further processing/reporting """ + """Derivative of geodiff ChangesetEntry suitable for further processing/reporting""" def __init__(self, changeset_entry, geom_idx, geom, qgs_distance_area=None): self.table = changeset_entry.table.name @@ -126,7 +131,7 @@ def __init__(self, changeset_entry, geom_idx, geom, qgs_distance_area=None): def changeset_report(changeset_reader, schema, mp): - """ Parse Geodiff changeset reader and create report from it. + """Parse Geodiff changeset reader and create report from it. Aggregate individual entries based on common table, operation and geom type. If QGIS API is available, then lengths and areas of individual entries are summed. @@ -150,7 +155,7 @@ def changeset_report(changeset_reader, schema, mp): if has_qgis: distance_area = QgsDistanceArea() - distance_area.setEllipsoid('WGS84') + distance_area.setEllipsoid("WGS84") else: distance_area = None # let's iterate through reader and populate entries @@ -158,8 +163,9 @@ def changeset_report(changeset_reader, schema, mp): schema_table = next((t for t in schema if t["table"] == entry.table.name), None) if schema_table: # get geometry index in both gpkg schema and diffs values - geom_idx = next((index for (index, col) in enumerate(schema_table["columns"]) if col["type"] == "geometry"), - None) + geom_idx = next( + (index for (index, col) in enumerate(schema_table["columns"]) if col["type"] == "geometry"), None + ) if geom_idx is None: continue @@ -183,18 +189,12 @@ def changeset_report(changeset_reader, schema, mp): # sum lenghts and areas only if it makes sense (valid dimension) area = sum([entry.area for entry in values if entry.area]) if values[0].dim == 2 else None length = sum([entry.length for entry in values if entry.length]) if values[0].dim > 0 else None - records.append({ - "table": table, - "operation": k[0], - "length": length, - "area": area, - "count": len(values) - }) + records.append({"table": table, "operation": k[0], "length": length, "area": area, "count": len(values)}) return records def create_report(mc, directory, since, to, out_file): - """ Creates report from geodiff changesets for a range of project versions in CSV format. + """Creates report from geodiff changesets for a range of project versions in CSV format. Report is created for all .gpkg files and all tables within where updates were done using Geodiff lib. Changeset contains operation (insert/update/delete) and geometry properties like length/perimeter and area. @@ -248,22 +248,22 @@ def create_report(mc, directory, since, to, out_file): mc.download_file(directory, f["path"], full_gpkg, to) # get gpkg schema - schema_file = full_gpkg + '-schema.json' # geodiff writes schema into a file + schema_file = full_gpkg + "-schema.json" # geodiff writes schema into a file if not os.path.exists(schema_file): mp.geodiff.schema("sqlite", "", full_gpkg, schema_file) - with open(schema_file, 'r') as sf: + with open(schema_file, "r") as sf: schema = json.load(sf).get("geodiff_schema") # add records for every version (diff) and all tables within geopackage for version in history_keys: - if "diff" not in f['history'][version]: - if f['history'][version]["change"] == "updated": + if "diff" not in f["history"][version]: + if f["history"][version]["change"] == "updated": warnings.append(f"Missing diff: {f['path']} was overwritten in {version} - broken diff history") else: warnings.append(f"Missing diff: {f['path']} was {f['history'][version]['change']} in {version}") continue - v_diff_file = mp.fpath_cache(f['history'][version]['diff']['path'], version=version) + v_diff_file = mp.fpath_cache(f["history"][version]["diff"]["path"], version=version) version_data = versions_map[version] cr = mp.geodiff.read_changeset(v_diff_file) report = changeset_report(cr, schema, mp) @@ -274,7 +274,7 @@ def create_report(mc, directory, since, to, out_file): "author": version_data["author"], "date": dt.date().isoformat(), "time": dt.time().isoformat(), - "version": version_data["name"] + "version": version_data["name"], } for row in report: records.append({**row, **version_fields}) @@ -286,7 +286,7 @@ def create_report(mc, directory, since, to, out_file): # export report to csv file out_dir = os.path.dirname(out_file) os.makedirs(out_dir, exist_ok=True) - with open(out_file, 'w', newline='') as f_csv: + with open(out_file, "w", newline="") as f_csv: writer = csv.DictWriter(f_csv, fieldnames=headers) writer.writeheader() writer.writerows(records) diff --git a/mergin/test/sqlite_con.py b/mergin/test/sqlite_con.py index d13caec..dc0fe05 100644 --- a/mergin/test/sqlite_con.py +++ b/mergin/test/sqlite_con.py @@ -11,5 +11,6 @@ while True: cmd = input() sys.stderr.write(cmd + "\n") - if cmd == 'stop': break + if cmd == "stop": + break cursor.execute(cmd) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 5dbffe0..698d9fe 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -17,34 +17,34 @@ get_versions_with_file_changes, unique_path_name, conflicted_copy_file_name, - edit_conflict_file_name + edit_conflict_file_name, ) from ..merginproject import pygeodiff from ..report import create_report -SERVER_URL = os.environ.get('TEST_MERGIN_URL') -API_USER = os.environ.get('TEST_API_USERNAME') -USER_PWD = os.environ.get('TEST_API_PASSWORD') -API_USER2 = os.environ.get('TEST_API_USERNAME2') -USER_PWD2 = os.environ.get('TEST_API_PASSWORD2') +SERVER_URL = os.environ.get("TEST_MERGIN_URL") +API_USER = os.environ.get("TEST_API_USERNAME") +USER_PWD = os.environ.get("TEST_API_PASSWORD") +API_USER2 = os.environ.get("TEST_API_USERNAME2") +USER_PWD2 = os.environ.get("TEST_API_PASSWORD2") TMP_DIR = tempfile.gettempdir() -TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') -CHANGED_SCHEMA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'modified_schema') +TEST_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_data") +CHANGED_SCHEMA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "modified_schema") -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def mc(): return create_client(API_USER, USER_PWD) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def mc2(): return create_client(API_USER2, USER_PWD2) def create_client(user, pwd): - assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com' and user and pwd + assert SERVER_URL and SERVER_URL.rstrip("/") != "https://app.merginmaps.com" and user and pwd return MerginClient(SERVER_URL, login=user, password=pwd) @@ -65,7 +65,7 @@ def remove_folders(dirs): def test_login(mc): - token = mc._auth_session['token'] + token = mc._auth_session["token"] assert MerginClient(mc.url, auth_token=token) invalid_token = "Completely invalid token...." @@ -76,41 +76,41 @@ def test_login(mc): with pytest.raises(TokenError, match=f"Invalid token data: {invalid_token}"): decode_token_data(invalid_token) - with pytest.raises(LoginError, match='Invalid username or password'): - mc.login('foo', 'bar') + with pytest.raises(LoginError, match="Invalid username or password"): + mc.login("foo", "bar") def test_create_delete_project(mc): - test_project = 'test_create_delete' - project = API_USER + '/' + test_project + test_project = "test_create_delete" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) - download_dir = os.path.join(TMP_DIR, 'download', test_project) + download_dir = os.path.join(TMP_DIR, "download", test_project) cleanup(mc, project, [project_dir, download_dir]) # create new (empty) project on server mc.create_project(test_project) - projects = mc.projects_list(flag='created') - assert any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) + projects = mc.projects_list(flag="created") + assert any(p for p in projects if p["name"] == test_project and p["namespace"] == API_USER) # try again - with pytest.raises(ClientError, match=f'Project {test_project} already exists'): + with pytest.raises(ClientError, match=f"Project {test_project} already exists"): mc.create_project(test_project) # remove project - mc.delete_project(API_USER + '/' + test_project) - projects = mc.projects_list(flag='created') - assert not any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) + mc.delete_project(API_USER + "/" + test_project) + projects = mc.projects_list(flag="created") + assert not any(p for p in projects if p["name"] == test_project and p["namespace"] == API_USER) # try again, nothing to delete with pytest.raises(ClientError): - mc.delete_project(API_USER + '/' + test_project) + mc.delete_project(API_USER + "/" + test_project) def test_create_remote_project_from_local(mc): - test_project = 'test_project' - project = API_USER + '/' + test_project + test_project = "test_project" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) - download_dir = os.path.join(TMP_DIR, 'download', test_project) + download_dir = os.path.join(TMP_DIR, "download", test_project) cleanup(mc, project, [project_dir, download_dir]) # prepare local project @@ -121,37 +121,34 @@ def test_create_remote_project_from_local(mc): # check basic metadata about created project project_info = mc.project_info(project) - assert project_info['version'] == 'v1' - assert project_info['name'] == test_project - assert project_info['namespace'] == API_USER + assert project_info["version"] == "v1" + assert project_info["name"] == test_project + assert project_info["namespace"] == API_USER versions = mc.project_versions(project) assert len(versions) == 1 - assert versions[0]['name'] == 'v1' - assert any(f for f in versions[0]['changes']['added'] if f['path'] == 'test.qgs') + assert versions[0]["name"] == "v1" + assert any(f for f in versions[0]["changes"]["added"] if f["path"] == "test.qgs") # check we can fully download remote project mc.download_project(project, download_dir) mp = MerginProject(download_dir) - downloads = { - 'dir': os.listdir(mp.dir), - 'meta': os.listdir(mp.meta_dir) - } - for f in os.listdir(project_dir) + ['.mergin']: - assert f in downloads['dir'] + downloads = {"dir": os.listdir(mp.dir), "meta": os.listdir(mp.meta_dir)} + for f in os.listdir(project_dir) + [".mergin"]: + assert f in downloads["dir"] if mp.is_versioned_file(f): - assert f in downloads['meta'] + assert f in downloads["meta"] # unable to download to the same directory - with pytest.raises(Exception, match='Project directory already exists'): + with pytest.raises(Exception, match="Project directory already exists"): mc.download_project(project, download_dir) def test_push_pull_changes(mc): - test_project = 'test_push' - project = API_USER + '/' + test_project + test_project = "test_push" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates - project_dir_2 = os.path.join(TMP_DIR, test_project+'_2') # concurrent project dir + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir cleanup(mc, project, [project_dir, project_dir_2]) # create remote project @@ -162,68 +159,71 @@ def test_push_pull_changes(mc): mc.download_project(project, project_dir_2) # test push changes (add, remove, rename, update) - f_added = 'new.txt' - with open(os.path.join(project_dir, f_added), 'w') as f: - f.write('new file') - f_removed = 'test.txt' + f_added = "new.txt" + with open(os.path.join(project_dir, f_added), "w") as f: + f.write("new file") + f_removed = "test.txt" os.remove(os.path.join(project_dir, f_removed)) - f_renamed = 'test_dir/test2.txt' - shutil.move(os.path.normpath(os.path.join(project_dir, f_renamed)), os.path.join(project_dir, 'renamed.txt')) - f_updated = 'test3.txt' - with open(os.path.join(project_dir, f_updated), 'w') as f: - f.write('Modified') + f_renamed = "test_dir/test2.txt" + shutil.move(os.path.normpath(os.path.join(project_dir, f_renamed)), os.path.join(project_dir, "renamed.txt")) + f_updated = "test3.txt" + with open(os.path.join(project_dir, f_updated), "w") as f: + f.write("Modified") # check changes before applied pull_changes, push_changes, _ = mc.project_status(project_dir) assert not sum(len(v) for v in pull_changes.values()) - assert next((f for f in push_changes['added'] if f['path'] == f_added), None) - assert next((f for f in push_changes['removed'] if f['path'] == f_removed), None) - assert next((f for f in push_changes['updated'] if f['path'] == f_updated), None) + assert next((f for f in push_changes["added"] if f["path"] == f_added), None) + assert next((f for f in push_changes["removed"] if f["path"] == f_removed), None) + assert next((f for f in push_changes["updated"] if f["path"] == f_updated), None) # renamed file will result in removed + added file - assert next((f for f in push_changes['removed'] if f['path'] == f_renamed), None) - assert next((f for f in push_changes['added'] if f['path'] == 'renamed.txt'), None) - assert not pull_changes['renamed'] # not supported + assert next((f for f in push_changes["removed"] if f["path"] == f_renamed), None) + assert next((f for f in push_changes["added"] if f["path"] == "renamed.txt"), None) + assert not pull_changes["renamed"] # not supported mc.push_project(project_dir) project_info = mc.project_info(project) - assert project_info['version'] == 'v2' - assert not next((f for f in project_info['files'] if f['path'] == f_removed), None) - assert not next((f for f in project_info['files'] if f['path'] == f_renamed), None) - assert next((f for f in project_info['files'] if f['path'] == 'renamed.txt'), None) - assert next((f for f in project_info['files'] if f['path'] == f_added), None) - f_remote_checksum = next((f['checksum'] for f in project_info['files'] if f['path'] == f_updated), None) + assert project_info["version"] == "v2" + assert not next((f for f in project_info["files"] if f["path"] == f_removed), None) + assert not next((f for f in project_info["files"] if f["path"] == f_renamed), None) + assert next((f for f in project_info["files"] if f["path"] == "renamed.txt"), None) + assert next((f for f in project_info["files"] if f["path"] == f_added), None) + f_remote_checksum = next((f["checksum"] for f in project_info["files"] if f["path"] == f_updated), None) assert generate_checksum(os.path.join(project_dir, f_updated)) == f_remote_checksum mp = MerginProject(project_dir) - assert len(project_info['files']) == len(mp.inspect_files()) + assert len(project_info["files"]) == len(mp.inspect_files()) project_versions = mc.project_versions(project) assert len(project_versions) == 2 - f_change = next((f for f in project_versions[-1]['changes']['updated'] if f['path'] == f_updated), None) - assert 'origin_checksum' not in f_change # internal client info + f_change = next((f for f in project_versions[-1]["changes"]["updated"] if f["path"] == f_updated), None) + assert "origin_checksum" not in f_change # internal client info # test parallel changes - with open(os.path.join(project_dir_2, f_updated), 'w') as f: - f.write('Make some conflict') + with open(os.path.join(project_dir_2, f_updated), "w") as f: + f.write("Make some conflict") f_conflict_checksum = generate_checksum(os.path.join(project_dir_2, f_updated)) # not at latest server version - with pytest.raises(ClientError, match='Please update your local copy'): + with pytest.raises(ClientError, match="Please update your local copy"): mc.push_project(project_dir_2) # check changes in project_dir_2 before applied pull_changes, push_changes, _ = mc.project_status(project_dir_2) - assert next((f for f in pull_changes['added'] if f['path'] == f_added), None) - assert next((f for f in pull_changes['removed'] if f['path'] == f_removed), None) - assert next((f for f in pull_changes['updated'] if f['path'] == f_updated), None) - assert next((f for f in pull_changes['removed'] if f['path'] == f_renamed), None) - assert next((f for f in pull_changes['added'] if f['path'] == 'renamed.txt'), None) + assert next((f for f in pull_changes["added"] if f["path"] == f_added), None) + assert next((f for f in pull_changes["removed"] if f["path"] == f_removed), None) + assert next((f for f in pull_changes["updated"] if f["path"] == f_updated), None) + assert next((f for f in pull_changes["removed"] if f["path"] == f_renamed), None) + assert next((f for f in pull_changes["added"] if f["path"] == "renamed.txt"), None) mc.pull_project(project_dir_2) assert os.path.exists(os.path.join(project_dir_2, f_added)) assert not os.path.exists(os.path.join(project_dir_2, f_removed)) assert not os.path.exists(os.path.join(project_dir_2, f_renamed)) - assert os.path.exists(os.path.join(project_dir_2, 'renamed.txt')) + assert os.path.exists(os.path.join(project_dir_2, "renamed.txt")) assert os.path.exists(os.path.join(project_dir_2, conflicted_copy_file_name(f_updated, API_USER, 1))) - assert generate_checksum(os.path.join(project_dir_2, conflicted_copy_file_name(f_updated, API_USER, 1))) == f_conflict_checksum + assert ( + generate_checksum(os.path.join(project_dir_2, conflicted_copy_file_name(f_updated, API_USER, 1))) + == f_conflict_checksum + ) assert generate_checksum(os.path.join(project_dir_2, f_updated)) == f_remote_checksum @@ -233,7 +233,7 @@ def test_cancel_push(mc): finished. """ test_project = "test_cancel_push" - project = API_USER + '/' + test_project + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project + "_3") # primary project dir for updates project_dir_2 = os.path.join(TMP_DIR, test_project + "_4") cleanup(mc, project, [project_dir, project_dir_2]) @@ -242,19 +242,19 @@ def test_cancel_push(mc): mc.create_project_and_push(test_project, project_dir) # modify the project (add, update) - f_added = 'new.txt' - with open(os.path.join(project_dir, f_added), 'w') as f: + f_added = "new.txt" + with open(os.path.join(project_dir, f_added), "w") as f: f.write("new file") f_updated = "test3.txt" modification = "Modified" - with open(os.path.join(project_dir, f_updated), 'w') as f: + with open(os.path.join(project_dir, f_updated), "w") as f: f.write(modification) # check changes before applied pull_changes, push_changes, _ = mc.project_status(project_dir) assert not sum(len(v) for v in pull_changes.values()) - assert next((f for f in push_changes['added'] if f['path'] == f_added), None) - assert next((f for f in push_changes['updated'] if f['path'] == f_updated), None) + assert next((f for f in push_changes["added"] if f["path"] == f_added), None) + assert next((f for f in push_changes["updated"] if f["path"] == f_updated), None) # start pushing and then cancel the job job = push_project_async(mc, project_dir) @@ -266,38 +266,38 @@ def test_cancel_push(mc): # download the project to a different directory and check the version and content mc.download_project(project, project_dir_2) mp = MerginProject(project_dir_2) - assert mp.metadata["version"] == 'v2' + assert mp.metadata["version"] == "v2" assert os.path.exists(os.path.join(project_dir_2, f_added)) - with open(os.path.join(project_dir_2, f_updated), 'r') as f: + with open(os.path.join(project_dir_2, f_updated), "r") as f: assert f.read() == modification def test_ignore_files(mc): - test_project = 'test_blacklist' - project = API_USER + '/' + test_project + test_project = "test_blacklist" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates cleanup(mc, project, [project_dir]) # create remote project shutil.copytree(TEST_DATA_DIR, project_dir) - shutil.copy(os.path.join(project_dir, 'test.qgs'), os.path.join(project_dir, 'test.qgs~')) + shutil.copy(os.path.join(project_dir, "test.qgs"), os.path.join(project_dir, "test.qgs~")) mc.create_project_and_push(test_project, project_dir) project_info = mc.project_info(project) - assert not next((f for f in project_info['files'] if f['path'] == 'test.qgs~'), None) + assert not next((f for f in project_info["files"] if f["path"] == "test.qgs~"), None) - with open(os.path.join(project_dir, '.directory'), 'w') as f: - f.write('test') + with open(os.path.join(project_dir, ".directory"), "w") as f: + f.write("test") mc.push_project(project_dir) - assert not next((f for f in project_info['files'] if f['path'] == '.directory'), None) + assert not next((f for f in project_info["files"] if f["path"] == ".directory"), None) def test_sync_diff(mc): - test_project = f'test_sync_diff' - project = API_USER + '/' + test_project + test_project = f"test_sync_diff" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates - project_dir_2 = os.path.join(TMP_DIR, test_project + '_2') # concurrent project dir with no changes - project_dir_3 = os.path.join(TMP_DIR, test_project + '_3') # concurrent project dir with local changes + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir with no changes + project_dir_3 = os.path.join(TMP_DIR, test_project + "_3") # concurrent project dir with local changes cleanup(mc, project, [project_dir, project_dir_2, project_dir_3]) # create remote project @@ -310,68 +310,70 @@ def test_sync_diff(mc): # test push changes with diffs: mp = MerginProject(project_dir) - f_updated = 'base.gpkg' + f_updated = "base.gpkg" # step 1) base.gpkg updated to inserted_1_A (inserted A feature) shutil.move(mp.fpath(f_updated), mp.fpath_meta(f_updated)) # make local copy for changeset calculation - shutil.copy(mp.fpath('inserted_1_A.gpkg'), mp.fpath(f_updated)) + shutil.copy(mp.fpath("inserted_1_A.gpkg"), mp.fpath(f_updated)) mc.push_project(project_dir) - mp.geodiff.create_changeset(mp.fpath(f_updated), mp.fpath_meta(f_updated), mp.fpath_meta('push_diff')) - assert not mp.geodiff.has_changes(mp.fpath_meta('push_diff')) + mp.geodiff.create_changeset(mp.fpath(f_updated), mp.fpath_meta(f_updated), mp.fpath_meta("push_diff")) + assert not mp.geodiff.has_changes(mp.fpath_meta("push_diff")) # step 2) base.gpkg updated to inserted_1_A_mod (modified 2 features) shutil.move(mp.fpath(f_updated), mp.fpath_meta(f_updated)) - shutil.copy(mp.fpath('inserted_1_A_mod.gpkg'), mp.fpath(f_updated)) + shutil.copy(mp.fpath("inserted_1_A_mod.gpkg"), mp.fpath(f_updated)) # introduce some other changes - f_removed = 'inserted_1_B.gpkg' + f_removed = "inserted_1_B.gpkg" os.remove(mp.fpath(f_removed)) - f_renamed = 'test_dir/modified_1_geom.gpkg' - shutil.move(mp.fpath(f_renamed), mp.fpath('renamed.gpkg')) + f_renamed = "test_dir/modified_1_geom.gpkg" + shutil.move(mp.fpath(f_renamed), mp.fpath("renamed.gpkg")) mc.push_project(project_dir) # check project after push project_info = mc.project_info(project) - assert project_info['version'] == 'v3' - f_remote = next((f for f in project_info['files'] if f['path'] == f_updated), None) - assert next((f for f in project_info['files'] if f['path'] == 'renamed.gpkg'), None) - assert not next((f for f in project_info['files'] if f['path'] == f_removed), None) + assert project_info["version"] == "v3" + f_remote = next((f for f in project_info["files"] if f["path"] == f_updated), None) + assert next((f for f in project_info["files"] if f["path"] == "renamed.gpkg"), None) + assert not next((f for f in project_info["files"] if f["path"] == f_removed), None) assert not os.path.exists(mp.fpath_meta(f_removed)) - assert 'diff' in f_remote - assert os.path.exists(mp.fpath_meta('renamed.gpkg')) + assert "diff" in f_remote + assert os.path.exists(mp.fpath_meta("renamed.gpkg")) # pull project in different directory mp2 = MerginProject(project_dir_2) mc.pull_project(project_dir_2) - mp2.geodiff.create_changeset(mp.fpath(f_updated), mp2.fpath(f_updated), mp2.fpath_meta('diff')) - assert not mp2.geodiff.has_changes(mp2.fpath_meta('diff')) + mp2.geodiff.create_changeset(mp.fpath(f_updated), mp2.fpath(f_updated), mp2.fpath_meta("diff")) + assert not mp2.geodiff.has_changes(mp2.fpath_meta("diff")) # introduce conflict local change (inserted B feature to base) mp3 = MerginProject(project_dir_3) - shutil.copy(mp3.fpath('inserted_1_B.gpkg'), mp3.fpath(f_updated)) - checksum = generate_checksum(mp3.fpath('inserted_1_B.gpkg')) + shutil.copy(mp3.fpath("inserted_1_B.gpkg"), mp3.fpath(f_updated)) + checksum = generate_checksum(mp3.fpath("inserted_1_B.gpkg")) mc.pull_project(project_dir_3) - assert not os.path.exists(mp3.fpath('base.gpkg_conflict_copy')) + assert not os.path.exists(mp3.fpath("base.gpkg_conflict_copy")) # push new changes from project_3 and pull in original project mc.push_project(project_dir_3) mc.pull_project(project_dir) - mp3.geodiff.create_changeset(mp.fpath(f_updated), mp3.fpath(f_updated), mp.fpath_meta('diff')) - assert not mp3.geodiff.has_changes(mp.fpath_meta('diff')) + mp3.geodiff.create_changeset(mp.fpath(f_updated), mp3.fpath(f_updated), mp.fpath_meta("diff")) + assert not mp3.geodiff.has_changes(mp.fpath_meta("diff")) def test_list_of_push_changes(mc): - PUSH_CHANGES_SUMMARY = "{'base.gpkg': {'geodiff_summary': [{'table': 'simple', 'insert': 1, 'update': 0, 'delete': 0}]}}" + PUSH_CHANGES_SUMMARY = ( + "{'base.gpkg': {'geodiff_summary': [{'table': 'simple', 'insert': 1, 'update': 0, 'delete': 0}]}}" + ) - test_project = 'test_list_of_push_changes' - project = API_USER + '/' + test_project + test_project = "test_list_of_push_changes" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates cleanup(mc, project, [project_dir]) shutil.copytree(TEST_DATA_DIR, project_dir) mc.create_project_and_push(test_project, project_dir) - f_updated = 'base.gpkg' + f_updated = "base.gpkg" mp = MerginProject(project_dir) - shutil.copy(mp.fpath('inserted_1_A.gpkg'), mp.fpath(f_updated)) + shutil.copy(mp.fpath("inserted_1_A.gpkg"), mp.fpath(f_updated)) mc._auth_session["expire"] = datetime.now().replace(tzinfo=pytz.utc) - timedelta(days=1) pull_changes, push_changes, push_changes_summary = mc.project_status(project_dir) assert str(push_changes_summary) == PUSH_CHANGES_SUMMARY @@ -379,8 +381,8 @@ def test_list_of_push_changes(mc): def test_token_renewal(mc): """Test token regeneration in case it has expired.""" - test_project = 'test_token_renewal' - project = API_USER + '/' + test_project + test_project = "test_token_renewal" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates cleanup(mc, project, [project_dir]) @@ -394,8 +396,8 @@ def test_token_renewal(mc): def test_force_gpkg_update(mc): - test_project = 'test_force_update' - project = API_USER + '/' + test_project + test_project = "test_force_update" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates cleanup(mc, project, [project_dir]) @@ -405,13 +407,15 @@ def test_force_gpkg_update(mc): # test push changes with force gpkg file upload: mp = MerginProject(project_dir) - f_updated = 'base.gpkg' + f_updated = "base.gpkg" checksum = generate_checksum(mp.fpath(f_updated)) # base.gpkg updated to modified_schema (inserted new column) - shutil.move(mp.fpath(f_updated), mp.fpath_meta(f_updated)) # make local copy for changeset calculation (which will fail) - shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, 'modified_schema.gpkg'), mp.fpath(f_updated)) - shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, 'modified_schema.gpkg-wal'), mp.fpath(f_updated + '-wal')) + shutil.move( + mp.fpath(f_updated), mp.fpath_meta(f_updated) + ) # make local copy for changeset calculation (which will fail) + shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, "modified_schema.gpkg"), mp.fpath(f_updated)) + shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, "modified_schema.gpkg-wal"), mp.fpath(f_updated + "-wal")) mc.push_project(project_dir) # by this point local file has been updated (changes committed from wal) updated_checksum = generate_checksum(mp.fpath(f_updated)) @@ -419,16 +423,16 @@ def test_force_gpkg_update(mc): # check project after push project_info = mc.project_info(project) - assert project_info['version'] == 'v2' - f_remote = next((f for f in project_info['files'] if f['path'] == f_updated), None) - assert 'diff' not in f_remote + assert project_info["version"] == "v2" + f_remote = next((f for f in project_info["files"] if f["path"] == f_updated), None) + assert "diff" not in f_remote def test_new_project_sync(mc): - """ Create a new project, download it, add a file and then do sync - it should not fail """ + """Create a new project, download it, add a file and then do sync - it should not fail""" - test_project = 'test_new_project_sync' - project = API_USER + '/' + test_project + test_project = "test_new_project_sync" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates cleanup(mc, project, [project_dir]) @@ -452,14 +456,14 @@ def test_new_project_sync(mc): def test_missing_basefile_pull(mc): - """ Test pull of a project where basefile of a .gpkg is missing for some reason + """Test pull of a project where basefile of a .gpkg is missing for some reason (it should gracefully handle it by downloading the missing basefile) """ - test_project = 'test_missing_basefile_pull' - project = API_USER + '/' + test_project + test_project = "test_missing_basefile_pull" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates - project_dir_2 = os.path.join(TMP_DIR, test_project + '_2') # concurrent project dir + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir test_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), test_project) cleanup(mc, project, [project_dir, project_dir_2]) @@ -477,7 +481,7 @@ def test_missing_basefile_pull(mc): shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_B.gpkg"), os.path.join(project_dir, "base.gpkg")) # remove the basefile to simulate the issue - os.remove(os.path.join(project_dir, '.mergin', 'base.gpkg')) + os.remove(os.path.join(project_dir, ".mergin", "base.gpkg")) # try to sync again -- it should not crash mc.pull_project(project_dir) @@ -485,12 +489,12 @@ def test_missing_basefile_pull(mc): def test_empty_file_in_subdir(mc): - """ Test pull of a project where there is an empty file in a sub-directory """ + """Test pull of a project where there is an empty file in a sub-directory""" - test_project = 'test_empty_file_in_subdir' - project = API_USER + '/' + test_project + test_project = "test_empty_file_in_subdir" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir for updates - project_dir_2 = os.path.join(TMP_DIR, test_project + '_2') # concurrent project dir + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir test_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), test_project) cleanup(mc, project, [project_dir, project_dir_2]) @@ -500,22 +504,21 @@ def test_empty_file_in_subdir(mc): # try to check out the project mc.download_project(project, project_dir_2) - assert os.path.exists(os.path.join(project_dir_2, 'subdir', 'empty.txt')) + assert os.path.exists(os.path.join(project_dir_2, "subdir", "empty.txt")) # add another empty file in a different subdir - os.mkdir(os.path.join(project_dir, 'subdir2')) - shutil.copy(os.path.join(project_dir, 'subdir', 'empty.txt'), - os.path.join(project_dir, 'subdir2', 'empty2.txt')) + os.mkdir(os.path.join(project_dir, "subdir2")) + shutil.copy(os.path.join(project_dir, "subdir", "empty.txt"), os.path.join(project_dir, "subdir2", "empty2.txt")) mc.push_project(project_dir) # check that pull works fine mc.pull_project(project_dir_2) - assert os.path.exists(os.path.join(project_dir_2, 'subdir2', 'empty2.txt')) + assert os.path.exists(os.path.join(project_dir_2, "subdir2", "empty2.txt")) def test_clone_project(mc): - test_project = 'test_clone_project' - test_project_fullname = API_USER + '/' + test_project + test_project = "test_clone_project" + test_project_fullname = API_USER + "/" + test_project # cleanups project_dir = os.path.join(TMP_DIR, test_project) @@ -523,23 +526,23 @@ def test_clone_project(mc): # create new (empty) project on server mc.create_project(test_project) - projects = mc.projects_list(flag='created') - assert any(p for p in projects if p['name'] == test_project and p['namespace'] == API_USER) + projects = mc.projects_list(flag="created") + assert any(p for p in projects if p["name"] == test_project and p["namespace"] == API_USER) cloned_project_name = test_project + "_cloned" # cleanup cloned project cloned_project_dir = os.path.join(TMP_DIR, cloned_project_name) - cleanup(mc, API_USER + '/' + cloned_project_name, [cloned_project_dir]) + cleanup(mc, API_USER + "/" + cloned_project_name, [cloned_project_dir]) # clone project mc.clone_project(test_project_fullname, cloned_project_name, API_USER) - projects = mc.projects_list(flag='created') - assert any(p for p in projects if p['name'] == cloned_project_name and p['namespace'] == API_USER) + projects = mc.projects_list(flag="created") + assert any(p for p in projects if p["name"] == cloned_project_name and p["namespace"] == API_USER) def test_set_read_write_access(mc): - test_project = 'test_set_read_write_access' - test_project_fullname = API_USER + '/' + test_project + test_project = "test_set_read_write_access" + test_project_fullname = API_USER + "/" + test_project # cleanups project_dir = os.path.join(TMP_DIR, test_project, API_USER) @@ -550,16 +553,16 @@ def test_set_read_write_access(mc): # Add writer access to another client project_info = get_project_info(mc, API_USER, test_project) - access = project_info['access'] - access['writersnames'].append(API_USER2) - access['readersnames'].append(API_USER2) + access = project_info["access"] + access["writersnames"].append(API_USER2) + access["readersnames"].append(API_USER2) mc.set_project_access(test_project_fullname, access) # check access project_info = get_project_info(mc, API_USER, test_project) - access = project_info['access'] - assert API_USER2 in access['writersnames'] - assert API_USER2 in access['readersnames'] + access = project_info["access"] + assert API_USER2 in access["writersnames"] + assert API_USER2 in access["readersnames"] def test_available_storage_validation(mc): @@ -567,8 +570,8 @@ def test_available_storage_validation(mc): Testing of storage limit - applies to user pushing changes into own project (namespace matching username). This test also tests giving read and write access to another user. Additionally tests also uploading of big file. """ - test_project = 'test_available_storage_validation' - test_project_fullname = API_USER + '/' + test_project + test_project = "test_available_storage_validation" + test_project_fullname = API_USER + "/" + test_project # cleanups project_dir = os.path.join(TMP_DIR, test_project, API_USER) @@ -582,7 +585,7 @@ def test_available_storage_validation(mc): # get user_info about storage capacity user_info = mc.user_info() - storage_remaining = user_info['storage'] - user_info['disk_usage'] + storage_remaining = user_info["storage"] - user_info["disk_usage"] # generate dummy data (remaining storage + extra 1024b) dummy_data_path = project_dir + "/data" @@ -601,8 +604,8 @@ def test_available_storage_validation(mc): # Expecting empty project project_info = get_project_info(mc, API_USER, test_project) - assert project_info['version'] == 'v0' - assert project_info['disk_usage'] == 0 + assert project_info["version"] == "v0" + assert project_info["disk_usage"] == 0 def test_available_storage_validation2(mc, mc2): @@ -616,8 +619,8 @@ def test_available_storage_validation2(mc, mc2): - API_USER2's free storage >= API_USER's free storage + 1024b (size of changes to be pushed) - both accounts should ideally have a free plan """ - test_project = 'test_available_storage_validation2' - test_project_fullname = API_USER2 + '/' + test_project + test_project = "test_available_storage_validation2" + test_project_fullname = API_USER2 + "/" + test_project # cleanups project_dir = os.path.join(TMP_DIR, test_project, API_USER) @@ -629,9 +632,9 @@ def test_available_storage_validation2(mc, mc2): # Add writer access to another client project_info = get_project_info(mc2, API_USER2, test_project) - access = project_info['access'] - access['writersnames'].append(API_USER) - access['readersnames'].append(API_USER) + access = project_info["access"] + access["writersnames"].append(API_USER) + access["readersnames"].append(API_USER) mc2.set_project_access(test_project_fullname, access) # download project @@ -639,7 +642,7 @@ def test_available_storage_validation2(mc, mc2): # get user_info about storage capacity user_info = mc.user_info() - storage_remaining = user_info['storage'] - user_info['disk_usage'] + storage_remaining = user_info["storage"] - user_info["disk_usage"] # generate dummy data (remaining storage + extra 1024b) dummy_data_path = project_dir + "/data" @@ -666,8 +669,8 @@ def get_project_info(mc, namespace, project_name): :param project_name: project's name :return: dict with project info """ - projects = mc.projects_list(flag='created') - test_project_list = [p for p in projects if p['name'] == project_name and p['namespace'] == namespace] + projects = mc.projects_list(flag="created") + test_project_list = [p for p in projects if p["name"] == project_name and p["namespace"] == namespace] assert len(test_project_list) == 1 return test_project_list[0] @@ -678,16 +681,13 @@ def _generate_big_file(filepath, size): :param filepath: full filepath :param size: the size in bytes """ - with open(filepath, 'wb') as fout: + with open(filepath, "wb") as fout: fout.write(b"\0" * size) def test_get_projects_by_name(mc): - """ Test server 'bulk' endpoint for projects' info""" - test_projects = { - "projectA": f"{API_USER}/projectA", - "projectB": f"{API_USER}/projectB" - } + """Test server 'bulk' endpoint for projects' info""" + test_projects = {"projectA": f"{API_USER}/projectA", "projectB": f"{API_USER}/projectB"} for name, full_name in test_projects.items(): cleanup(mc, full_name, []) @@ -698,17 +698,17 @@ def test_get_projects_by_name(mc): for name, full_name in test_projects.items(): assert full_name in resp assert resp[full_name]["name"] == name - assert resp[full_name]["version"] == 'v0' + assert resp[full_name]["version"] == "v0" def test_download_versions(mc): - test_project = 'test_download' - project = API_USER + '/' + test_project + test_project = "test_download" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # download dirs - project_dir_v1 = os.path.join(TMP_DIR, test_project + '_v1') - project_dir_v2 = os.path.join(TMP_DIR, test_project + '_v2') - project_dir_v3 = os.path.join(TMP_DIR, test_project + '_v3') + project_dir_v1 = os.path.join(TMP_DIR, test_project + "_v1") + project_dir_v2 = os.path.join(TMP_DIR, test_project + "_v2") + project_dir_v3 = os.path.join(TMP_DIR, test_project + "_v3") cleanup(mc, project, [project_dir, project_dir_v1, project_dir_v2, project_dir_v3]) # create remote project @@ -716,25 +716,25 @@ def test_download_versions(mc): mc.create_project_and_push(test_project, project_dir) # create new version - v2 - f_added = 'new.txt' - with open(os.path.join(project_dir, f_added), 'w') as f: - f.write('new file') + f_added = "new.txt" + with open(os.path.join(project_dir, f_added), "w") as f: + f.write("new file") mc.push_project(project_dir) project_info = mc.project_info(project) - assert project_info['version'] == 'v2' + assert project_info["version"] == "v2" - mc.download_project(project, project_dir_v1, 'v1') - assert os.path.exists(os.path.join(project_dir_v1, 'base.gpkg')) + mc.download_project(project, project_dir_v1, "v1") + assert os.path.exists(os.path.join(project_dir_v1, "base.gpkg")) assert not os.path.exists(os.path.join(project_dir_v2, f_added)) # added only in v2 - mc.download_project(project, project_dir_v2, 'v2') + mc.download_project(project, project_dir_v2, "v2") assert os.path.exists(os.path.join(project_dir_v2, f_added)) - assert os.path.exists(os.path.join(project_dir_v1, 'base.gpkg')) # added in v1 but still present in v2 + assert os.path.exists(os.path.join(project_dir_v1, "base.gpkg")) # added in v1 but still present in v2 # try to download not-existing version with pytest.raises(ClientError): - mc.download_project(project, project_dir_v3, 'v3') + mc.download_project(project, project_dir_v3, "v3") def test_paginated_project_list(mc): @@ -751,7 +751,7 @@ def test_paginated_project_list(mc): sorted_test_names = [n for n in sorted(test_projects.keys())] resp = mc.paginated_projects_list( - flag='created', name="test_paginated", page=1, per_page=10, order_params="name_asc" + flag="created", name="test_paginated", page=1, per_page=10, order_params="name_asc" ) projects = resp["projects"] count = resp["count"] @@ -761,12 +761,12 @@ def test_paginated_project_list(mc): assert project["name"] == sorted_test_names[i] resp = mc.paginated_projects_list( - flag='created', name="test_paginated", page=2, per_page=2, order_params="name_asc" + flag="created", name="test_paginated", page=2, per_page=2, order_params="name_asc" ) projects = resp["projects"] assert len(projects) == 2 for i, project in enumerate(projects): - assert project["name"] == sorted_test_names[i+2] + assert project["name"] == sorted_test_names[i + 2] def test_missing_local_file_pull(mc): @@ -774,7 +774,7 @@ def test_missing_local_file_pull(mc): test_project = "test_dir" file_to_remove = "test2.txt" - project = API_USER + '/' + test_project + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project + "_5") # primary project dir for updates project_dir_2 = os.path.join(TMP_DIR, test_project + "_6") # concurrent project dir test_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "test_data", test_project) @@ -803,15 +803,15 @@ def test_logging(mc): mc.log.info("Test info log...") # remove the Null handler and set the env variable with the global log file path mc.log.handlers = [] - os.environ['MERGIN_CLIENT_LOG'] = os.path.join(TMP_DIR, 'global-mergin-log.txt') - assert os.environ.get('MERGIN_CLIENT_LOG', None) is not None - token = mc._auth_session['token'] + os.environ["MERGIN_CLIENT_LOG"] = os.path.join(TMP_DIR, "global-mergin-log.txt") + assert os.environ.get("MERGIN_CLIENT_LOG", None) is not None + token = mc._auth_session["token"] mc1 = MerginClient(mc.url, auth_token=token) assert isinstance(mc1.log.handlers[0], logging.FileHandler) mc1.log.info("Test log info to the log file...") # cleanup mc.log.handlers = [] - del os.environ['MERGIN_CLIENT_LOG'] + del os.environ["MERGIN_CLIENT_LOG"] def test_server_compatibility(mc): @@ -820,7 +820,7 @@ def test_server_compatibility(mc): def create_versioned_project(mc, project_name, project_dir, updated_file, remove=True, overwrite=False): - project = API_USER + '/' + project_name + project = API_USER + "/" + project_name cleanup(mc, project, [project_dir]) # create remote project @@ -830,7 +830,11 @@ def create_versioned_project(mc, project_name, project_dir, updated_file, remove mp = MerginProject(project_dir) # create versions 2-4 - changes = ("inserted_1_A.gpkg", "inserted_1_A_mod.gpkg", "inserted_1_B.gpkg",) + changes = ( + "inserted_1_A.gpkg", + "inserted_1_A_mod.gpkg", + "inserted_1_B.gpkg", + ) for change in changes: shutil.copy(mp.fpath(change), mp.fpath(updated_file)) mc.push_project(project_dir) @@ -842,16 +846,16 @@ def create_versioned_project(mc, project_name, project_dir, updated_file, remove # create version with forced overwrite (broken history) if overwrite: shutil.move(mp.fpath(updated_file), mp.fpath_meta(updated_file)) - shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, 'modified_schema.gpkg'), mp.fpath(updated_file)) - shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, 'modified_schema.gpkg-wal'), mp.fpath(updated_file + '-wal')) + shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, "modified_schema.gpkg"), mp.fpath(updated_file)) + shutil.copy(os.path.join(CHANGED_SCHEMA_DIR, "modified_schema.gpkg-wal"), mp.fpath(updated_file + "-wal")) mc.push_project(project_dir) return mp def test_get_versions_with_file_changes(mc): """Test getting versions where the file was changed.""" - test_project = 'test_file_modified_versions' - project = API_USER + '/' + test_project + test_project = "test_file_modified_versions" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) f_updated = "base.gpkg" @@ -884,8 +888,8 @@ def check_gpkg_same_content(mergin_project, gpkg_path_1, gpkg_path_2): def test_download_file(mc): """Test downloading single file at specified versions.""" - test_project = 'test_download_file' - project = API_USER + '/' + test_project + test_project = "test_download_file" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) f_updated = "base.gpkg" @@ -911,8 +915,8 @@ def test_download_file(mc): def test_download_diffs(mc): """Test download diffs for a project file between specified project versions.""" - test_project = 'test_download_diffs' - project = API_USER + '/' + test_project + test_project = "test_download_diffs" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) download_dir = os.path.join(project_dir, "diffs") # project for downloading files at various versions f_updated = "base.gpkg" @@ -930,7 +934,7 @@ def test_download_diffs(mc): assert mp.geodiff.changes_count(diff_file) == 1 changes_file = diff_file + ".changes1-2" mp.geodiff.list_changes_summary(diff_file, changes_file) - with open(changes_file, 'r') as f: + with open(changes_file, "r") as f: changes = json.loads(f.read())["geodiff_summary"][0] assert changes["insert"] == 1 assert changes["update"] == 0 @@ -939,7 +943,7 @@ def test_download_diffs(mc): mc.get_file_diff(project_dir, f_updated, diff_file, "v2", "v4") changes_file = diff_file + ".changes2-4" mp.geodiff.list_changes_summary(diff_file, changes_file) - with open(changes_file, 'r') as f: + with open(changes_file, "r") as f: changes = json.loads(f.read())["geodiff_summary"][0] assert changes["insert"] == 0 assert changes["update"] == 1 @@ -956,10 +960,10 @@ def test_download_diffs(mc): def test_modify_project_permissions(mc): - test_project = 'test_project' - project = API_USER + '/' + test_project + test_project = "test_project" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) - download_dir = os.path.join(TMP_DIR, 'download', test_project) + download_dir = os.path.join(TMP_DIR, "download", test_project) cleanup(mc, project, [project_dir, download_dir]) # prepare local project @@ -970,9 +974,9 @@ def test_modify_project_permissions(mc): # check basic metadata about created project project_info = mc.project_info(project) - assert project_info['version'] == 'v1' - assert project_info['name'] == test_project - assert project_info['namespace'] == API_USER + assert project_info["version"] == "v1" + assert project_info["name"] == test_project + assert project_info["namespace"] == API_USER permissions = mc.project_user_permissions(project) assert permissions["owners"] == [API_USER] @@ -993,65 +997,69 @@ def test_modify_project_permissions(mc): def _use_wal(db_file): - """ Ensures that sqlite database is using WAL journal mode """ + """Ensures that sqlite database is using WAL journal mode""" con = sqlite3.connect(db_file) cursor = con.cursor() - cursor.execute('PRAGMA journal_mode=wal;') + cursor.execute("PRAGMA journal_mode=wal;") def _create_test_table(db_file): - """ Creates a table called 'test' in sqlite database. Useful to simulate change of database schema. """ + """Creates a table called 'test' in sqlite database. Useful to simulate change of database schema.""" con = sqlite3.connect(db_file) cursor = con.cursor() - cursor.execute('CREATE TABLE test (fid SERIAL, txt TEXT);') - cursor.execute('INSERT INTO test VALUES (123, \'hello\');') - cursor.execute('COMMIT;') + cursor.execute("CREATE TABLE test (fid SERIAL, txt TEXT);") + cursor.execute("INSERT INTO test VALUES (123, 'hello');") + cursor.execute("COMMIT;") def _create_spatial_table(db_file): - """ Creates a spatial table called 'test' in sqlite database. Useful to simulate change of database schema. """ + """Creates a spatial table called 'test' in sqlite database. Useful to simulate change of database schema.""" con = sqlite3.connect(db_file) cursor = con.cursor() - cursor.execute('CREATE TABLE geo_test (fid SERIAL, geometry POINT NOT NULL, txt TEXT);') - cursor.execute('INSERT INTO gpkg_contents VALUES (\'geo_test\', \'features\',\'description\',\'geo_test\',\'2019-06-18T14:52:50.928Z\',-1.08892,0.0424077,-0.363885,0.562244,4326);') - cursor.execute('INSERT INTO gpkg_geometry_columns VALUES (\'geo_test\',\'geometry\',\'POINT\',4326, 0, 0 )') - cursor.execute('COMMIT;') + cursor.execute("CREATE TABLE geo_test (fid SERIAL, geometry POINT NOT NULL, txt TEXT);") + cursor.execute( + "INSERT INTO gpkg_contents VALUES ('geo_test', 'features','description','geo_test','2019-06-18T14:52:50.928Z',-1.08892,0.0424077,-0.363885,0.562244,4326);" + ) + cursor.execute("INSERT INTO gpkg_geometry_columns VALUES ('geo_test','geometry','POINT',4326, 0, 0 )") + cursor.execute("COMMIT;") + def _delete_spatial_table(db_file): - """ Drops spatial table called 'test' in sqlite database. Useful to simulate change of database schema. """ + """Drops spatial table called 'test' in sqlite database. Useful to simulate change of database schema.""" con = sqlite3.connect(db_file) cursor = con.cursor() - cursor.execute('DROP TABLE poi;') - cursor.execute('DELETE FROM gpkg_geometry_columns WHERE table_name=\'poi\';') - cursor.execute('DELETE FROM gpkg_contents WHERE table_name=\'poi\';') - cursor.execute('COMMIT;') + cursor.execute("DROP TABLE poi;") + cursor.execute("DELETE FROM gpkg_geometry_columns WHERE table_name='poi';") + cursor.execute("DELETE FROM gpkg_contents WHERE table_name='poi';") + cursor.execute("COMMIT;") + def _check_test_table(db_file): - """ Checks whether the 'test' table exists and has one row - otherwise fails with an exception. """ - #con_verify = sqlite3.connect(db_file) - #cursor_verify = con_verify.cursor() - #cursor_verify.execute('select count(*) from test;') - #assert cursor_verify.fetchone()[0] == 1 - assert _get_table_row_count(db_file, 'test') == 1 + """Checks whether the 'test' table exists and has one row - otherwise fails with an exception.""" + # con_verify = sqlite3.connect(db_file) + # cursor_verify = con_verify.cursor() + # cursor_verify.execute('select count(*) from test;') + # assert cursor_verify.fetchone()[0] == 1 + assert _get_table_row_count(db_file, "test") == 1 def _get_table_row_count(db_file, table): con_verify = sqlite3.connect(db_file) cursor_verify = con_verify.cursor() - cursor_verify.execute('select count(*) from {};'.format(table)) + cursor_verify.execute("select count(*) from {};".format(table)) return cursor_verify.fetchone()[0] def _is_file_updated(filename, changes_dict): - """ checks whether a file is listed among updated files (for pull or push changes) """ - for f in changes_dict['updated']: - if f['path'] == filename: + """checks whether a file is listed among updated files (for pull or push changes)""" + for f in changes_dict["updated"]: + if f["path"] == filename: return True return False class AnotherSqliteConn: - """ This simulates another app (e.g. QGIS) having a connection open, potentially + """This simulates another app (e.g. QGIS) having a connection open, potentially with some active reader/writer. Note: we use a subprocess here instead of just using sqlite3 module from python @@ -1060,57 +1068,59 @@ class AnotherSqliteConn: use another process, things are fine. This is a limitation of how we package pygeodiff currently. """ + def __init__(self, filename): self.proc = subprocess.Popen( - ['python3', os.path.join(os.path.dirname(__file__), 'sqlite_con.py'), filename], + ["python3", os.path.join(os.path.dirname(__file__), "sqlite_con.py"), filename], stdin=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + ) def run(self, cmd): - self.proc.stdin.write(cmd.encode()+b'\n') + self.proc.stdin.write(cmd.encode() + b"\n") self.proc.stdin.flush() def close(self): - out,err = self.proc.communicate(b'stop\n') + out, err = self.proc.communicate(b"stop\n") if self.proc.returncode != 0: - raise ValueError("subprocess error:\n" + err.decode('utf-8')) + raise ValueError("subprocess error:\n" + err.decode("utf-8")) def test_push_gpkg_schema_change(mc): - """ Test that changes in GPKG get picked up if there were recent changes to it by another + """Test that changes in GPKG get picked up if there were recent changes to it by another client and at the same time geodiff fails to find changes (a new table is added) """ - test_project = 'test_push_gpkg_schema_change' - project = API_USER + '/' + test_project + test_project = "test_push_gpkg_schema_change" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) - test_gpkg = os.path.join(project_dir, 'test.gpkg') - test_gpkg_basefile = os.path.join(project_dir, '.mergin', 'test.gpkg') - project_dir_verify = os.path.join(TMP_DIR, test_project + '_verify') - test_gpkg_verify = os.path.join(project_dir_verify, 'test.gpkg') + test_gpkg = os.path.join(project_dir, "test.gpkg") + test_gpkg_basefile = os.path.join(project_dir, ".mergin", "test.gpkg") + project_dir_verify = os.path.join(TMP_DIR, test_project + "_verify") + test_gpkg_verify = os.path.join(project_dir_verify, "test.gpkg") cleanup(mc, project, [project_dir, project_dir_verify]) # create remote project os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) - #shutil.copytree(TEST_DATA_DIR, project_dir) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) + # shutil.copytree(TEST_DATA_DIR, project_dir) mc.create_project_and_push(test_project, project_dir) mp = MerginProject(project_dir) - mp.log.info(' // create changeset') + mp.log.info(" // create changeset") - mp.geodiff.create_changeset(mp.fpath('test.gpkg'), mp.fpath_meta('test.gpkg'), mp.fpath_meta('diff-0')) + mp.geodiff.create_changeset(mp.fpath("test.gpkg"), mp.fpath_meta("test.gpkg"), mp.fpath_meta("diff-0")) - mp.log.info(' // use wal') + mp.log.info(" // use wal") _use_wal(test_gpkg) - mp.log.info(' // make changes to DB') + mp.log.info(" // make changes to DB") # open a connection and keep it open (qgis does this with a pool of connections too) acon2 = AnotherSqliteConn(test_gpkg) - acon2.run('select count(*) from simple;') + acon2.run("select count(*) from simple;") # add a new table to ensure that geodiff will fail due to unsupported change # (this simulates an independent reader/writer like GDAL) @@ -1120,18 +1130,18 @@ def test_push_gpkg_schema_change(mc): with pytest.raises(sqlite3.OperationalError): _check_test_table(test_gpkg_basefile) - mp.log.info(' // create changeset (2)') + mp.log.info(" // create changeset (2)") # why already here there is wal recovery - it could be because of two sqlite libs linked in one executable # INDEED THAT WAS THE PROBLEM, now running geodiff 1.0 with shared sqlite lib seems to work fine. with pytest.raises(pygeodiff.geodifflib.GeoDiffLibError): - mp.geodiff.create_changeset(mp.fpath('test.gpkg'), mp.fpath_meta('test.gpkg'), mp.fpath_meta('diff-1')) + mp.geodiff.create_changeset(mp.fpath("test.gpkg"), mp.fpath_meta("test.gpkg"), mp.fpath_meta("diff-1")) _check_test_table(test_gpkg) with pytest.raises(sqlite3.OperationalError): _check_test_table(test_gpkg_basefile) - mp.log.info(' // push project') + mp.log.info(" // push project") # push pending changes (it should include addition of the new table) # at this point we still have an open sqlite connection to the GPKG, so checkpointing will not work correctly) @@ -1159,19 +1169,19 @@ def test_rebase_local_schema_change(mc, extra_connection): i.e. a conflict file is created with the content of the local changes. """ - test_project = 'test_rebase_local_schema_change' + test_project = "test_rebase_local_schema_change" if extra_connection: - test_project += '_extra_conn' - project = API_USER + '/' + test_project + test_project += "_extra_conn" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - project_dir_2 = os.path.join(TMP_DIR, test_project+'_2') # concurrent project dir - test_gpkg = os.path.join(project_dir, 'test.gpkg') - test_gpkg_basefile = os.path.join(project_dir, '.mergin', 'test.gpkg') + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir + test_gpkg = os.path.join(project_dir, "test.gpkg") + test_gpkg_basefile = os.path.join(project_dir, ".mergin", "test.gpkg") test_gpkg_conflict = conflicted_copy_file_name(test_gpkg, API_USER, 1) cleanup(mc, project, [project_dir, project_dir_2]) os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL, that's the more common and more difficult scenario mc.create_project_and_push(test_project, project_dir) @@ -1179,20 +1189,20 @@ def test_rebase_local_schema_change(mc, extra_connection): # open a connection and keep it open (qgis does this with a pool of connections too) con_extra = sqlite3.connect(test_gpkg) cursor_extra = con_extra.cursor() - cursor_extra.execute('select count(*) from simple;') + cursor_extra.execute("select count(*) from simple;") # Download project to the concurrent dir + add a feature + push a new version mc.download_project(project, project_dir_2) # download project to concurrent dir mp_2 = MerginProject(project_dir_2) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), mp_2.fpath('test.gpkg')) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), mp_2.fpath("test.gpkg")) mc.push_project(project_dir_2) # Change schema in the primary project dir _create_test_table(test_gpkg) pull_changes, push_changes, _ = mc.project_status(project_dir) - assert _is_file_updated('test.gpkg', pull_changes) - assert _is_file_updated('test.gpkg', push_changes) + assert _is_file_updated("test.gpkg", pull_changes) + assert _is_file_updated("test.gpkg", push_changes) assert not os.path.exists(test_gpkg_conflict) @@ -1212,9 +1222,9 @@ def test_rebase_local_schema_change(mc, extra_connection): # check that the local file + basefile contain the new row, and the conflict copy doesn't - assert _get_table_row_count(test_gpkg, 'simple') == 4 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 4 - assert _get_table_row_count(test_gpkg_conflict, 'simple') == 3 + assert _get_table_row_count(test_gpkg, "simple") == 4 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 4 + assert _get_table_row_count(test_gpkg_conflict, "simple") == 3 @pytest.mark.parametrize("extra_connection", [False, True]) @@ -1224,20 +1234,20 @@ def test_rebase_remote_schema_change(mc, extra_connection): i.e. a conflict file is created with the content of the local changes. """ - test_project = 'test_rebase_remote_schema_change' + test_project = "test_rebase_remote_schema_change" if extra_connection: - test_project += '_extra_conn' - project = API_USER + '/' + test_project + test_project += "_extra_conn" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - project_dir_2 = os.path.join(TMP_DIR, test_project+'_2') # concurrent project dir - test_gpkg = os.path.join(project_dir, 'test.gpkg') - test_gpkg_2 = os.path.join(project_dir_2, 'test.gpkg') - test_gpkg_basefile = os.path.join(project_dir, '.mergin', 'test.gpkg') + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir + test_gpkg = os.path.join(project_dir, "test.gpkg") + test_gpkg_2 = os.path.join(project_dir_2, "test.gpkg") + test_gpkg_basefile = os.path.join(project_dir, ".mergin", "test.gpkg") test_gpkg_conflict = conflicted_copy_file_name(test_gpkg, API_USER, 1) cleanup(mc, project, [project_dir, project_dir_2]) os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL, that's the more common and more difficult scenario mc.create_project_and_push(test_project, project_dir) @@ -1247,18 +1257,18 @@ def test_rebase_remote_schema_change(mc, extra_connection): mc.push_project(project_dir_2) # do changes in the local DB (added a row) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL if extra_connection: # open a connection and keep it open (qgis does this with a pool of connections too) con_extra = sqlite3.connect(test_gpkg) cursor_extra = con_extra.cursor() - cursor_extra.execute('select count(*) from simple;') + cursor_extra.execute("select count(*) from simple;") pull_changes, push_changes, _ = mc.project_status(project_dir) - assert _is_file_updated('test.gpkg', pull_changes) - assert _is_file_updated('test.gpkg', push_changes) + assert _is_file_updated("test.gpkg", pull_changes) + assert _is_file_updated("test.gpkg", push_changes) assert not os.path.exists(test_gpkg_conflict) @@ -1277,9 +1287,9 @@ def test_rebase_remote_schema_change(mc, extra_connection): # check that the local file + basefile don't contain the new row, and the conflict copy does - assert _get_table_row_count(test_gpkg, 'simple') == 3 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 3 - assert _get_table_row_count(test_gpkg_conflict, 'simple') == 4 + assert _get_table_row_count(test_gpkg, "simple") == 3 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 3 + assert _get_table_row_count(test_gpkg_conflict, "simple") == 4 @pytest.mark.parametrize("extra_connection", [False, True]) @@ -1289,42 +1299,42 @@ def test_rebase_success(mc, extra_connection): i.e. changes are merged together and no conflict files are created. """ - test_project = 'test_rebase_success' + test_project = "test_rebase_success" if extra_connection: - test_project += '_extra_conn' - project = API_USER + '/' + test_project + test_project += "_extra_conn" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - project_dir_2 = os.path.join(TMP_DIR, test_project+'_2') # concurrent project dir - test_gpkg = os.path.join(project_dir, 'test.gpkg') - test_gpkg_2 = os.path.join(project_dir_2, 'test.gpkg') - test_gpkg_basefile = os.path.join(project_dir, '.mergin', 'test.gpkg') + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir + test_gpkg = os.path.join(project_dir, "test.gpkg") + test_gpkg_2 = os.path.join(project_dir_2, "test.gpkg") + test_gpkg_basefile = os.path.join(project_dir, ".mergin", "test.gpkg") test_gpkg_conflict = conflicted_copy_file_name(test_gpkg, API_USER, 1) cleanup(mc, project, [project_dir, project_dir_2]) os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL, that's the more common and more difficult scenario mc.create_project_and_push(test_project, project_dir) # Download project to the concurrent dir + add a row + push a new version mc.download_project(project, project_dir_2) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), test_gpkg_2) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), test_gpkg_2) _use_wal(test_gpkg) # make sure we use WAL mc.push_project(project_dir_2) # do changes in the local DB (added a row) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_B.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_B.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL if extra_connection: # open a connection and keep it open (qgis does this with a pool of connections too) con_extra = sqlite3.connect(test_gpkg) cursor_extra = con_extra.cursor() - cursor_extra.execute('select count(*) from simple;') + cursor_extra.execute("select count(*) from simple;") pull_changes, push_changes, _ = mc.project_status(project_dir) - assert _is_file_updated('test.gpkg', pull_changes) - assert _is_file_updated('test.gpkg', push_changes) + assert _is_file_updated("test.gpkg", pull_changes) + assert _is_file_updated("test.gpkg", push_changes) assert not os.path.exists(test_gpkg_conflict) @@ -1334,8 +1344,8 @@ def test_rebase_success(mc, extra_connection): # check that the local file + basefile don't contain the new row, and the conflict copy does - assert _get_table_row_count(test_gpkg, 'simple') == 5 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 4 + assert _get_table_row_count(test_gpkg, "simple") == 5 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 4 def test_conflict_file_names(): @@ -1344,38 +1354,37 @@ def test_conflict_file_names(): """ data = [ - ('/home/test/geo.gpkg', 'jack', 10, '/home/test/geo (conflicted copy, jack v10).gpkg'), - ('/home/test/g.pkg', 'j', 0, '/home/test/g (conflicted copy, j v0).pkg'), - ('home/test/geo.gpkg', 'jack', 10, 'home/test/geo (conflicted copy, jack v10).gpkg'), - ('geo.gpkg', 'jack', 10, 'geo (conflicted copy, jack v10).gpkg'), - ('/home/../geo.gpkg', 'jack', 10, '/geo (conflicted copy, jack v10).gpkg'), - ('/home/./geo.gpkg', 'jack', 10, '/home/geo (conflicted copy, jack v10).gpkg'), - ('/home/test/geo.gpkg', '', 10, '/home/test/geo (conflicted copy, v10).gpkg'), - ('/home/test/geo.gpkg', 'jack', -1, '/home/test/geo (conflicted copy, jack v-1).gpkg'), - ('/home/test/geo.tar.gz', 'jack', 100, '/home/test/geo (conflicted copy, jack v100).tar.gz'), - ('', 'jack', 1, '' ), - ('/home/test/survey.qgs', 'jack', 10, '/home/test/survey (conflicted copy, jack v10).qgs~'), - ('/home/test/survey.QGZ', 'jack', 10, '/home/test/survey (conflicted copy, jack v10).QGZ~'), - ] + ("/home/test/geo.gpkg", "jack", 10, "/home/test/geo (conflicted copy, jack v10).gpkg"), + ("/home/test/g.pkg", "j", 0, "/home/test/g (conflicted copy, j v0).pkg"), + ("home/test/geo.gpkg", "jack", 10, "home/test/geo (conflicted copy, jack v10).gpkg"), + ("geo.gpkg", "jack", 10, "geo (conflicted copy, jack v10).gpkg"), + ("/home/../geo.gpkg", "jack", 10, "/geo (conflicted copy, jack v10).gpkg"), + ("/home/./geo.gpkg", "jack", 10, "/home/geo (conflicted copy, jack v10).gpkg"), + ("/home/test/geo.gpkg", "", 10, "/home/test/geo (conflicted copy, v10).gpkg"), + ("/home/test/geo.gpkg", "jack", -1, "/home/test/geo (conflicted copy, jack v-1).gpkg"), + ("/home/test/geo.tar.gz", "jack", 100, "/home/test/geo (conflicted copy, jack v100).tar.gz"), + ("", "jack", 1, ""), + ("/home/test/survey.qgs", "jack", 10, "/home/test/survey (conflicted copy, jack v10).qgs~"), + ("/home/test/survey.QGZ", "jack", 10, "/home/test/survey (conflicted copy, jack v10).QGZ~"), + ] for i in data: file_name = conflicted_copy_file_name(i[0], i[1], i[2]) assert file_name == i[3] - data = [ - ('/home/test/geo.json', 'jack', 10, '/home/test/geo (edit conflict, jack v10).json'), - ('/home/test/g.jsn', 'j', 0, '/home/test/g (edit conflict, j v0).json'), - ('home/test/geo.json', 'jack', 10, 'home/test/geo (edit conflict, jack v10).json'), - ('geo.json', 'jack', 10, 'geo (edit conflict, jack v10).json'), - ('/home/../geo.json', 'jack', 10, '/geo (edit conflict, jack v10).json'), - ('/home/./geo.json', 'jack', 10, '/home/geo (edit conflict, jack v10).json'), - ('/home/test/geo.json', '', 10, '/home/test/geo (edit conflict, v10).json'), - ('/home/test/geo.json', 'jack', -1, '/home/test/geo (edit conflict, jack v-1).json'), - ('/home/test/geo.gpkg', 'jack', 10, '/home/test/geo (edit conflict, jack v10).json'), - ('/home/test/geo.tar.gz', 'jack', 100, '/home/test/geo (edit conflict, jack v100).json'), - ('', 'jack', 1, '') - ] + ("/home/test/geo.json", "jack", 10, "/home/test/geo (edit conflict, jack v10).json"), + ("/home/test/g.jsn", "j", 0, "/home/test/g (edit conflict, j v0).json"), + ("home/test/geo.json", "jack", 10, "home/test/geo (edit conflict, jack v10).json"), + ("geo.json", "jack", 10, "geo (edit conflict, jack v10).json"), + ("/home/../geo.json", "jack", 10, "/geo (edit conflict, jack v10).json"), + ("/home/./geo.json", "jack", 10, "/home/geo (edit conflict, jack v10).json"), + ("/home/test/geo.json", "", 10, "/home/test/geo (edit conflict, v10).json"), + ("/home/test/geo.json", "jack", -1, "/home/test/geo (edit conflict, jack v-1).json"), + ("/home/test/geo.gpkg", "jack", 10, "/home/test/geo (edit conflict, jack v10).json"), + ("/home/test/geo.tar.gz", "jack", 100, "/home/test/geo (edit conflict, jack v100).json"), + ("", "jack", 1, ""), + ] for i in data: file_name = edit_conflict_file_name(i[0], i[1], i[2]) @@ -1386,7 +1395,7 @@ def test_unique_path_names(): """ Test generation of unique file names. """ - project_dir = os.path.join(TMP_DIR, 'unique_file_names') + project_dir = os.path.join(TMP_DIR, "unique_file_names") remove_folders([project_dir]) @@ -1406,26 +1415,30 @@ def test_unique_path_names(): # - another (1).txt # - another (2).txt # - arch.tar.gz - data = {'folderA': {'files': ['fileA.txt', 'fileA (1).txt', 'fileB.txt'], 'folderAB': {}, 'folderAB (1)': {}}, 'files': ['file.txt', 'another.txt', 'another (1).txt', 'another (2).txt', 'arch.tar.gz']} + data = { + "folderA": {"files": ["fileA.txt", "fileA (1).txt", "fileB.txt"], "folderAB": {}, "folderAB (1)": {}}, + "files": ["file.txt", "another.txt", "another (1).txt", "another (2).txt", "arch.tar.gz"], + } create_directory(project_dir, data) data = [ - ('file.txt', 'file (1).txt'), - ('another.txt', 'another (3).txt'), - ('folderA', 'folderA (1)'), - ('non.txt', 'non.txt'), - ('data.gpkg', 'data.gpkg'), - ('arch.tar.gz', 'arch (1).tar.gz'), - ('folderA/folder', 'folderA/folder'), - ('folderA/fileA.txt', 'folderA/fileA (2).txt'), - ('folderA/fileB.txt', 'folderA/fileB (1).txt'), - ('folderA/fileC.txt', 'folderA/fileC.txt'), - ('folderA/folderAB', 'folderA/folderAB (2)'), - ] + ("file.txt", "file (1).txt"), + ("another.txt", "another (3).txt"), + ("folderA", "folderA (1)"), + ("non.txt", "non.txt"), + ("data.gpkg", "data.gpkg"), + ("arch.tar.gz", "arch (1).tar.gz"), + ("folderA/folder", "folderA/folder"), + ("folderA/fileA.txt", "folderA/fileA (2).txt"), + ("folderA/fileB.txt", "folderA/fileB (1).txt"), + ("folderA/fileC.txt", "folderA/fileC.txt"), + ("folderA/folderAB", "folderA/folderAB (2)"), + ] for i in data: file_name = unique_path_name(os.path.join(project_dir, i[0])) assert file_name == os.path.join(project_dir, i[1]) + def create_directory(root, data): for k, v in data.items(): if isinstance(v, dict): @@ -1435,7 +1448,8 @@ def create_directory(root, data): create_directory(dir_name, v) elif isinstance(v, list): for file_name in v: - open(os.path.join(root, file_name), 'w').close() + open(os.path.join(root, file_name), "w").close() + def test_unfinished_pull(mc): """ @@ -1443,20 +1457,22 @@ def test_unfinished_pull(mc): and failed copy of the database file is handled correctly, i.e. an unfinished_pull directory is created with the content of the server changes. """ - test_project = 'test_unfinished_pull' - project = API_USER + '/' + test_project + test_project = "test_unfinished_pull" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - project_dir_2 = os.path.join(TMP_DIR, test_project+'_2') # concurrent project dir - unfinished_pull_dir = os.path.join(TMP_DIR, test_project, '.mergin', 'unfinished_pull') # unfinished_pull dir for the primary project - test_gpkg = os.path.join(project_dir, 'test.gpkg') - test_gpkg_2 = os.path.join(project_dir_2, 'test.gpkg') - test_gpkg_basefile = os.path.join(project_dir, '.mergin', 'test.gpkg') + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir + unfinished_pull_dir = os.path.join( + TMP_DIR, test_project, ".mergin", "unfinished_pull" + ) # unfinished_pull dir for the primary project + test_gpkg = os.path.join(project_dir, "test.gpkg") + test_gpkg_2 = os.path.join(project_dir_2, "test.gpkg") + test_gpkg_basefile = os.path.join(project_dir, ".mergin", "test.gpkg") test_gpkg_conflict = conflicted_copy_file_name(test_gpkg, API_USER, 2) - test_gpkg_unfinished_pull = os.path.join(project_dir, '.mergin', 'unfinished_pull', 'test.gpkg') + test_gpkg_unfinished_pull = os.path.join(project_dir, ".mergin", "unfinished_pull", "test.gpkg") cleanup(mc, project, [project_dir, project_dir_2]) os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL, that's the more common and more difficult scenario mc.create_project_and_push(test_project, project_dir) @@ -1466,19 +1482,19 @@ def test_unfinished_pull(mc): mc.push_project(project_dir_2) # do changes in the local DB (added a row) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL pull_changes, push_changes, _ = mc.project_status(project_dir) - assert _is_file_updated('test.gpkg', pull_changes) - assert _is_file_updated('test.gpkg', push_changes) + assert _is_file_updated("test.gpkg", pull_changes) + assert _is_file_updated("test.gpkg", push_changes) assert not os.path.exists(test_gpkg_conflict) assert not mc.has_unfinished_pull(project_dir) # lock base file to emulate situation when we can't overwrite it, because # it is used by another process - subprocess.run(['sudo', 'chattr', '+i', test_gpkg]) + subprocess.run(["sudo", "chattr", "+i", test_gpkg]) mc.pull_project(project_dir) @@ -1495,12 +1511,12 @@ def test_unfinished_pull(mc): _check_test_table(test_gpkg_basefile) # check that the local file contain the new row, while basefile and server version don't - assert _get_table_row_count(test_gpkg, 'simple') == 4 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 3 - assert _get_table_row_count(test_gpkg_unfinished_pull, 'simple') == 3 + assert _get_table_row_count(test_gpkg, "simple") == 4 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 3 + assert _get_table_row_count(test_gpkg_unfinished_pull, "simple") == 3 # unlock base file, so we can apply changes from the unfinished pull - subprocess.run(['sudo', 'chattr', '-i', test_gpkg]) + subprocess.run(["sudo", "chattr", "-i", test_gpkg]) mc.resolve_unfinished_pull(project_dir) @@ -1516,29 +1532,32 @@ def test_unfinished_pull(mc): _check_test_table(test_gpkg_conflict) # check that the local file + basefile don't contain the new row, and the conflict copy does - assert _get_table_row_count(test_gpkg, 'simple') == 3 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 3 - assert _get_table_row_count(test_gpkg_conflict, 'simple') == 4 + assert _get_table_row_count(test_gpkg, "simple") == 3 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 3 + assert _get_table_row_count(test_gpkg_conflict, "simple") == 4 + def test_unfinished_pull_push(mc): """ Checks client behaviour when performing push and pull of the project in the unfinished pull state. """ - test_project = 'test_unfinished_pull_push' - project = API_USER + '/' + test_project + test_project = "test_unfinished_pull_push" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - project_dir_2 = os.path.join(TMP_DIR, test_project+'_2') # concurrent project dir - unfinished_pull_dir = os.path.join(TMP_DIR, test_project, '.mergin', 'unfinished_pull') # unfinished_pull dir for the primary project - test_gpkg = os.path.join(project_dir, 'test.gpkg') - test_gpkg_2 = os.path.join(project_dir_2, 'test.gpkg') - test_gpkg_basefile = os.path.join(project_dir, '.mergin', 'test.gpkg') + project_dir_2 = os.path.join(TMP_DIR, test_project + "_2") # concurrent project dir + unfinished_pull_dir = os.path.join( + TMP_DIR, test_project, ".mergin", "unfinished_pull" + ) # unfinished_pull dir for the primary project + test_gpkg = os.path.join(project_dir, "test.gpkg") + test_gpkg_2 = os.path.join(project_dir_2, "test.gpkg") + test_gpkg_basefile = os.path.join(project_dir, ".mergin", "test.gpkg") test_gpkg_conflict = conflicted_copy_file_name(test_gpkg, API_USER, 2) - test_gpkg_unfinished_pull = os.path.join(project_dir, '.mergin', 'unfinished_pull', 'test.gpkg') + test_gpkg_unfinished_pull = os.path.join(project_dir, ".mergin", "unfinished_pull", "test.gpkg") cleanup(mc, project, [project_dir, project_dir_2]) os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL, that's the more common and more difficult scenario mc.create_project_and_push(test_project, project_dir) @@ -1548,19 +1567,19 @@ def test_unfinished_pull_push(mc): mc.push_project(project_dir_2) # do changes in the local DB (added a row) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), test_gpkg) _use_wal(test_gpkg) # make sure we use WAL pull_changes, push_changes, _ = mc.project_status(project_dir) - assert _is_file_updated('test.gpkg', pull_changes) - assert _is_file_updated('test.gpkg', push_changes) + assert _is_file_updated("test.gpkg", pull_changes) + assert _is_file_updated("test.gpkg", push_changes) assert not os.path.exists(test_gpkg_conflict) assert not mc.has_unfinished_pull(project_dir) # lock base file to emulate situation when we can't overwrite it, because # it is used by another process - subprocess.run(['sudo', 'chattr', '+i', test_gpkg]) + subprocess.run(["sudo", "chattr", "+i", test_gpkg]) mc.pull_project(project_dir) @@ -1577,9 +1596,9 @@ def test_unfinished_pull_push(mc): _check_test_table(test_gpkg_basefile) # check that the local file contain the new row, while basefile and server version don't - assert _get_table_row_count(test_gpkg, 'simple') == 4 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 3 - assert _get_table_row_count(test_gpkg_unfinished_pull, 'simple') == 3 + assert _get_table_row_count(test_gpkg, "simple") == 4 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 3 + assert _get_table_row_count(test_gpkg_unfinished_pull, "simple") == 3 # attempt to push project in the unfinished pull state should # fail with ClientError @@ -1592,7 +1611,7 @@ def test_unfinished_pull_push(mc): mc.pull_project(project_dir) # unlock base file, so we can apply changes from the unfinished pull - subprocess.run(['sudo', 'chattr', '-i', test_gpkg]) + subprocess.run(["sudo", "chattr", "-i", test_gpkg]) # perform pull. This should resolve unfinished pull first and then # collect data from the server @@ -1610,14 +1629,15 @@ def test_unfinished_pull_push(mc): _check_test_table(test_gpkg_conflict) # check that the local file + basefile don't contain the new row, and the conflict copy does - assert _get_table_row_count(test_gpkg, 'simple') == 3 - assert _get_table_row_count(test_gpkg_basefile, 'simple') == 3 - assert _get_table_row_count(test_gpkg_conflict, 'simple') == 4 + assert _get_table_row_count(test_gpkg, "simple") == 3 + assert _get_table_row_count(test_gpkg_basefile, "simple") == 3 + assert _get_table_row_count(test_gpkg_conflict, "simple") == 4 + def test_project_versions_list(mc): - """Test getting project versions in various ranges """ - test_project = 'test_project_versions' - project = API_USER + '/' + test_project + """Test getting project versions in various ranges""" + test_project = "test_project_versions" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) create_versioned_project(mc, test_project, project_dir, "base.gpkg") project_info = mc.project_info(project) @@ -1645,9 +1665,10 @@ def test_project_versions_list(mc): assert versions[0]["name"] == "v2" assert versions[-1]["name"] == "v4" + def test_report(mc): - test_project = 'test_report' - project = API_USER + '/' + test_project + test_project = "test_report" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) f_updated = "base.gpkg" mp = create_versioned_project(mc, test_project, project_dir, f_updated, remove=False, overwrite=True) @@ -1657,7 +1678,7 @@ def test_report(mc): since = "v2" to = "v4" proj_name = project.replace(os.path.sep, "-") - report_file = os.path.join(TMP_DIR, "report", f'{proj_name}-{since}-{to}.csv') + report_file = os.path.join(TMP_DIR, "report", f"{proj_name}-{since}-{to}.csv") if os.path.exists(report_file): os.remove(report_file) warnings = create_report(mc, directory, since, to, report_file) @@ -1666,7 +1687,9 @@ def test_report(mc): # assert headers and content in report file with open(report_file, "r") as rf: content = rf.read() - headers = ",".join(["file", "table", "author", "date", "time", "version", "operation", "length", "area", "count"]) + headers = ",".join( + ["file", "table", "author", "date", "time", "version", "operation", "length", "area", "count"] + ) assert headers in content assert "base.gpkg,simple,test_plugin" in content assert "v3,update,,,2" in content @@ -1691,8 +1714,8 @@ def test_project_versions_list(mc, mc2): """ Test retrieving user permissions """ - test_project = 'test_permissions' - test_project_fullname = API_USER2 + '/' + test_project + test_project = "test_permissions" + test_project_fullname = API_USER2 + "/" + test_project # cleanups project_dir = os.path.join(TMP_DIR, test_project, API_USER) @@ -1704,8 +1727,8 @@ def test_project_versions_list(mc, mc2): # Add reader access to another client project_info = get_project_info(mc2, API_USER2, test_project) - access = project_info['access'] - access['readersnames'].append(API_USER) + access = project_info["access"] + access["readersnames"].append(API_USER) mc2.set_project_access(test_project_fullname, access) # reader should not have write access @@ -1713,15 +1736,15 @@ def test_project_versions_list(mc, mc2): # Add writer access to another client project_info = get_project_info(mc2, API_USER2, test_project) - access = project_info['access'] - access['writersnames'].append(API_USER) + access = project_info["access"] + access["writersnames"].append(API_USER) mc2.set_project_access(test_project_fullname, access) # now user shold have write access assert mc.has_writing_permissions(test_project_fullname) # test organization permissions - test_project_fullname = 'testorg' + '/' + 'test_org_permissions' + test_project_fullname = "testorg" + "/" + "test_org_permissions" # owner should have write access assert mc.has_writing_permissions(test_project_fullname) @@ -1734,32 +1757,32 @@ def test_report_failure(mc): """Check that report generated without errors when a table was added and then deleted. """ - test_project = 'test_report_failure' - project = API_USER + '/' + test_project + test_project = "test_report_failure" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - test_gpkg = os.path.join(project_dir, 'test.gpkg') + test_gpkg = os.path.join(project_dir, "test.gpkg") report_file = os.path.join(TMP_DIR, "report.csv") cleanup(mc, project, [project_dir]) os.makedirs(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), test_gpkg) mc.create_project_and_push(test_project, project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), test_gpkg) mc.push_project(project_dir) # add a new table to the geopackage - shutil.copy(os.path.join(TEST_DATA_DIR, 'two_tables.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables.gpkg"), test_gpkg) mc.push_project(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'two_tables_1_A.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_1_A.gpkg"), test_gpkg) mc.push_project(project_dir) warnings = create_report(mc, project_dir, "v1", "v4", report_file) assert warnings - shutil.copy(os.path.join(TEST_DATA_DIR, 'two_tables_drop.gpkg'), test_gpkg) + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), test_gpkg) mc.push_project(project_dir) warnings = create_report(mc, project_dir, "v1", "v5", report_file) @@ -1770,23 +1793,23 @@ def test_changesets_download(mc): """Check that downloading diffs works correctly, including case when changesets are cached. """ - test_project = 'test_changesets_download' - project = API_USER + '/' + test_project + test_project = "test_changesets_download" + project = API_USER + "/" + test_project project_dir = os.path.join(TMP_DIR, test_project) # primary project dir - test_gpkg = 'test.gpkg' - file_path = os.path.join(project_dir, 'test.gpkg') + test_gpkg = "test.gpkg" + file_path = os.path.join(project_dir, "test.gpkg") download_dir = os.path.join(TMP_DIR, "changesets") cleanup(mc, project, [project_dir]) os.makedirs(project_dir, exist_ok=True) - shutil.copy(os.path.join(TEST_DATA_DIR, 'base.gpkg'), file_path) + shutil.copy(os.path.join(TEST_DATA_DIR, "base.gpkg"), file_path) mc.create_project_and_push(test_project, project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A.gpkg'), file_path) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A.gpkg"), file_path) mc.push_project(project_dir) - shutil.copy(os.path.join(TEST_DATA_DIR, 'inserted_1_A_mod.gpkg'), file_path) + shutil.copy(os.path.join(TEST_DATA_DIR, "inserted_1_A_mod.gpkg"), file_path) mc.push_project(project_dir) mp = MerginProject(project_dir) diff --git a/mergin/utils.py b/mergin/utils.py index 59f0040..8602a9f 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -18,7 +18,7 @@ def generate_checksum(file, chunk_size=4096): :return: sha1 checksum """ checksum = hashlib.sha1() - with open(file, 'rb') as f: + with open(file, "rb") as f: while True: chunk = f.read(chunk_size) if not chunk: @@ -35,7 +35,7 @@ def save_to_file(stream, path): directory = os.path.abspath(os.path.dirname(path)) os.makedirs(directory, exist_ok=True) - with open(path, 'wb') as output: + with open(path, "wb") as output: writer = io.BufferedWriter(output, buffer_size=32768) while True: part = stream.read(4096) @@ -67,8 +67,8 @@ def find(items, fn): def int_version(version): - """ Convert v format of version to integer representation. """ - return int(version.lstrip('v')) if re.match(r'v\d', version) else None + """Convert v format of version to integer representation.""" + return int(version.lstrip("v")) if re.match(r"v\d", version) else None def do_sqlite_checkpoint(path, log=None): @@ -84,7 +84,7 @@ def do_sqlite_checkpoint(path, log=None): """ new_size = None new_checksum = None - if ".gpkg" in path and os.path.exists(f'{path}-wal'): + if ".gpkg" in path and os.path.exists(f"{path}-wal"): if log: log.info("checkpoint - going to add it in " + path) conn = sqlite3.connect(path) @@ -103,8 +103,7 @@ def do_sqlite_checkpoint(path, log=None): return new_size, new_checksum -def get_versions_with_file_changes( - mc, project_path, file_path, version_from=None, version_to=None, file_history=None): +def get_versions_with_file_changes(mc, project_path, file_path, version_from=None, version_to=None, file_history=None): """ Get the project version tags where the file was added, modified or deleted. @@ -145,7 +144,7 @@ def get_versions_with_file_changes( elif version == version_to: idx_to = idx break - return [f"v{ver_nr}" for ver_nr in all_version_numbers[idx_from:idx_to + 1]] + return [f"v{ver_nr}" for ver_nr in all_version_numbers[idx_from : idx_to + 1]] def unique_path_name(path): @@ -165,8 +164,8 @@ def unique_path_name(path): is_dir = os.path.isdir(path) head, tail = os.path.split(os.path.normpath(path)) - ext = ''.join(Path(tail).suffixes) - file_name = tail.replace(ext, '') + ext = "".join(Path(tail).suffixes) + file_name = tail.replace(ext, "") i = 0 while os.path.exists(unique_path): @@ -196,17 +195,17 @@ def conflicted_copy_file_name(path, user, version): :rtype: str """ if not path: - return '' + return "" head, tail = os.path.split(os.path.normpath(path)) - ext = ''.join(Path(tail).suffixes) - file_name = tail.replace(ext, '') + ext = "".join(Path(tail).suffixes) + file_name = tail.replace(ext, "") # in case of QGIS project files we have to add "~" (tilde) to suffix # to avoid having several QGIS project files inside Mergin Maps project. # See https://github.com/lutraconsulting/qgis-mergin-plugin/issues/382 # for more details if ext.lower() in (".qgz", ".qgs"): - ext += '~' + ext += "~" return os.path.join(head, file_name) + f" (conflicted copy, {user} v{version}){ext}" @@ -226,9 +225,9 @@ def edit_conflict_file_name(path, user, version): :rtype: str """ if not path: - return '' + return "" head, tail = os.path.split(os.path.normpath(path)) - ext = ''.join(Path(tail).suffixes) - file_name = tail.replace(ext, '') + ext = "".join(Path(tail).suffixes) + file_name = tail.replace(ext, "") return os.path.join(head, file_name) + f" (edit conflict, {user} v{version}).json" diff --git a/mergin/version.py b/mergin/version.py index b739f90..aa07760 100644 --- a/mergin/version.py +++ b/mergin/version.py @@ -1,6 +1,5 @@ - # The version is also stored in ../setup.py -__version__ = '0.7.3' +__version__ = "0.7.3" # There seems to be no single nice way to keep version info just in one place: # https://packaging.python.org/guides/single-sourcing-package-version/ diff --git a/scripts/check_all.bash b/scripts/check_all.bash index 1d80757..833986a 100755 --- a/scripts/check_all.bash +++ b/scripts/check_all.bash @@ -1,5 +1,5 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PWD=`pwd` cd $DIR -black --verbose -l 120 $DIR/../mergin +black -l 120 $DIR/../mergin cd $PWD diff --git a/scripts/update_version.py b/scripts/update_version.py index fa7b933..184d6a8 100644 --- a/scripts/update_version.py +++ b/scripts/update_version.py @@ -22,7 +22,7 @@ def replace_in_file(filepath, regex, sub): about_file = os.path.join(dir_path, os.pardir, "mergin", "version.py") print("patching " + about_file) -replace_in_file(about_file, "__version__\s=\s'.*", "__version__ = '" + ver + "'") +replace_in_file(about_file, "__version__\s=\s\".*", "__version__ = \"" + ver + "\"") setup_file = os.path.join(dir_path, os.pardir, "setup.py") print("patching " + setup_file)