Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into fix-download-nonexi…
Browse files Browse the repository at this point in the history
…sting-diff
  • Loading branch information
MarcelGeo committed Sep 5, 2024
2 parents c391e8b + 870b5b6 commit bb8a19f
Show file tree
Hide file tree
Showing 15 changed files with 698 additions and 155 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/autotests.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: Auto Tests
on: [push]
env:
TEST_MERGIN_URL: https://test.dev.merginmaps.com/
TEST_MERGIN_URL: https://app.dev.merginmaps.com/
TEST_API_USERNAME: test_plugin
TEST_API_PASSWORD: ${{ secrets.MERGINTEST_API_PASSWORD }}
TEST_API_USERNAME2: test_plugin2
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ htmlcov
.pytest_cache
deps
venv
.vscode/settings.json
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
# Changelog

## 0.9.2

- Update rules when pushing with editor permission level (#208)

## 0.9.1

- Support for "editor" permission level for server >= 2024.4.0. Editors have more limited functionality than "writer" permission: they are not allowed to modify the QGIS project file or change structure of tables (#202, #207)
- Better handling of unexpected errors during the initial download of a project (#201)
- Fixes to CI (#205)

## 0.9.0

- Add `reset_local_changes()` API call and `reset` CLI command to make it possible to discard any changes in the local project directory (#107)
Expand Down
10 changes: 6 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ If you plan to run `mergin` command multiple times and you wish to avoid logging
you can use "login" command to get authorization token.
It will ask for password and then output environment variable with auth token. The returned token
is not permanent - it will expire after several hours.
```
```bash
$ mergin --username john login
Password: topsecret
Login successful!
Expand All @@ -160,15 +160,15 @@ it is possible to run other commands without specifying username/password.
### Installing deps

Python 3.7+ required. Create `mergin/deps` folder where [geodiff](https://github.com/MerginMaps/geodiff) lib is supposed to be and install dependencies:
```
```bash
rm -r mergin/deps
mkdir mergin/deps
pip install python-dateutil pytz
pip install pygeodiff --target=mergin/deps
```

For using mergin client with its dependencies packaged locally run:
```
```bash
pip install wheel
python3 setup.py sdist bdist_wheel
mkdir -p mergin/deps
Expand All @@ -180,13 +180,15 @@ For using mergin client with its dependencies packaged locally run:
### Tests
For running test do:

```
```bash
cd mergin
export TEST_MERGIN_URL=<url> # testing server
export TEST_API_USERNAME=<username>
export TEST_API_PASSWORD=<pwd>
export TEST_API_USERNAME2=<username2>
export TEST_API_PASSWORD2=<pwd2>
# workspace name with controlled available storage space (e.g. 20MB), default value: testpluginstorage
export TEST_STORAGE_WORKSPACE=<workspacename>
pip install pytest pytest-cov coveralls
pytest --cov-report html --cov=mergin mergin/test/
```
18 changes: 13 additions & 5 deletions mergin/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,13 +315,21 @@ def share(ctx, project):
return
access_list = mc.project_user_permissions(project)

for username in access_list.get("owners"):
owners = access_list.get("owners", [])
writers = access_list.get("writers", [])
editors = access_list.get("editors", [])
readers = access_list.get("readers", [])

for username in owners:
click.echo("{:20}\t{:20}".format(username, "owner"))
for username in access_list.get("writers"):
if username not in access_list.get("owners"):
for username in writers:
if username not in owners:
click.echo("{:20}\t{:20}".format(username, "writer"))
for username in access_list.get("readers"):
if username not in access_list.get("writers"):
for username in editors:
if username not in writers:
click.echo("{:20}\t{:20}".format(username, "editor"))
for username in readers:
if username not in editors:
click.echo("{:20}\t{:20}".format(username, "reader"))


Expand Down
86 changes: 54 additions & 32 deletions mergin/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import typing
import warnings

from .common import ClientError, LoginError, InvalidProject
from .common import ClientError, LoginError, InvalidProject, ErrorCode
from .merginproject import MerginProject
from .client_pull import (
download_file_finalize,
Expand Down Expand Up @@ -205,19 +205,20 @@ def _do_request(self, request):
try:
return self.opener.open(request)
except urllib.error.HTTPError as e:
if e.headers.get("Content-Type", "") == "application/problem+json":
info = json.load(e)
err_detail = info.get("detail")
else:
err_detail = e.read().decode("utf-8")
server_response = json.load(e)

# We first to try to get the value from the response otherwise we set a default value
err_detail = server_response.get("detail", e.read().decode("utf-8"))
server_code = server_response.get("code", None)

error_msg = (
f"HTTP Error: {e.code} {e.reason}\n"
f"URL: {request.get_full_url()}\n"
f"Method: {request.get_method()}\n"
f"Detail: {err_detail}"
raise ClientError(
detail=err_detail,
url=request.get_full_url(),
server_code=server_code,
server_response=server_response,
http_error=e.code,
http_method=request.get_method(),
)
raise ClientError(error_msg)
except urllib.error.URLError as e:
# e.g. when DNS resolution fails (no internet connection?)
raise ClientError("Error requesting " + request.full_url + ": " + str(e))
Expand Down Expand Up @@ -429,9 +430,9 @@ def create_workspace(self, workspace_name):

try:
self.post("/v1/workspace", params, {"Content-Type": "application/json"})
except Exception as e:
detail = f"Workspace name: {workspace_name}"
raise ClientError(str(e), detail)
except ClientError as e:
e.extra = f"Workspace name: {workspace_name}"
raise e

def create_project(self, project_name, is_public=False, namespace=None):
"""
Expand Down Expand Up @@ -478,9 +479,9 @@ def create_project(self, project_name, is_public=False, namespace=None):
namespace = self.username()
try:
self.post(f"/v1/project/{namespace}", params, {"Content-Type": "application/json"})
except Exception as e:
detail = f"Namespace: {namespace}, project name: {project_name}"
raise ClientError(str(e), detail)
except ClientError as e:
e.extra = f"Namespace: {namespace}, project name: {project_name}"
raise e

def create_project_and_push(self, project_name, directory, is_public=False, namespace=None):
"""
Expand Down Expand Up @@ -761,8 +762,11 @@ def set_project_access(self, project_path, access):
"""
Updates access for given project.
:param project_path: project full name (<namespace>/<name>)
:param access: dict <readersnames, writersnames, ownersnames> -> list of str username we want to give access to
:param access: dict <readersnames, editorsnames, writersnames, ownersnames> -> list of str username we want to give access to (editorsnames are only supported on servers at version 2024.4.0 or later)
"""
if "editorsnames" in access and not self.has_editor_support():
raise NotImplementedError("Editors are only supported on servers at version 2024.4.0 or later")

if not self._user_info:
raise Exception("Authentication required")

Expand All @@ -773,28 +777,34 @@ def set_project_access(self, project_path, access):
try:
request = urllib.request.Request(url, data=json.dumps(params).encode(), headers=json_headers, method="PUT")
self._do_request(request)
except Exception as e:
detail = f"Project path: {project_path}"
raise ClientError(str(e), detail)
except ClientError as e:
e.extra = f"Project path: {project_path}"
raise e

def add_user_permissions_to_project(self, project_path, usernames, permission_level):
"""
Add specified permissions to specified users to project
:param project_path: project full name (<namespace>/<name>)
:param usernames: list of usernames to be granted specified permission level
:param permission_level: string (reader, writer, owner)
:param permission_level: string (reader, editor, writer, owner)
Editor permission_level is only supported on servers at version 2024.4.0 or later.
"""
if permission_level not in ["owner", "reader", "writer"]:
if permission_level not in ["owner", "reader", "writer", "editor"] or (
permission_level == "editor" and not self.has_editor_support()
):
raise ClientError("Unsupported permission level")

project_info = self.project_info(project_path)
access = project_info.get("access")
for name in usernames:
if permission_level == "owner":
access.get("ownersnames").append(name)
if permission_level == "writer" or permission_level == "owner":
if permission_level in ("writer", "owner"):
access.get("writersnames").append(name)
if permission_level == "writer" or permission_level == "owner" or permission_level == "reader":
if permission_level in ("writer", "owner", "editor") and "editorsnames" in access:
access.get("editorsnames").append(name)
if permission_level in ("writer", "owner", "editor", "reader"):
access.get("readersnames").append(name)
self.set_project_access(project_path, access)

Expand All @@ -807,11 +817,13 @@ def remove_user_permissions_from_project(self, project_path, usernames):
project_info = self.project_info(project_path)
access = project_info.get("access")
for name in usernames:
if name in access.get("ownersnames"):
if name in access.get("ownersnames", []):
access.get("ownersnames").remove(name)
if name in access.get("writersnames"):
if name in access.get("writersnames", []):
access.get("writersnames").remove(name)
if name in access.get("readersnames"):
if name in access.get("editorsnames", []):
access.get("editorsnames").remove(name)
if name in access.get("readersnames", []):
access.get("readersnames").remove(name)
self.set_project_access(project_path, access)

Expand All @@ -821,14 +833,18 @@ def project_user_permissions(self, project_path):
:param project_path: project full name (<namespace>/<name>)
:return dict("owners": list(usernames),
"writers": list(usernames),
"editors": list(usernames) - only on servers at version 2024.4.0 or later,
"readers": list(usernames))
"""
project_info = self.project_info(project_path)
access = project_info.get("access")
result = {}
result["owners"] = access.get("ownersnames")
result["writers"] = access.get("writersnames")
result["readers"] = access.get("readersnames")
if "editorsnames" in access:
result["editors"] = access.get("editorsnames", [])

result["owners"] = access.get("ownersnames", [])
result["writers"] = access.get("writersnames", [])
result["readers"] = access.get("readersnames", [])
return result

def push_project(self, directory):
Expand Down Expand Up @@ -1220,3 +1236,9 @@ def download_files(
job = download_files_async(self, project_dir, file_paths, output_paths, version=version)
pull_project_wait(job)
download_files_finalize(job)

def has_editor_support(self):
"""
Returns whether the server version is acceptable for editor support.
"""
return is_version_acceptable(self.server_version(), "2024.4.0")
26 changes: 13 additions & 13 deletions mergin/client_pull.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

from .common import CHUNK_SIZE, ClientError
from .merginproject import MerginProject
from .utils import save_to_file, get_versions_with_file_changes
from .utils import save_to_file


# status = download_project_async(...)
Expand Down Expand Up @@ -199,9 +199,9 @@ def download_project_is_running(job):
"""
for future in job.futures:
if future.done() and future.exception() is not None:
job.mp.log.error(
"Error while downloading project: " + "".join(traceback.format_exception(future.exception()))
)
exc = future.exception()
traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__)
job.mp.log.error("Error while downloading project: " + "".join(traceback_lines))
job.mp.log.info("--- download aborted")
job.failure_log_file = _cleanup_failed_download(job.directory, job.mp)
raise future.exception()
Expand All @@ -225,9 +225,9 @@ def download_project_finalize(job):
# make sure any exceptions from threads are not lost
for future in job.futures:
if future.exception() is not None:
job.mp.log.error(
"Error while downloading project: " + "".join(traceback.format_exception(future.exception()))
)
exc = future.exception()
traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__)
job.mp.log.error("Error while downloading project: " + "".join(traceback_lines))
job.mp.log.info("--- download aborted")
job.failure_log_file = _cleanup_failed_download(job.directory, job.mp)
raise future.exception()
Expand Down Expand Up @@ -340,7 +340,7 @@ def __init__(
mp,
project_info,
basefiles_to_patch,
user_name,
mc,
):
self.project_path = project_path
self.pull_changes = (
Expand All @@ -358,7 +358,7 @@ def __init__(
self.basefiles_to_patch = (
basefiles_to_patch # list of tuples (relative path within project, list of diff files in temp dir to apply)
)
self.user_name = user_name
self.mc = mc

def dump(self):
print("--- JOB ---", self.total_size, "bytes")
Expand Down Expand Up @@ -494,7 +494,7 @@ def pull_project_async(mc, directory):
mp,
server_info,
basefiles_to_patch,
mc.username(),
mc,
)

# start download
Expand Down Expand Up @@ -570,7 +570,7 @@ def merge(self):
raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file))


def pull_project_finalize(job):
def pull_project_finalize(job: PullJob):
"""
To be called when pull in the background is finished and we need to do the finalization (merge chunks etc.)
Expand Down Expand Up @@ -623,7 +623,7 @@ def pull_project_finalize(job):
raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile))

try:
conflicts = job.mp.apply_pull_changes(job.pull_changes, job.temp_dir, job.user_name)
conflicts = job.mp.apply_pull_changes(job.pull_changes, job.temp_dir, job.project_info, job.mc)
except Exception as e:
job.mp.log.error("Failed to apply pull changes: " + str(e))
job.mp.log.info("--- pull aborted")
Expand Down Expand Up @@ -723,7 +723,7 @@ def download_diffs_async(mc, project_directory, file_path, versions):
mp,
server_info,
{},
mc.username(),
mc,
)

# start download
Expand Down
Loading

0 comments on commit bb8a19f

Please sign in to comment.