From 9be3f93caeb596091d62692cb8e4dae05624098d Mon Sep 17 00:00:00 2001 From: Bane Sullivan Date: Mon, 29 May 2023 16:27:13 -0600 Subject: [PATCH] Utilize new rasterio source (#49) * Utilize new rasterio source * Fix typo * Bump CI Python * Limit django<4.2 for girder project * Bump large-image * Notes * Trigger CI * Trigger CI * Install pyproj --- .github/workflows/ci.yml | 10 ++++----- .github/workflows/wheel.yml | 2 +- DEVELOPMENT.md | 2 +- README.md | 28 +++++++++++++++++++----- demo/README.md | 3 ++- demo/requirements.txt | 2 ++ django_large_image/rest/tiles.py | 2 +- django_large_image/tilesource.py | 2 +- project/example/core/tests/test_data.py | 10 ++++----- project/example/core/tests/test_tiles.py | 5 ++++- project/setup.py | 5 +++-- requirements.txt | 5 ----- tox.ini | 2 +- 13 files changed, 49 insertions(+), 29 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa5dfe1..88972a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - name: Install tox run: | pip install --upgrade pip @@ -45,7 +45,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - name: Install tox run: | pip install --upgrade pip @@ -92,7 +92,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - name: Install tox run: | pip install --upgrade pip @@ -123,11 +123,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.11" - name: Install Dependencies run: | pip install --upgrade pip - pip install -r requirements.txt -r demo/requirements.txt + pip install -e .[colormaps] -r demo/requirements.txt - name: Run tests working-directory: demo run: | diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index b943a7f..4e959da 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: "3.9" + python-version: "3.11" - name: Build Weel run: | pip install wheel twine diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e479951..a79ef63 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -26,7 +26,7 @@ but allows developers to run Python code on their native system. ### Initial Setup 1. Run `docker-compose -f ./docker-compose.yml up -d` -2. Install Python 3.9 +2. Install Python 3.11 3. Install [`psycopg2` build prerequisites](https://www.psycopg.org/docs/install.html#build-prerequisites) 4. Create and activate a new Python virtualenv diff --git a/README.md b/README.md index 0e03899..55638cc 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,9 @@ Miscellaneous: Out of the box, `django-large-image` only depends on the core `large-image` module, but you will need a `large-image-source-*` module in order for this to work. Most of our users probably want to work with geospatial images so we -will focus on the `large-image-source-gdal` case, but it is worth noting that -`large-image` has source modules for a wide variety of image formats -(e.g., medical image formats for microscopy). +will focus on the `large-image-source-gdal` and ``large-image-source-rasterio` +cases, but it is worth noting that `large-image` has source modules for a wide +variety of image formats (e.g., medical image formats for microscopy). See [`large-image`](https://github.com/girder/large_image#installation)'s installation instructions for more details. @@ -126,6 +126,16 @@ installation instructions for more details. ### 🎡 pip +### Rasterio + +```bash +pip install \ + django-large-image \ + 'large-image[rasterio,pil]>=1.22' +``` + +### GDAL + **Tip:* installing GDAL is notoriously difficult, so at Kitware we provide pre-built Python wheels with the GDAL binary bundled for easily installation in production **linux** environments. To install our GDAL wheel, use: @@ -143,6 +153,10 @@ pip install \ Or install with `conda`: +```bash +conda install -c conda-forge django-large-image large-image-source-rasterio +``` + ```bash conda install -c conda-forge django-large-image large-image-source-gdal ``` @@ -166,7 +180,7 @@ The following are the provided mixin classes and their use case: - `LargeImageMixin`: for use with a standard, non-detail `ViewSet`. Users must implement `get_path()` - `LargeImageDetailMixin`: for use with a detail viewset like `GenericViewSet`. Users must implement `get_path()` - `LargeImageFileDetailMixin`: (most commonly used) for use with a detail viewset like `GenericViewSet` where the associated model has a `FileField` storing the image data. -- `LargeImageVSIFileDetailMixin`: (geospatial) for use with a detail viewset like `GenericViewSet` where the associated model has a `FileField` storing the image data that is intended to be read with GDAL. This will access the data over GDAL's Virtual File System interface (a VSI path). +- `LargeImageVSIFileDetailMixin`: (geospatial) for use with a detail viewset like `GenericViewSet` where the associated model has a `FileField` storing the image data that is intended to be read with GDAL/rasterio. This will access the data over GDAL's Virtual File System interface (a VSI path). Most users will want to use `LargeImageFileDetailMixin` and so the following example demonstrate how to use it: @@ -445,7 +459,7 @@ model in your application. Here is a starting point: import os from example.core import models from celery import shared_task -import large_image_converter +import large_image_converter # requires large-image-source-gdal @shared_task @@ -461,6 +475,10 @@ def task_convert_cog(my_model_pk): ... ``` +If using the `rasterio`-based source module, we recommend using +[`rio-cogeo`](https://github.com/cogeotiff/rio-cogeo) +over `large_image_converter`. + ## Using with django-raster [`django-raster`](https://github.com/geodesign/django-raster) is a popular diff --git a/demo/README.md b/demo/README.md index 4c23422..a6a7aef 100644 --- a/demo/README.md +++ b/demo/README.md @@ -5,8 +5,9 @@ for storing images. Install from root directory ``` -pip install --find-links https://girder.github.io/large_image_wheels \ +pip install \ -e . \ + large-image[rasterio,pil]>=1.22 \ gunicorn \ whitenoise \ pytest \ diff --git a/demo/requirements.txt b/demo/requirements.txt index b7d9e40..5e9bd7e 100644 --- a/demo/requirements.txt +++ b/demo/requirements.txt @@ -1,3 +1,5 @@ +large-image[rasterio,pil]>=1.22 +pyproj gunicorn whitenoise pytest diff --git a/django_large_image/rest/tiles.py b/django_large_image/rest/tiles.py index cbfe694..1576749 100644 --- a/django_large_image/rest/tiles.py +++ b/django_large_image/rest/tiles.py @@ -73,7 +73,7 @@ def tile_corners( 'xmax': xmax, 'ymin': ymin, 'ymax': ymax, - # 'proj4': source.getProj4String(), + # 'proj4': source.getProj4String(), # not supported by rasterio } return Response(metadata) diff --git a/django_large_image/tilesource.py b/django_large_image/tilesource.py index 0ae8657..d238e82 100644 --- a/django_large_image/tilesource.py +++ b/django_large_image/tilesource.py @@ -62,7 +62,7 @@ def _metadata_helper(source: FileTileSource, metadata: dict): metadata.setdefault('geospatial', is_geospatial(source)) if metadata['geospatial']: metadata['bounds'] = get_bounds(source) - # metadata['proj4'] = (source.getProj4String(),) + # metadata['proj4'] = (source.getProj4String(),) # not supported by rasterio if 'frames' not in metadata: metadata['frames'] = False diff --git a/project/example/core/tests/test_data.py b/project/example/core/tests/test_data.py index cc2d7fd..ddbd5bb 100644 --- a/project/example/core/tests/test_data.py +++ b/project/example/core/tests/test_data.py @@ -11,7 +11,7 @@ @pytest.mark.parametrize('format', get_formats()) def test_thumbnail(authenticated_api_client, image_file_geotiff, format): response = authenticated_api_client.get( - f'/api/image-file/{image_file_geotiff.pk}/data/thumbnail.{format}' + f'/api/image-file/{image_file_geotiff.pk}/data/thumbnail.{format}?projection=EPSG:3857' ) assert status.is_success(response.status_code) assert response['Content-Type'] == get_mime_type(format) @@ -68,7 +68,7 @@ def test_pixel(authenticated_api_client, image_file_geotiff): @pytest.mark.parametrize('format', get_formats()) def test_region_pixel(authenticated_api_client, image_file_geotiff, ome_image, format): response = authenticated_api_client.get( - f'/api/image-file/{image_file_geotiff.pk}/data/region.{format}?left=0&right=10&bottom=10&top=0&units=pixels' + f'/api/image-file/{image_file_geotiff.pk}/data/region.{format}?projection=EPSG:3857&left=0&right=10&bottom=10&top=0&units=pixels' ) assert status.is_success(response.status_code) assert response['Content-Type'] == get_mime_type(format) @@ -77,7 +77,7 @@ def test_region_pixel(authenticated_api_client, image_file_geotiff, ome_image, f @pytest.mark.django_db(transaction=True) def test_region_pixel_out_of_bounds(authenticated_api_client, image_file_geotiff, ome_image): response = authenticated_api_client.get( - f'/api/image-file/{image_file_geotiff.pk}/data/region.tif?left=10000000&right=20000000&bottom=20000000&top=10000000&units=pixels' + f'/api/image-file/{image_file_geotiff.pk}/data/region.tif?projection=EPSG:3857&left=10000000&right=20000000&bottom=20000000&top=10000000&units=pixels' ) assert status.is_client_error(response.status_code) @@ -85,13 +85,13 @@ def test_region_pixel_out_of_bounds(authenticated_api_client, image_file_geotiff @pytest.mark.django_db(transaction=True) def test_region_geo(authenticated_api_client, image_file_geotiff): response = authenticated_api_client.get( - f'/api/image-file/{image_file_geotiff.pk}/data/region.tif?units=EPSG:4326&left=-117.4567824262003&right=-117.10373770277764&bottom=32.635234150046514&top=32.964410481130365' + f'/api/image-file/{image_file_geotiff.pk}/data/region.tif?projection=EPSG:3857&units=EPSG:4326&left=-117.4567824262003&right=-117.10373770277764&bottom=32.635234150046514&top=32.964410481130365' ) assert status.is_success(response.status_code) assert response['Content-Type'] == 'image/tiff' # Leave units out response = authenticated_api_client.get( - f'/api/image-file/{image_file_geotiff.pk}/data/region.tif?left=-117.4567824262003&right=-117.10373770277764&bottom=32.635234150046514&top=32.964410481130365' + f'/api/image-file/{image_file_geotiff.pk}/data/region.tif?projection=EPSG:3857&left=-117.4567824262003&right=-117.10373770277764&bottom=32.635234150046514&top=32.964410481130365' ) assert status.is_success(response.status_code) assert response['Content-Type'] == 'image/tiff' diff --git a/project/example/core/tests/test_tiles.py b/project/example/core/tests/test_tiles.py index f4a489b..5a4025a 100644 --- a/project/example/core/tests/test_tiles.py +++ b/project/example/core/tests/test_tiles.py @@ -39,7 +39,10 @@ def test_tile_corners(authenticated_api_client, image_file_geotiff): ) assert status.is_success(response.status_code) data = response.data - # assert data['proj4'] + assert 'xmin' in data + assert 'xmax' in data + assert 'ymin' in data + assert 'ymax' in data @pytest.mark.django_db(transaction=True) diff --git a/project/setup.py b/project/setup.py index 597c5ac..a3bcf21 100644 --- a/project/setup.py +++ b/project/setup.py @@ -37,7 +37,7 @@ include_package_data=True, install_requires=[ 'celery', - 'django', + 'django<4.2', 'django-allauth', 'django-configurations[database,email]', 'django-extensions', @@ -50,7 +50,8 @@ 'django-s3-file-field[boto3]', 'gunicorn', 'django-large-image', - 'large-image[gdal,pil,ometiff,converter,vips,openslide,openjpeg]>=1.16.2', + 'large-image[rasterio,pil,ometiff,vips,openslide,openjpeg]>=1.22', + 'pyproj', 'pooch', ], extras_require={ diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 708b327..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# This is for Heroku demo deployment ---find-links https://girder.github.io/large_image_wheels -GDAL --e .[colormaps] --e ./project diff --git a/tox.ini b/tox.ini index 3cb3e08..ffc4eff 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ envlist = type, [testenv] -basepython=python3.9 +basepython=python3.11 setenv = PIP_FIND_LINKS = https://girder.github.io/large_image_wheels DJANGO_CONFIGURATION = TestingConfiguration