forked from OSGeo/gdal
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Python bindings: add a osgeo.gdal_fsspec module that on import will r…
…egister GDAL VSI file system handlers as fsspec AbstractFileSystem
- Loading branch information
Showing
11 changed files
with
582 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,229 @@ | ||
#!/usr/bin/env pytest | ||
# -*- coding: utf-8 -*- | ||
############################################################################### | ||
# Project: GDAL/OGR Test Suite | ||
# Purpose: Test gdal_fsspec module | ||
# Author: Even Rouault <even dot rouault at spatialys.com> | ||
# | ||
############################################################################### | ||
# Copyright (c) 20124, Even Rouault <even dot rouault at spatialys.com> | ||
# | ||
# SPDX-License-Identifier: MIT | ||
############################################################################### | ||
|
||
import pytest | ||
|
||
from osgeo import gdal | ||
|
||
fsspec = pytest.importorskip("fsspec") | ||
pytest.importorskip("fsspec.spec") | ||
|
||
from osgeo import gdal_fsspec # NOQA | ||
|
||
|
||
def test_gdal_fsspec_open_read(): | ||
|
||
with fsspec.open("vsi://data/byte.tif") as f: | ||
assert len(f.read()) == gdal.VSIStatL("data/byte.tif").size | ||
|
||
|
||
def test_gdal_fsspec_info_file(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
info = fs.info("data/byte.tif") | ||
assert "mtime" in info | ||
del info["mtime"] | ||
assert (info["mode"] & 32768) != 0 | ||
del info["mode"] | ||
assert info == { | ||
"name": "data/byte.tif", | ||
"size": 736, | ||
"type": "file", | ||
} | ||
|
||
|
||
def test_gdal_fsspec_info_dir(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
info = fs.info("data") | ||
assert (info["mode"] & 16384) != 0 | ||
del info["mode"] | ||
assert info == { | ||
"name": "data", | ||
"size": 0, | ||
"type": "directory", | ||
} | ||
|
||
|
||
def test_gdal_fsspec_info_error(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
with pytest.raises(FileNotFoundError): | ||
fs.info("/i/do/not/exist") | ||
|
||
|
||
def test_gdal_fsspec_ls(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
ret = fs.ls("data") | ||
assert len(ret) > 2 | ||
item_of_interest = None | ||
for item in ret: | ||
if item["name"] == "data/byte.tif": | ||
item_of_interest = item | ||
break | ||
assert item_of_interest | ||
assert "mtime" in item_of_interest | ||
del item_of_interest["mtime"] | ||
assert item_of_interest == { | ||
"name": "data/byte.tif", | ||
"size": 736, | ||
"type": "file", | ||
} | ||
|
||
|
||
def test_gdal_fsspec_ls_file(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
ret = fs.ls("data/byte.tif") | ||
assert ret == ["data/byte.tif"] | ||
|
||
|
||
def test_gdal_fsspec_ls_error(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
with pytest.raises(FileNotFoundError): | ||
fs.ls("vsi://i/do/not/exist") | ||
|
||
|
||
def test_gdal_fsspec_modified(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
modified = fs.modified("data/byte.tif") | ||
assert modified is not None | ||
import datetime | ||
|
||
assert isinstance(modified, datetime.datetime) | ||
|
||
|
||
def test_gdal_fsspec_modified_error(): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
with pytest.raises(FileNotFoundError): | ||
fs.modified("vsi://i/do/not/exist") | ||
|
||
|
||
def test_gdal_fsspec_rm(): | ||
|
||
with fsspec.open("vsimem:///foo.bin", "wb") as f: | ||
f.write(b"""bar""") | ||
fs = fsspec.filesystem("vsimem") | ||
fs.info("/foo.bin") | ||
fs.rm("/foo.bin") | ||
with pytest.raises(FileNotFoundError): | ||
fs.info("/foo.bin") | ||
|
||
|
||
def test_gdal_fsspec_rm_error(): | ||
|
||
fs = fsspec.filesystem("vsimem") | ||
with pytest.raises(FileNotFoundError): | ||
fs.rm("/foo.bin") | ||
|
||
|
||
def test_gdal_fsspec_copy(): | ||
|
||
with fsspec.open("vsimem://foo.bin", "wb") as f: | ||
f.write(b"""bar""") | ||
fs = fsspec.filesystem("vsimem") | ||
fs.copy("/foo.bin", "/bar.bin") | ||
assert fs.info("/bar.bin")["size"] == 3 | ||
assert fs.info("/foo.bin")["size"] == 3 | ||
fs.rm("/foo.bin") | ||
fs.rm("/bar.bin") | ||
|
||
|
||
def test_gdal_fsspec_copy_error(): | ||
|
||
fs = fsspec.filesystem("vsimem") | ||
with pytest.raises(FileNotFoundError): | ||
fs.copy("/foo.bin", "/bar.bin") | ||
|
||
|
||
def test_gdal_fsspec_mv(): | ||
|
||
with fsspec.open("vsimem://foo.bin", "wb") as f: | ||
f.write(b"""bar""") | ||
fs = fsspec.filesystem("vsimem") | ||
fs.mv("/foo.bin", "/bar.bin") | ||
assert fs.info("/bar.bin")["size"] == 3 | ||
with pytest.raises(FileNotFoundError): | ||
fs.info("/foo.bin") | ||
fs.rm("/bar.bin") | ||
|
||
|
||
def test_gdal_fsspec_mv_error(): | ||
|
||
fs = fsspec.filesystem("vsimem") | ||
with pytest.raises(FileNotFoundError): | ||
fs.mv("/foo.bin", "/bar.bin") | ||
|
||
|
||
def test_gdal_fsspec_mkdir(tmp_path): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
|
||
my_path = str(tmp_path) + "/my_dir" | ||
|
||
fs.mkdir(my_path) | ||
assert fs.info(my_path)["type"] == "directory" | ||
with pytest.raises(FileExistsError): | ||
fs.mkdir(my_path) | ||
fs.rmdir(my_path) | ||
|
||
fs.mkdir(my_path + "/my_subdir") | ||
assert fs.info(my_path)["type"] == "directory" | ||
assert fs.info(my_path + "/my_subdir")["type"] == "directory" | ||
fs.rmdir(my_path + "/my_subdir") | ||
fs.rmdir(my_path) | ||
with pytest.raises(FileNotFoundError): | ||
fs.info(my_path) | ||
|
||
fs = fsspec.filesystem("vsi") | ||
with pytest.raises(Exception): | ||
fs.mkdir(my_path + "/my_subdir", create_parents=False) | ||
with pytest.raises(FileNotFoundError): | ||
fs.info(my_path) | ||
|
||
|
||
def test_gdal_fsspec_makedirs(tmp_path): | ||
|
||
fs = fsspec.filesystem("vsi") | ||
|
||
my_path = str(tmp_path) + "/my_dir" | ||
fs.makedirs(my_path) | ||
assert fs.info(my_path)["type"] == "directory" | ||
with pytest.raises(FileExistsError): | ||
fs.makedirs(my_path) | ||
fs.makedirs(my_path, exist_ok=True) | ||
fs.rmdir(my_path) | ||
|
||
|
||
def test_gdal_fsspec_usable_by_pyarrow_dataset(tmp_vsimem): | ||
|
||
ds = pytest.importorskip("pyarrow.dataset") | ||
|
||
tmp_vsimem_file = str(tmp_vsimem / "tmp.parquet") | ||
gdal.FileFromMemBuffer( | ||
tmp_vsimem_file, open("../ogr/data/parquet/test.parquet", "rb").read() | ||
) | ||
|
||
fs_vsimem = fsspec.filesystem("vsimem") | ||
|
||
assert ( | ||
ds.dataset(tmp_vsimem_file[len("/vsimem") :], filesystem=fs_vsimem) is not None | ||
) | ||
|
||
assert ( | ||
ds.dataset(str(tmp_vsimem)[len("/vsimem") :], filesystem=fs_vsimem) is not None | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
.. | ||
The documentation displayed on this page is automatically generated from | ||
Python docstrings. See https://gdal.org/development/dev_documentation.html | ||
for information on updating this content. | ||
osgeo.gdal_fsspec module | ||
======================== | ||
|
||
.. automodule:: osgeo.gdal_fsspec | ||
:members: | ||
:undoc-members: | ||
:show-inheritance: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,6 +12,7 @@ Submodules | |
|
||
osgeo.gdal | ||
osgeo.gdal_array | ||
osgeo.gdal_fsspec | ||
osgeo.gdalconst | ||
osgeo.gnm | ||
osgeo.ogr | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.