From 80252865ae788879d64eeb04fc49f73662f72cd9 Mon Sep 17 00:00:00 2001 From: "Simeon H.K. Fitch" Date: Sun, 13 Mar 2022 13:04:58 -0400 Subject: [PATCH] Misc version fiddling. --- .../datasource/tiles/TilesDataSource.scala | 1 - project/PythonBuildPlugin.scala | 4 +-- project/RFDependenciesPlugin.scala | 6 ++-- pyrasterframes/src/main/python/MANIFEST.in | 2 +- .../src/main/python/deps/jars/README.md | 4 --- .../main/python/pyrasterframes/pyproject.toml | 4 --- .../src/main/python/pyrasterframes/utils.py | 30 +++++++------------ .../main/python/requirements-condaforge.txt | 2 +- pyrasterframes/src/main/python/setup.py | 5 +--- version.sbt | 2 +- 10 files changed, 19 insertions(+), 41 deletions(-) delete mode 100644 pyrasterframes/src/main/python/deps/jars/README.md delete mode 100644 pyrasterframes/src/main/python/pyrasterframes/pyproject.toml diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/tiles/TilesDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/tiles/TilesDataSource.scala index 92c319632..beb001ef9 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/tiles/TilesDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/tiles/TilesDataSource.scala @@ -87,7 +87,6 @@ class TilesDataSource extends DataSourceRegister with CreatableRelationProvider val fName = "catalog.csv" val hPath = new Path(new Path(pathURI), "_" + fName) pipeline - .coalesce(1) .write .option("header", "true") .csv(hPath.toString) diff --git a/project/PythonBuildPlugin.scala b/project/PythonBuildPlugin.scala index c48df5e16..d5cecd123 100644 --- a/project/PythonBuildPlugin.scala +++ b/project/PythonBuildPlugin.scala @@ -83,7 +83,7 @@ object PythonBuildPlugin extends AutoPlugin { val log = streams.value.log val buildDir = (Python / target).value val asmbl = (Compile / assembly).value - val dest = buildDir / "deps" / "jars" / asmbl.getName + val dest = buildDir / "pyrasterframes" / "jars" / asmbl.getName IO.copyFile(asmbl, dest) log.info(s"PyRasterFrames assembly written to '$dest'") dest @@ -93,7 +93,7 @@ object PythonBuildPlugin extends AutoPlugin { val log = streams.value.log val buildDir = (Python / target).value - val jars = (buildDir / "deps" / "jars" ** "*.jar").get() + val jars = (buildDir / "pyrasterframes" / "jars" ** "*.jar").get() if (jars.size > 1) { throw new MessageOnlyException("Two assemblies found in the package. Run 'clean'.\n" + jars.mkString(", ")) } diff --git a/project/RFDependenciesPlugin.scala b/project/RFDependenciesPlugin.scala index 8ac74a84f..8929ab69a 100644 --- a/project/RFDependenciesPlugin.scala +++ b/project/RFDependenciesPlugin.scala @@ -69,10 +69,10 @@ object RFDependenciesPlugin extends AutoPlugin { "oss-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", "jitpack" at "https://jitpack.io" ), - // dependencyOverrides += "com.azavea.gdal" % "gdal-warp-bindings" % "33.f746890", // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py rfSparkVersion := "3.1.2", - rfGeoTrellisVersion := "3.6.1-SNAPSHOT", - rfGeoMesaVersion := "3.2.0" + rfGeoTrellisVersion := "3.6.1", + rfGeoMesaVersion := "3.2.0", + excludeDependencies += "log4j" % "log4j" ) } diff --git a/pyrasterframes/src/main/python/MANIFEST.in b/pyrasterframes/src/main/python/MANIFEST.in index 68903bdfe..88f63e05a 100644 --- a/pyrasterframes/src/main/python/MANIFEST.in +++ b/pyrasterframes/src/main/python/MANIFEST.in @@ -1,3 +1,3 @@ global-exclude *.py[cod] __pycache__ .DS_Store -recursive-include deps/jars *.jar +recursive-include pyrasterframes/jars *.jar diff --git a/pyrasterframes/src/main/python/deps/jars/README.md b/pyrasterframes/src/main/python/deps/jars/README.md deleted file mode 100644 index 693a02cb2..000000000 --- a/pyrasterframes/src/main/python/deps/jars/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# pyrasterframes.jars - -Submodule containing JARs needed for pyrasterframe - diff --git a/pyrasterframes/src/main/python/pyrasterframes/pyproject.toml b/pyrasterframes/src/main/python/pyrasterframes/pyproject.toml deleted file mode 100644 index b62a44d4a..000000000 --- a/pyrasterframes/src/main/python/pyrasterframes/pyproject.toml +++ /dev/null @@ -1,4 +0,0 @@ -[build-system] -# Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel"] # PEP 508 specifications. -build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/pyrasterframes/src/main/python/pyrasterframes/utils.py b/pyrasterframes/src/main/python/pyrasterframes/utils.py index d7ee97e2e..e88bde590 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/utils.py +++ b/pyrasterframes/src/main/python/pyrasterframes/utils.py @@ -23,9 +23,9 @@ from pyspark import SparkConf import os from . import RFContext -from typing import Union, Dict +from typing import Union, Dict, Optional -__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly", "gdal_version", 'is_notebook', 'gdal_version', 'build_info', 'quiet_logs'] +__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly", "gdal_version", 'gdal_version', 'build_info', 'quiet_logs'] def find_pyrasterframes_jar_dir() -> str: @@ -39,20 +39,10 @@ def find_pyrasterframes_jar_dir() -> str: try: module_home = find_spec("pyrasterframes").origin jar_dir = os.path.join(os.path.dirname(module_home), 'jars') - except ImportError: - pass - - # Case for when we're running from source build - if jar_dir is None or not os.path.exists(jar_dir): - def pdir(curr): - return os.path.dirname(curr) - - here = pdir(os.path.realpath(__file__)) - target_dir = pdir(pdir(here)) - # See if we're running outside of sbt build and adjust - if os.path.basename(target_dir) != "target": - target_dir = os.path.join(pdir(pdir(target_dir)), 'target') - jar_dir = os.path.join(target_dir, 'scala-2.12') + except ImportError as e: + import logging + logging.critical("Error finding runtime JAR directory", exc_info=e) + raise e return os.path.realpath(jar_dir) @@ -62,9 +52,9 @@ def find_pyrasterframes_assembly() -> Union[bytes, str]: jarpath = glob.glob(os.path.join(jar_dir, 'pyrasterframes-assembly*.jar')) if not len(jarpath) == 1: - raise RuntimeError(""" -Expected to find exactly one assembly. Found '{}' instead. -Try running 'sbt pyrasterframes/clean pyrasterframes/package' first. """.format(jarpath)) + raise RuntimeError(f""" +Expected to find exactly one assembly in '{jar_dir}'. +Found '{jarpath}' instead.""") return jarpath[0] @@ -74,7 +64,7 @@ def quiet_logs(sc): logger.LogManager.getLogger("akka").setLevel(logger.Level.ERROR) -def create_rf_spark_session(master="local[*]", **kwargs: str) -> SparkSession: +def create_rf_spark_session(master="local[*]", **kwargs: str) -> Optional[SparkSession]: """ Create a SparkSession with pyrasterframes enabled and configured. """ jar_path = find_pyrasterframes_assembly() diff --git a/pyrasterframes/src/main/python/requirements-condaforge.txt b/pyrasterframes/src/main/python/requirements-condaforge.txt index 900b3789f..827a7f431 100644 --- a/pyrasterframes/src/main/python/requirements-condaforge.txt +++ b/pyrasterframes/src/main/python/requirements-condaforge.txt @@ -1,4 +1,4 @@ # These packages should be installed from conda-forge, given their complex binary components. -gdal==3.1.2 +gdal rasterio[s3] rtree diff --git a/pyrasterframes/src/main/python/setup.py b/pyrasterframes/src/main/python/setup.py index a3d953671..d7a665cdf 100644 --- a/pyrasterframes/src/main/python/setup.py +++ b/pyrasterframes/src/main/python/setup.py @@ -147,7 +147,7 @@ def dest_file(self, src_file): matplotlib = 'matplotlib' fiona = 'fiona' folium = 'folium' -gdal = 'gdal==3.1.2' +gdal = 'gdal' geopandas = 'geopandas' ipykernel = 'ipykernel' ipython = 'ipython' @@ -232,9 +232,6 @@ def dest_file(self, src_file): 'geomesa_pyspark', 'pyrasterframes.jars', ], - package_dir={ - 'pyrasterframes.jars': 'deps/jars' - }, package_data={ 'pyrasterframes.jars': ['*.jar'] }, diff --git a/version.sbt b/version.sbt index aa2cbb42e..1f9ae9ed8 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "0.10.1-SNAPSHOT" +ThisBuild / version := "0.10.1"