From 5672faf6d150b6340aa2da95aa6e91f9fdff735e Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 25 Aug 2020 14:26:02 +0200 Subject: [PATCH 1/5] Improve packaging 1) Switch from `get_version` to `setuptools_scm` which is more stable 2) Implement the changes @flyingsheep proposed for scanpy (https://github.com/theislab/scanpy/pull/1374/, https://github.com/theislab/scanpy/pull/1376/). This cleans up some redundancies and imports the package dependencies only at run time, not at build time. --- pyproject.toml | 23 +++++++++++++++++------ scirpy/__init__.py | 21 +++++++++------------ scirpy/_compat.py | 16 ++++++++++++++++ scirpy/_metadata.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 scirpy/_metadata.py diff --git a/pyproject.toml b/pyproject.toml index 6f1cc3bac..ebd6b0ac7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,15 @@ [build-system] -requires = ['flit'] -build-backend = 'flit.buildapi' +requires = [ + 'flit_core >=2,<4', + 'setuptools_scm', + 'pytoml', + 'importlib_metadata>=0.7; python_version < "3.8"' +] +build-backend = 'flit_core.buildapi' [tool.flit.metadata] module = 'scirpy' -author = 'Gregor Sturm' +author = 'Gregor Sturm, Tamas Szabo' author-email = 'gregor.sturm@i-med.ac.at' home-page = 'https://github.com/grst/scirpy' description-file = "README.rst" @@ -18,7 +23,6 @@ classifiers = [ ] requires-python = '>= 3.6' requires = [ - 'get_version', 'anndata>=0.7.3', 'scanpy>=1.5.1', 'pandas>=0.21', @@ -31,7 +35,11 @@ requires = [ 'networkx', 'squarify', 'airr', - 'tqdm>=4.29.1' # See https://github.com/icbi-lab/scirpy/issues/128#issuecomment-632646608 + 'tqdm>=4.29.1', # See https://github.com/icbi-lab/scirpy/issues/128#issuecomment-632646608 + # for getting the version + 'setuptools_scm', + 'pytoml', + 'importlib_metadata>=0.7; python_version < "3.8"' ] [tool.flit.metadata.requires-extra] @@ -46,7 +54,7 @@ doc = [ 'sphinx>=3.0.1,<3.1', 'sphinx_autodoc_typehints>=1.8.0', 'sphinx_rtd_theme>=0.4', - 'scanpydoc>=0.4.5', + 'scanpydoc>=0.5', 'typing_extensions; python_version < "3.8"', # for `Literal` # for tutorial 'leidenalg', @@ -57,3 +65,6 @@ doc = [ 'jupyter_client', 'ipykernel', ] + +[tool.flit.metadata.urls] +Documentation = 'https://icbi-lab.github.io/scirpy/' \ No newline at end of file diff --git a/scirpy/__init__.py b/scirpy/__init__.py index bc0cdf215..330c6d821 100644 --- a/scirpy/__init__.py +++ b/scirpy/__init__.py @@ -1,15 +1,12 @@ """Python library for single-cell TCR analysis""" -from get_version import get_version -__version__ = get_version(__file__) -del get_version +from ._metadata import __version__, __author__, __email__, within_flit -__author__ = ", ".join(["Gregor Sturm", "Tamas Szabo"]) - -from scanpy import AnnData, read_h5ad -from . import io -from . import util -from . import _preprocessing as pp -from . import _tools as tl -from . import _plotting as pl -from . import datasets +if not within_flit(): + from scanpy import AnnData, read_h5ad + from . import io + from . import util + from . import _preprocessing as pp + from . import _tools as tl + from . import _plotting as pl + from . import datasets diff --git a/scirpy/_compat.py b/scirpy/_compat.py index 0ef291a27..1712dbef0 100644 --- a/scirpy/_compat.py +++ b/scirpy/_compat.py @@ -13,3 +13,19 @@ def __getitem__(cls, values): class Literal(metaclass=LiteralMeta): pass + + +def pkg_metadata(package): + try: + from importlib.metadata import metadata as m + except ImportError: # < Python 3.8: Use backport module + from importlib_metadata import metadata as m + return m(package) + + +def pkg_version(package): + try: + from importlib.metadata import version as v + except ImportError: # < Python 3.8: Use backport module + from importlib_metadata import version as v + return version.parse(v(package)) diff --git a/scirpy/_metadata.py b/scirpy/_metadata.py new file mode 100644 index 000000000..b79f29c43 --- /dev/null +++ b/scirpy/_metadata.py @@ -0,0 +1,30 @@ +"""Metadata. Adapted from https://github.com/theislab/scanpy/pull/1374/.""" +import traceback +from pathlib import Path + +here = Path(__file__).parent + +try: + from setuptools_scm import get_version + import pytoml + + proj = pytoml.loads((here.parent / "pyproject.toml").read_text()) + metadata = proj["tool"]["flit"]["metadata"] + + __version__ = get_version(root="..", relative_to=__file__) + __author__ = metadata["author"] + __email__ = metadata["author-email"] +except (LookupError, FileNotFoundError): + from ._compat import pkg_metadata + + metadata = pkg_metadata(here.name) + __version__ = metadata["Version"] + __author__ = metadata["Author"] + __email__ = metadata["Author-email"] + + +def within_flit(): + for frame in traceback.extract_stack(): + if frame.name == "get_docstring_and_version_via_import": + return True + return False From 8489ad471a487c766a586d136013873bab873ded Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 25 Aug 2020 14:52:27 +0200 Subject: [PATCH 2/5] github actions fetch depth debug gh-actions Debugging gh-actions Debugging gh-actions --- .editorconfig | 12 ++++++++++ .github/workflows/conda.yml | 14 +++++++----- .github/workflows/docs.yml | 9 ++++---- .github/workflows/pythonpublish.yml | 35 +++++++++++++++-------------- .github/workflows/test.yml | 7 +++--- .gitignore | 1 + pyproject.toml | 11 ++++----- scirpy/_metadata.py | 3 ++- workspace.code-workspace | 8 ------- 9 files changed, 54 insertions(+), 46 deletions(-) create mode 100644 .editorconfig delete mode 100644 workspace.code-workspace diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..abdf1b63a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +max_line_length = 88 + +[*.py] +indent_size = 4 +indent_style = space \ No newline at end of file diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 5fa98585d..341882322 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -5,7 +5,7 @@ on: branches: [master] pull_request: branches: [master] - + jobs: tests: runs-on: ${{ matrix.os }} @@ -14,15 +14,17 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest"] - + steps: - uses: actions/checkout@v2 - - - name: Setup Minoconda + with: + fetch-depth: 0 # required for setuptools-scm + + - name: Setup Minoconda uses: goanpeca/setup-miniconda@v1 with: auto-update-conda: true - + - name: Set-up channels and install conda build run: | conda config --add channels defaults @@ -30,7 +32,7 @@ jobs: conda config --add channels conda-forge conda install -y conda-build conda-verify shell: bash - + - name: build and test package run: | cd .conda diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 64c9bc596..592897fd1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,6 +18,8 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 # required for setuptools-scm - name: Fetch all tags s.t. get_version can retrieve the correct version. run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* @@ -64,11 +66,7 @@ jobs: - name: Install dependencies run: | - python -m pip install flit - flit install --deps develop - env: - FLIT_ROOT_INSTALL: 1 - + pip install .[doc,test,rpack] - name: run sphinx run: | cd docs && make html SPHINXOPTS="-W --keep-going" @@ -101,3 +99,4 @@ jobs: TARGET_FOLDER: ${{ env.target_dir }} CLEAN: true CLEAN_EXCLUDE: '["heads", "pull", "tags"]' + SINGLE_COMMIT: true diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index a1a76346b..05586c5b4 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -6,23 +6,24 @@ on: jobs: deploy: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install flit - flit install - - name: Build and publish - env: - FLIT_USERNAME: ${{ secrets.PYPI_USERNAME }} - FLIT_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - flit build - flit publish + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # required for setuptools-scm + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install flit + flit install + - name: Build and publish + env: + FLIT_USERNAME: ${{ secrets.PYPI_USERNAME }} + FLIT_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + flit build + flit publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aaec0f515..6e2225ff6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 # required for setuptools-scm - uses: actions/cache@v1 with: path: ~/.cache/pip @@ -43,10 +45,7 @@ jobs: pip install $pkg - name: Install dependencies run: | - python -m pip install flit - flit install --deps all - env: - FLIT_ROOT_INSTALL: 1 + pip install .[test,rpack] - name: Check black formatting run: | black --check scirpy tests diff --git a/.gitignore b/.gitignore index 9f4e62221..b60da1f38 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.code-workspace .vscode/* !.vscode/settings.json.default notebooks diff --git a/pyproject.toml b/pyproject.toml index ebd6b0ac7..35e663cd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,9 @@ [build-system] requires = [ - 'flit_core >=2,<4', + 'flit_core >=2,<4', 'setuptools_scm', - 'pytoml', + 'pytoml', + 'wheel', 'importlib_metadata>=0.7; python_version < "3.8"' ] build-backend = 'flit_core.buildapi' @@ -38,12 +39,12 @@ requires = [ 'tqdm>=4.29.1', # See https://github.com/icbi-lab/scirpy/issues/128#issuecomment-632646608 # for getting the version 'setuptools_scm', - 'pytoml', + 'pytoml', 'importlib_metadata>=0.7; python_version < "3.8"' ] [tool.flit.metadata.requires-extra] -optional = [ +rpack = [ 'rectangle-packer', ] test = [ @@ -67,4 +68,4 @@ doc = [ ] [tool.flit.metadata.urls] -Documentation = 'https://icbi-lab.github.io/scirpy/' \ No newline at end of file +Documentation = 'https://icbi-lab.github.io/scirpy/' diff --git a/scirpy/_metadata.py b/scirpy/_metadata.py index b79f29c43..cbee97fc3 100644 --- a/scirpy/_metadata.py +++ b/scirpy/_metadata.py @@ -14,7 +14,8 @@ __version__ = get_version(root="..", relative_to=__file__) __author__ = metadata["author"] __email__ = metadata["author-email"] -except (LookupError, FileNotFoundError): + +except (ImportError, LookupError, FileNotFoundError): from ._compat import pkg_metadata metadata = pkg_metadata(here.name) diff --git a/workspace.code-workspace b/workspace.code-workspace deleted file mode 100644 index 022c3e5cc..000000000 --- a/workspace.code-workspace +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": {} -} From e6d482308fdbc87d20a7eec9bfd76cdd138f76e1 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 25 Aug 2020 16:02:48 +0200 Subject: [PATCH 3/5] Test conda against more python versions --- .conda/meta.yaml | 26 ++++---------------------- .github/workflows/conda.yml | 3 ++- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index f995ee73a..c06365588 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -16,27 +16,9 @@ build: requirements: # The way the version is resolved from SCM (flit + get_version) - # requires all dependencies to be installed at build time. - host: - - python >=3.6 - - flit - - get_version - - anndata >=0.7.3 - - scanpy>=1.5.1 - - pandas>=0.21 - - numpy=1.18 - - scipy - - parasail-python - - scikit-learn - - python-levenshtein - - python-igraph - - networkx - - squarify - - tqdm>=4.29.1 - - airr>=1.2 + # requires all dependencies to be installed at build time. run: - python >=3.6 - - get_version - anndata >=0.7.3 - scanpy>=1.5.1 - pandas>=0.21 @@ -71,11 +53,11 @@ test: about: home: https://icbi-lab.github.io/scirpy - dev_url: https://github.com/icbi-lab/scirpy + dev_url: https://github.com/icbi-lab/scirpy license: BSD-3 license_family: BSD - summary: A Scanpy extension for analyzing single-cell T-cell receptor sequencing data. + summary: A Scanpy extension for analyzing single-cell T-cell receptor sequencing data. extra: identifiers: - - doi:10.1101/2020.04.10.035865 + - doi:10.1101/2020.04.10.035865 diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 341882322..1a53d441b 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -13,6 +13,7 @@ jobs: strategy: fail-fast: false matrix: + python-version: [3.6, 3.7, 3.8] os: ["ubuntu-latest", "macos-latest"] steps: @@ -36,5 +37,5 @@ jobs: - name: build and test package run: | cd .conda - conda build . --no-anaconda-upload + conda build . --no-anaconda-upload --python ${{ matrix.python-version }} shell: bash From aa29715883bdc149a16077afb2eac651e2268638 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 25 Aug 2020 16:03:11 +0200 Subject: [PATCH 4/5] Fix #175 --- .editorconfig | 5 ++++- docs/Makefile | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.editorconfig b/.editorconfig index abdf1b63a..ece403346 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,4 +9,7 @@ max_line_length = 88 [*.py] indent_size = 4 -indent_style = space \ No newline at end of file +indent_style = space + +[Makefile] +indent_style = tab diff --git a/docs/Makefile b/docs/Makefile index 5053252f7..ec5917ad9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -14,7 +14,7 @@ help: .PHONY: help Makefile -livehtml: | clean +livehtml: | clean symlink sleep 1 && touch *.rst sphinx-autobuild \ -p 0 \ @@ -32,7 +32,15 @@ livehtml: | clean --ignore "*.tex" \ -b html $(SOURCEDIR) $(BUILDDIR)/html -clean: +html: | symlink + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +symlink: + mkdir -p generated + # link static to the generated directory, s.t. relative links keep working. + ln -sn ../_static generated/_static || true + +clean: rm -rf generated @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) From 1c2becd78dfd6d49bbbd3afa9ea96a11cfaf4913 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 25 Aug 2020 16:11:46 +0200 Subject: [PATCH 5/5] Fix conda test Fix conda recipe fix symlink switch to sphinx scanpydoc theme Add missing import --- .conda/meta.yaml | 7 +++++++ .github/workflows/conda.yml | 4 ++-- docs/Makefile | 4 ++-- docs/conf.py | 4 ++-- pyproject.toml | 2 +- scirpy/_compat.py | 2 ++ 6 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index c06365588..aaa3baa36 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -17,6 +17,13 @@ build: requirements: # The way the version is resolved from SCM (flit + get_version) # requires all dependencies to be installed at build time. + host: + - python >=3.6 + - flit + - setuptools_scm + - pytoml + - importlib_metadata + run: - python >=3.6 - anndata >=0.7.3 diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 1a53d441b..94db68983 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -9,7 +9,6 @@ on: jobs: tests: runs-on: ${{ matrix.os }} - name: conda ${{ matrix.os }} strategy: fail-fast: false matrix: @@ -25,6 +24,7 @@ jobs: uses: goanpeca/setup-miniconda@v1 with: auto-update-conda: true + python-version: ${{ matrix.python-version }} - name: Set-up channels and install conda build run: | @@ -37,5 +37,5 @@ jobs: - name: build and test package run: | cd .conda - conda build . --no-anaconda-upload --python ${{ matrix.python-version }} + conda build --no-anaconda-upload . shell: bash diff --git a/docs/Makefile b/docs/Makefile index ec5917ad9..8f5900e76 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -36,9 +36,9 @@ html: | symlink @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) symlink: - mkdir -p generated + mkdir -p _build/html/generated # link static to the generated directory, s.t. relative links keep working. - ln -sn ../_static generated/_static || true + ln -sn ../_static _build/html/generated/_static || true clean: rm -rf generated diff --git a/docs/conf.py b/docs/conf.py index 20dd79cb3..a4af8e86f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -93,7 +93,7 @@ # -- HTML styling ---------------------------------------------------------------------- -html_theme = "sphinx_rtd_theme" +html_theme = "scanpydoc" # add custom stylesheet # https://stackoverflow.com/a/43186995/2340703 html_static_path = ["_static"] @@ -110,7 +110,7 @@ def setup(app): - app.add_css_file("custom.css") + pass # -- Supress 'reference target not found' errors --------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 35e663cd4..1639c7faf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,8 +54,8 @@ test = [ doc = [ 'sphinx>=3.0.1,<3.1', 'sphinx_autodoc_typehints>=1.8.0', - 'sphinx_rtd_theme>=0.4', 'scanpydoc>=0.5', + 'sphinx_rtd_theme>=0.4', 'typing_extensions; python_version < "3.8"', # for `Literal` # for tutorial 'leidenalg', diff --git a/scirpy/_compat.py b/scirpy/_compat.py index 1712dbef0..23b1b2e76 100644 --- a/scirpy/_compat.py +++ b/scirpy/_compat.py @@ -1,3 +1,5 @@ +from packaging import version + try: from typing import Literal except ImportError: