diff --git a/.github/workflows/ci-experiment.yml b/.github/workflows/ci-experiment.yml new file mode 100644 index 00000000..2507779b --- /dev/null +++ b/.github/workflows/ci-experiment.yml @@ -0,0 +1,111 @@ +name: CI experiments + +# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + pytest: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04] + python-version: [2.7, 3.6, 3.8] + + # Timeout: https://stackoverflow.com/a/59076067/4521646 + timeout-minutes: 35 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update Pip + run: | + pip install --quiet "pip>=20.1" --upgrade --user # needed for get pip cacher folder + + - name: Set py2.7 dependencies + if: matrix.python-version == 2.7 + run: | + cp -r requirements-py27.txt requirements.txt + + # Note: This uses an internal pip API and may not always work + # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}- + + - name: Install dependencies + run: | + pip install $(grep "numpy" requirements.txt) + # required for matplotlib @py2 + pip install -U backports.functools_lru_cache + # v46 crashes openslide-python install + pip install "setuptools<46" -U + pip install --requirement requirements.txt --quiet + python --version + pip --version + pip list + shell: bash + + - name: update OS + run: | + sudo apt-get install -y openslide-tools tree + + - name: Dataset processing + run: | + mkdir output + python bm_dataset/rescale_tissue_images.py -i "./data-images/rat-kidney_/scale-5pc/*.jpg" --scales 5 -ext .png --nb_workers 2 + python bm_dataset/rescale_tissue_landmarks.py -a ./data-images -d ./output --nb_selected 0.5 --nb_total 200 + python bm_dataset/generate_regist_pairs.py -i "./data-images/images/artificial_*.jpg" -l "./data-images/landmarks/artificial_*.csv" -csv ./data-images/cover_artificial.csv --mode each2all + + - name: General experiments + run: | + mkdir applications && mkdir results && touch configs/sample_config.yaml + # python bm_experiments/bm_comp_perform.py -o ./results -n 1 # reach time-out while running in 36 threads + python birl/bm_template.py -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results --visual --unique -cfg configs/sample_config.yaml + rm ./data-images/*_/*/*_HE.csv # remove target landmarks from histol. tissue + python birl/bm_template.py -n anhir -t ./data-images/pairs-imgs-lnds_histol.csv -d ./data-images -o ./results --preprocessing matching-rgb gray -cfg configs/sample_config.yaml + python bm_experiments/evaluate_experiment.py -d ./data-images -e ./results/BmTemplate_anhir --visual + tree ./results/BmTemplate + shell: bash + + - name: prepare Fiji + run: | + # todo: cache this app and dowload only if needed + wget https://downloads.imagej.net/fiji/archive/20200708-1553/fiji-linux64.zip -O applications/fiji-linux64.zip --progress=bar:force:noscroll + unzip -q applications/fiji-linux64.zip -d applications/ + rm applications/fiji-linux64.zip + shell: bash + + - name: Experiment - bUnwarpJ + env: + DISPLAY: "" + run: | + rm -rf results && mkdir results + python bm_experiments/bm_bUnwarpJ.py -n mix -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_bUnwarpJ_histol.yaml --preprocessing matching-rgb --visual + python bm_experiments/bm_bUnwarpJ.py -n anhir -t ./data-images/pairs-imgs-lnds_histol.csv -d ./data-images -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_bUnwarpJ-SIFT_histol.yaml --unique + tree -L 3 ./results + rm -rf ./applications/Fiji.app + shell: bash + + - name: Experiment - RVSS + env: + DISPLAY: "" + run: | + rm -rf results && mkdir results + python bm_experiments/bm_RVSS.py -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_RVSS_histol.yaml --visual + tree -L 3 ./results + rm -rf ./applications/Fiji.app + shell: bash diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml new file mode 100644 index 00000000..03110724 --- /dev/null +++ b/.github/workflows/ci-testing.yml @@ -0,0 +1,124 @@ +name: CI testing + +# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows +on: # Trigger the workflow on push or pull request, but only for the master branch + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + pytest: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04, windows-2019, macOS-10.15] + python-version: [2.7, 3.6, 3.8] + requires: ['minimal', 'latest'] + exclude: + - python-version: 2.7 + requires: 'minimal' + - python-version: 3.8 + requires: 'minimal' + + # Timeout: https://stackoverflow.com/a/59076067/4521646 + timeout-minutes: 35 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update Pip + run: | + pip install --quiet "pip>=20.1" --upgrade --user # needed for get pip cacher folder + + - name: Set py2.7 dependencies + if: matrix.python-version == 2.7 + run: | + cp -r requirements-py27.txt requirements.txt + + - name: Set min. dependencies + if: matrix.requires == 'minimal' + run: | + python -c "fname = 'requirements.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" + python -c "fname = 'tests/requirements.txt' ; req = open(fname).read().replace('>=', '==') ; open(fname, 'w').write(req)" + + # Note: This uses an internal pip API and may not always work + # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow + - name: Get pip cache + id: pip-cache + run: | + python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- + + - name: Install dependencies + run: | + pip install $(grep "numpy" requirements.txt) + # required for matplotlib @py2 + pip install -U backports.functools_lru_cache + # v46 crashes openslide-python install + pip install "setuptools<46" -U + pip install --requirement requirements.txt --quiet --upgrade + pip install --requirement tests/requirements.txt --quiet --upgrade + python --version + pip --version + pip list + shell: bash + + - name: Tests + env: + DISPLAY: "" + run: | + check-manifest --ignore tox.ini + python setup.py check --metadata --strict + python setup.py install --dry-run --user + coverage run --source birl -m py.test birl tests bm_dataset bm_experiments bm_ANHIR -v --doctest-modules --durations=25 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml + flake8 . + + - name: Upload pytest test results + uses: actions/upload-artifact@master + with: + name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }} + path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml + # Use always() to always run this step to publish test results when there are test failures + if: always() + + - name: Statistics + if: success() + run: | + coverage report + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + if: success() + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml + flags: cpu,pytest + name: CPU-coverage + fail_ci_if_error: false + + - name: Run sample experiment + run: | + mkdir results + touch configs/sample_config.yaml + python bm_experiments/bm_comp_perform.py -o ./results -n 1 + python birl/bm_template.py -t ./data-images/pairs-imgs-lnds_mix.csv -o ./results --visual --unique -cfg configs/sample_config.yaml + # remove target landmarks from histol. tissue + rm ./data-images/*_/*/*_HE.csv + python birl/bm_template.py -n anhir -t ./data-images/pairs-imgs-lnds_histol.csv -d ./data-images -o ./results --preprocessing matching-rgb gray -cfg configs/sample_config.yaml + python bm_experiments/evaluate_experiment.py -d ./data-images -e ./results/BmTemplate_anhir --visual + shell: bash \ No newline at end of file diff --git a/birl/__init__.py b/birl/__init__.py index dd90a825..fdca3e49 100644 --- a/birl/__init__.py +++ b/birl/__init__.py @@ -13,7 +13,7 @@ traceback.print_exc() -__version__ = "0.2.4" +__version__ = "0.2.5" __author__ = "Jiri Borovec" __author_email__ = "jiri.borovec@fel.cvut.cz" __license__ = "BSD 3-clause" diff --git a/birl/utilities/registration.py b/birl/utilities/registration.py index 0af07863..bc6236d5 100644 --- a/birl/utilities/registration.py +++ b/birl/utilities/registration.py @@ -34,7 +34,7 @@ def estimate_affine_transform(points_0, points_1): >>> pts0 = np.array([[4., 116.], [4., 4.], [26., 4.], [26., 116.]], dtype=int) >>> pts1 = np.array([[61., 56.], [61., -56.], [39., -56.], [39., 56.]]) >>> mx, mx_inv, pts0_w, pts1_w = estimate_affine_transform(pts0, pts1) - >>> np.round(mx, 2) + >>> np.round(mx, 2) + 0. # eliminate negative zeros array([[ -1., 0., 65.], [ 0., 1., -60.], [ 0., 0., 1.]]) diff --git a/bm_experiments/__init__.py b/bm_experiments/__init__.py index 6ff462df..8169c3e7 100644 --- a/bm_experiments/__init__.py +++ b/bm_experiments/__init__.py @@ -1,3 +1,3 @@ import birl -birl +birl.utilities diff --git a/bm_experiments/bm_comp_perform.py b/bm_experiments/bm_comp_perform.py index 87d9f4f2..489aa3a4 100644 --- a/bm_experiments/bm_comp_perform.py +++ b/bm_experiments/bm_comp_perform.py @@ -35,6 +35,8 @@ from functools import partial import tqdm +import matplotlib +matplotlib.use('Agg') import numpy as np from skimage import data, io, __version__ as skimage__version from skimage.transform import resize, warp, AffineTransform diff --git a/circle.yml b/circle.yml index 9cc6f500..92a292d2 100644 --- a/circle.yml +++ b/circle.yml @@ -190,6 +190,6 @@ workflows: - Formatting - Py2-Tests - Py3-Tests - - Py2-Experiments - - Py3-Experiments + #- Py2-Experiments + #- Py3-Experiments - Build-Docs diff --git a/requirements.txt b/requirements.txt index 40a04d93..8f700f46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ -numpy>=1.8.2 -scipy>=0.10.0 -pandas>=0.17.1 -six>=1.7.3 -pillow>=2.1.0, !=7.1 # 7.x fails to load PNG via skimage +numpy>=1.13.3 +scipy>=1.0 +pandas>=0.21 +six>=1.10 +pillow>=4.0, !=7.1 # 7.x fails to load PNG via skimage matplotlib>=2.0.2 tqdm>=4.7.4 -pyyaml -scikit-image -opencv-python-headless -openslide-python -nibabel +pyyaml>=3.12 +scikit-image>=0.13 +opencv-python-headless>=3.4.0.14 +openslide-python>=1.0 +nibabel>=2.1 SimpleITK psutil pathos \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index 075212ad..b94ff778 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,7 @@ -nose>=1.3.7 +nose>=1.0 coverage codecov -pytest>=3.0.5 +pytest>4.0 pytest-cov flake8 check-manifest