Add regression tests (#4)

* Add missing requirement for TIFF reading * Add regression tests for BBBC039 * Fix bug in regression tests * Add regression tests for GOWT1 datasets * Add regression tests for NIH3T3 datasets * Add regression tests for U2OS dataset * Add regression tests workflow * Update workflow * Fix workflow * Fix expected regression test results * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Update regressiontests.yml * Update regressiontests.yml * Update regressiontests.yml * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Fix workflow * Add `SUPERDSM_INTERMEDIATE_OUTPUT` environment variable Set `SUPERDSM_INTERMEDIATE_OUTPUT=0` to disable intermediate console output. * Add `MKL_DEBUG_CPU_TYPE=5` to workflow * Update regressiontests.yml * Update regressiontests.yml * Update regressiontests.yml * Fix workflow * Set workflow job timeout to 24 hours * Fix dependency versions for reproducibility * Fix conda channels * Disable `use_only_tar_bz2` * Add test example * Pin all dependency versions * Unset `MKL_DEBUG_CPU_TYPE=5` * Add CPU vendor recognition * Add more images to test task * Add missing expected test results * Set `MKL_DEBUG_CPU_TYPE=1` on non-AMD CPUs * Upload artifact with results upon failure * Unset `MKL_DEBUG_CPU_TYPE` if set to `5` on Intel CPUs * Add trigger * Adapt expected results of regression tests for Intel Xeon CPU * Activate remaining regression tests * Fix version of `upload-artifact` * Add debug info * Fix * Reenable test * Combine regression tests into bash scripts * Add testsuite workflow * Update workflow name * Add dry run for debugging * Fix regression tests * Add regression test results for AMD Threadripper 3970X CPU * Do not change `MKL_DEBUG_CPU_TYPE` on Intel CPUs * Fix regression tests expected results * Fix expected regression test results for AMD CPU * Fix
BMCV · Feb 18, 2024 · 15b834c · 15b834c
1 parent eb1bf50
commit 15b834c
Show file tree

Hide file tree

Showing 1,692 changed files with 118,968 additions and 15 deletions.
diff --git a/.github/workflows/regressiontests.yml b/.github/workflows/regressiontests.yml
@@ -0,0 +1,71 @@
+name: Regression tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: ['develop']
+    paths:
+      - .github/workflows/regressiontests.yml
+      - superdsm
+      - examples
+      - superdsm.yml
+      - tests/regression
+
+jobs:
+
+  regression_tests:
+    name: "Test: ${{ matrix.taskdir }}"
+    timeout-minutes: 1440
+    runs-on: self-hosted
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        taskdir:
+          - U2OS
+          - NIH3T3
+          - GOWT1-1
+          - GOWT1-2
+          - BBBC039
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: 'latest'
+          channels: conda-forge, bioconda, defaults
+          auto-update-conda: true
+          auto-activate-base: false
+          activate-environment: superdsm
+          environment-file: superdsm.yml
+
+      - name: Download image data
+        run: |
+          cd examples
+          python load_data.py
+
+      - name: Run SuperDSM
+        run: |
+            python -m "superdsm.batch" examples --task-dir "${{ matrix.taskdir }}"
+            python -m "superdsm.batch" examples --task-dir "${{ matrix.taskdir }}" --run
+        env:
+          SUPERDSM_INTERMEDIATE_OUTPUT: false
+
+      - name: Validate results
+        id: validation
+        run: |
+          mkdir "actual_csv"
+          sh "tests/regression/validate-${{ matrix.taskdir }}.sh" "actual_csv"
+
+      - name: Upload artifact
+        if: failure() && steps.validation.outcome != 'success'
+        uses: actions/upload-artifact@v3  ## v4 requires GLIBC_2.28 which is not found on host
+        with:
+          name: Results ${{ matrix.taskdir }}
+          path: actual_csv/${{ matrix.taskdir }}
diff --git a/.github/workflows/testsuite.yml b/.github/workflows/testsuite.yml
@@ -0,0 +1,34 @@
+name: Test suite
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: ['develop']
+
+jobs:
+
+  run_testsuite:
+    name: Test suite
+    timeout-minutes: 1440
+    runs-on: self-hosted
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: 'latest'
+          channels: conda-forge, bioconda, defaults
+          auto-update-conda: true
+          auto-activate-base: false
+          activate-environment: superdsm
+          environment-file: superdsm.yml
+
+      - name: Run SuperDSM
+        run: python -m "unittest"
diff --git a/.github/workflows/validate_pr.yml b/.github/workflows/validate_pr.yml
@@ -0,0 +1,18 @@
+name: Validate pull request
+
+on:
+  pull_request:
+
+jobs:
+
+  validate_branches:
+    name: Validate branches
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Validate branches
+        if: github.base_ref == 'master' && github.head_ref != 'develop'
+        run: |
+          echo "Contributions should be made against the develop branch, see README.rst."
+          exit 1
diff --git a/requirements.txt b/requirements.txt
@@ -8,3 +8,4 @@ cvxopt==1.2.6
 cvxpy==1.1.13
 matplotlib>=3.0
 mkl>=2020.0
+imagecodecs==2022.9.26
diff --git a/superdsm.yml b/superdsm.yml
@@ -2,18 +2,202 @@ name: superdsm
 channels:
 - conda-forge
 - bioconda
+- defaults
 dependencies:
-- python==3.8.5
-- numpy==1.20
-- scipy==1.6.3
-- scikit-image==0.18.1
-- ipython>=7.31.1
-- dill==0.3.2
-- cvxopt==1.2.6
-- cvxpy==1.1.13
-- matplotlib>=3.0
-- mkl==2020.0
-- blas==1.0=mkl
-- pip
+- _libgcc_mutex=0.1=conda_forge
+- _openmp_mutex=4.5=2_kmp_llvm
+- alsa-lib=1.2.3.2=h166bdaf_0
+- aom=3.5.0=h27087fc_0
+- asttokens=2.4.1=pyhd8ed1ab_0
+- backcall=0.2.0=pyh9f0ad1d_0
+- blas=1.0=mkl
+- blosc=1.21.5=h0f2a231_0
+- brotli=1.0.9=h166bdaf_9
+- brotli-bin=1.0.9=h166bdaf_9
+- brotli-python=1.0.9=py38hfa26641_9
+- brunsli=0.1=h9c3ff4c_0
+- bzip2=1.0.8=hd590300_5
+- c-ares=1.26.0=hd590300_0
+- c-blosc2=2.13.2=hb4ffafa_0
+- ca-certificates=2024.2.2=hbcca054_0
+- certifi=2024.2.2=pyhd8ed1ab_0
+- cfitsio=4.1.0=hd9d235c_0
+- charls=2.3.4=h9c3ff4c_0
+- charset-normalizer=3.3.2=pyhd8ed1ab_0
+- click=8.1.7=unix_pyh707e725_0
+- cloudpickle=3.0.0=pyhd8ed1ab_0
+- contourpy=1.1.1=py38h7f3f72f_1
+- cvxopt=1.2.6=py38hd3996d2_1
+- cvxpy=1.1.13=py38h578d9bd_0
+- cvxpy-base=1.1.13=py38h1abd341_0
+- cycler=0.12.1=pyhd8ed1ab_0
+- cytoolz=0.12.3=py38h01eb140_0
+- dask-core=2023.5.0=pyhd8ed1ab_0
+- dav1d=1.0.0=h166bdaf_1
+- dbus=1.13.6=hfdff14a_1
+- decorator=5.1.1=pyhd8ed1ab_0
+- dill=0.3.2=pyh9f0ad1d_0
+- dsdp=5.8=hd9d9efa_1203
+- ecos=2.0.10=py38h71d37f0_1
+- executing=2.0.1=pyhd8ed1ab_0
+- expat=2.5.0=hcb278e6_1
+- fftw=3.3.10=nompi_hc118613_108
+- fontconfig=2.14.2=h14ed4e7_0
+- fonttools=4.48.1=py38h01eb140_0
+- freetype=2.12.1=h267a509_2
+- fsspec=2024.2.0=pyhca7485f_0
+- gettext=0.21.1=h27087fc_0
+- giflib=5.2.1=h0b41bf4_3
+- glib=2.66.3=h58526e2_0
+- glpk=4.65=h9202a9a_1004
+- gmp=6.3.0=h59595ed_0
+- gsl=2.7=he838d99_0
+- gst-plugins-base=1.14.5=h0935bb2_2
+- gstreamer=1.14.5=h36ae1b5_2
+- icu=67.1=he1b5a44_0
+- idna=3.6=pyhd8ed1ab_0
+- imagecodecs=2022.8.8=py38hf09e3b1_5
+- imageio=2.34.0=pyh4b66e23_0
+- importlib-metadata=7.0.1=pyha770c72_0
+- importlib_metadata=7.0.1=hd8ed1ab_0
+- importlib_resources=6.1.1=pyhd8ed1ab_0
+- ipython=8.12.2=pyh41d4057_0
+- jedi=0.19.1=pyhd8ed1ab_0
+- jpeg=9e=h0b41bf4_3
+- jxrlib=1.1=hd590300_3
+- keyutils=1.6.1=h166bdaf_0
+- kiwisolver=1.4.5=py38h7f3f72f_1
+- krb5=1.20.1=hf9c8cef_0
+- lcms2=2.14=h6ed2654_0
+- ld_impl_linux-64=2.40=h41732ed_0
+- lerc=4.0.0=h27087fc_0
+- libaec=1.1.2=h59595ed_1
+- libavif=0.10.1=h5cdd6b5_2
+- libblas=3.9.0=1_h86c2bf4_netlib
+- libbrotlicommon=1.0.9=h166bdaf_9
+- libbrotlidec=1.0.9=h166bdaf_9
+- libbrotlienc=1.0.9=h166bdaf_9
+- libcblas=3.9.0=5_h92ddd45_netlib
+- libclang=10.0.1=default_hde54327_1
+- libcurl=7.87.0=h6312ad2_0
+- libdeflate=1.14=h166bdaf_0
+- libedit=3.1.20191231=he28a2e2_2
+- libev=4.33=hd590300_2
+- libevent=2.1.10=h9b69904_4
+- libexpat=2.5.0=hcb278e6_1
+- libffi=3.2.1=he1b5a44_1007
+- libgcc-ng=13.2.0=h807b86a_5
+- libgfortran-ng=13.2.0=h69a702a_5
+- libgfortran5=13.2.0=ha4646dd_5
+- libglib=2.66.3=hbe7bbb4_0
+- libiconv=1.17=hd590300_2
+- liblapack=3.9.0=5_h92ddd45_netlib
+- libllvm10=10.0.1=he513fc3_3
+- libnghttp2=1.51.0=hdcd2b5c_0
+- libpng=1.6.42=h2797004_0
+- libpq=12.15=h37d81fd_1
+- libsqlite=3.45.1=h2797004_0
+- libssh2=1.10.0=haa6b8db_3
+- libstdcxx-ng=13.2.0=h7e041cc_5
+- libtiff=4.4.0=h82bc61c_5
+- libuuid=2.38.1=h0b41bf4_0
+- libwebp-base=1.3.2=hd590300_0
+- libxcb=1.16=hd590300_0
+- libxkbcommon=0.10.0=he1b5a44_0
+- libxml2=2.9.10=h68273f3_2
+- libzlib=1.2.13=hd590300_5
+- libzopfli=1.0.3=h9c3ff4c_0
+- llvm-openmp=17.0.6=h4dfa4b3_0
+- locket=1.0.0=pyhd8ed1ab_0
+- lz4-c=1.9.4=hcb278e6_0
+- matplotlib=3.7.1=py38h578d9bd_0
+- matplotlib-base=3.7.1=py38h417a72b_1
+- matplotlib-inline=0.1.6=pyhd8ed1ab_0
+- metis=5.1.0=h59595ed_1007
+- mkl=2020.0=166
+- mpfr=4.2.1=h9458935_0
+- munkres=1.1.4=pyh9f0ad1d_0
+- mysql-common=8.0.32=h14678bc_0
+- mysql-libs=8.0.32=h54cf53e_0
+- ncurses=6.4=h59595ed_2
+- networkx=3.1=pyhd8ed1ab_0
+- nspr=4.35=h27087fc_0
+- nss=3.97=h1d7d5a4_0
+- numpy=1.20.0=py38h18fd61f_0
+- openjpeg=2.5.0=h7d73246_1
+- openssl=1.1.1w=hd590300_0
+- osqp=0.6.2.post0=py38h43a58ef_3
+- packaging=23.2=pyhd8ed1ab_0
+- parso=0.8.3=pyhd8ed1ab_0
+- partd=1.4.1=pyhd8ed1ab_0
+- pcre=8.45=h9c3ff4c_0
+- pexpect=4.9.0=pyhd8ed1ab_0
+- pickleshare=0.7.5=py_1003
+- pillow=10.2.0=py38h5eee18b_0
+- pip=24.0=pyhd8ed1ab_0
+- platformdirs=4.2.0=pyhd8ed1ab_0
+- pooch=1.8.0=pyhd8ed1ab_0
+- prompt-toolkit=3.0.42=pyha770c72_0
+- prompt_toolkit=3.0.42=hd8ed1ab_0
+- pthread-stubs=0.4=h36c2ea0_1001
+- ptyprocess=0.7.0=pyhd3deb0d_0
+- pure_eval=0.2.2=pyhd8ed1ab_0
+- pygments=2.17.2=pyhd8ed1ab_0
+- pyparsing=3.1.1=pyhd8ed1ab_0
+- pyqt=5.12.3=py38h578d9bd_8
+- pyqt-impl=5.12.3=py38h0ffb2e6_8
+- pyqt5-sip=4.19.18=py38h709712a_8
+- pyqtchart=5.12=py38h7400c14_8
+- pyqtwebengine=5.12.1=py38h7400c14_8
+- pysocks=1.7.1=pyha2e5f31_6
+- python=3.8.5=h1103e12_9_cpython
+- python-dateutil=2.8.2=pyhd8ed1ab_0
+- python_abi=3.8=4_cp38
+- pywavelets=1.3.0=py38h71d37f0_1
+- pyyaml=6.0.1=py38h01eb140_1
+- qdldl-python=0.1.5=py38h43a58ef_2
+- qt=5.12.9=h1f2b2cb_0
+- readline=8.2=h8228510_1
+- requests=2.31.0=pyhd8ed1ab_0
+- scikit-image=0.18.1=py38h51da96c_0
+- scipy=1.6.3=py38h7b17777_0
+- scs=2.1.4=py38h6afa1d1_0
+- setuptools=69.0.3=pyhd8ed1ab_0
+- six=1.16.0=pyh6c4a22f_0
+- snappy=1.1.10=h9fff704_0
+- sqlite=3.45.1=h2c6b66d_0
+- stack_data=0.6.2=pyhd8ed1ab_0
+- suitesparse=5.10.1=h9e50725_1
+- tbb=2021.7.0=h924138e_0
+- tifffile=2022.10.10=pyhd8ed1ab_0
+- tk=8.6.13=noxft_h4845f30_101
+- toolz=0.12.1=pyhd8ed1ab_0
+- tornado=6.3.3=py38h01eb140_1
+- traitlets=5.14.1=pyhd8ed1ab_0
+- typing_extensions=4.9.0=pyha770c72_0
+- unicodedata2=15.1.0=py38h01eb140_0
+- urllib3=2.2.0=pyhd8ed1ab_0
+- wcwidth=0.2.13=pyhd8ed1ab_0
+- wheel=0.42.0=pyhd8ed1ab_0
+- xorg-libxau=1.0.11=hd590300_0
+- xorg-libxdmcp=1.1.3=h7f98852_0
+- xz=5.2.6=h166bdaf_0
+- yaml=0.2.5=h7f98852_2
+- zfp=1.0.1=h59595ed_0
+- zipp=3.17.0=pyhd8ed1ab_0
+- zlib=1.2.13=hd590300_5
+- zlib-ng=2.0.7=h0b41bf4_0
+- zstd=1.5.5=hfc55251_0
 - pip:
-  - ray>=0.8.7
+  - aiosignal==1.3.1
+  - attrs==23.2.0
+  - filelock==3.13.1
+  - frozenlist==1.4.1
+  - jsonschema==4.21.1
+  - jsonschema-specifications==2023.12.1
+  - msgpack==1.0.7
+  - pkgutil-resolve-name==1.3.10
+  - protobuf==4.25.2
+  - ray==2.9.2
+  - referencing==0.33.0
+  - rpds-py==0.17.1
diff --git a/superdsm/__init__.py b/superdsm/__init__.py
@@ -1,3 +1,5 @@
+from . import cpu_setup
+
 from .version import *
 __version__ = VERSION
 

diff --git a/superdsm/batch.py b/superdsm/batch.py
@@ -150,6 +150,13 @@ def _write_performance_report(task_path, performance_path, data, overall_perform
             csv_writer.writerow(row)
 
 
+def _write_env_report(env_path):
+    with open(str(env_path), 'w', newline='') as fout:
+        csv_writer = csv.writer(fout, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL)
+        for env_key, env_val in os.environ.items():
+            csv_writer.writerow([env_key, env_val])
+
+
 DATA_DILL_GZ_FILENAME = 'data.dill.gz'
 
 
@@ -193,6 +200,7 @@ def __init__(self, path, data, parent_task=None):
             self.        result_path = path / DATA_DILL_GZ_FILENAME
             self.       timings_path = path / 'timings.csv'
             self.   performance_path = path / 'performance.csv'
+            self.           env_path = path / 'env.csv'
             self.  timings_json_path = path / '.timings.json'
             self.        digest_path = path / '.digest'
             self.    digest_cfg_path = path / '.digest.cfg.json'
@@ -341,6 +349,7 @@ def run(self, task_info=None, dry=False, verbosity=0, force=False, one_shot=Fals
                     with self.digest_cfg_path.open('w') as fout:
                         self.config.dump_json(fout)
                     _write_performance_report(self.path, self.performance_path, data, performance)
+                    _write_env_report(self.env_path)
                 out2.write(Text.style('Results written to: ', Text.BOLD) + self._fmt_path(self.result_path))
             if not dry and not one_shot: self.digest_path.write_text(self.config_digest)
             for obj_name in ('data', 'shallow_data'):