C++ improvements #447
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compile, Test, and Deploy | |
| on: | |
| pull_request: {} | |
| push: | |
| branches: | |
| - main | |
| release: | |
| types: [published] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint-java: | |
| runs-on: 'ubuntu-latest' | |
| continue-on-error: true | |
| name: Lint Java | |
| defaults: | |
| run: | |
| working-directory: java | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '11' | |
| distribution: 'corretto' | |
| - name: Check Formatting | |
| run: mvn --batch-mode com.spotify.fmt:fmt-maven-plugin:check | |
| lint-cpp: | |
| runs-on: 'ubuntu-latest' | |
| continue-on-error: true | |
| name: Lint C++ | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Check C++ Formatting | |
| uses: jidicula/[email protected] | |
| with: | |
| clang-format-version: 16 | |
| run-java-tests: | |
| continue-on-error: true | |
| name: Test with Java ${{ matrix.java-version }} on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: java | |
| strategy: | |
| matrix: | |
| os: ['ubuntu-latest', windows-latest, macos-12] | |
| java-version: ['11', '17'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'corretto' | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| if: runner.os == 'Windows' | |
| - name: Compile | |
| run: make | |
| - name: Run Tests | |
| run: mvn --batch-mode test | |
| run-java-tests-with-address-sanitizer: | |
| if: false # Disabled due to ASan log spam in output | |
| continue-on-error: true | |
| name: Test with Java ${{ matrix.java-version }} + Address Sanitizer | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: java | |
| strategy: | |
| matrix: | |
| java-version: ['11', '17'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'corretto' | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Compile | |
| env: | |
| USE_ASAN: "1" | |
| run: make target/classes/linux-x64/libvoyager.so | |
| - name: Run Tests | |
| run: | | |
| ASAN_OPTIONS=detect_leaks=0 | |
| LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) \ | |
| mvn --batch-mode test | |
| run-python-tests: | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true | |
| defaults: | |
| run: | |
| working-directory: python | |
| strategy: | |
| matrix: | |
| python-version: | |
| # - '3.7' | |
| - '3.8' | |
| # - '3.9' | |
| # - '3.10' | |
| # - '3.11' | |
| # TODO: Switch back to macos-latest once https://github.com/actions/python-versions/pull/114 is fixed | |
| os: ['ubuntu-latest', windows-latest, macos-12] | |
| name: Test with Python ${{ matrix.python-version }} on ${{ matrix.os }} | |
| steps: | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install Linux dependencies | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y pkg-config | |
| - name: Install test dependencies | |
| env: | |
| # on macOS and with Python 3.10: building NumPy from source fails without these options: | |
| NPY_BLAS_ORDER: "" | |
| NPY_LAPACK_ORDER: "" | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install wheel | |
| pip install -r dev-requirements.txt | |
| - name: Build voyager locally | |
| run: python setup.py develop | |
| - name: Run tox tests | |
| run: tox | |
| run-python-tests-with-address-sanitizer: | |
| defaults: | |
| run: | |
| working-directory: python | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| python-version: | |
| # - '3.7' | |
| - '3.8' | |
| # - '3.9' | |
| # - '3.10' | |
| # - '3.11' | |
| os: ['ubuntu-latest'] | |
| name: Test with Python ${{ matrix.python-version }} + Address Sanitizer | |
| steps: | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install Linux dependencies | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y pkg-config | |
| - name: Install test dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install wheel | |
| pip install -r dev-requirements.txt | |
| - name: Build voyager locally | |
| env: | |
| DEBUG: "0" | |
| USE_ASAN: "1" | |
| CC: clang | |
| CXX: clang++ | |
| run: python setup.py develop | |
| - name: Run tox tests with ASan loaded | |
| # pytest can exit before all Python objects have been destroyed, | |
| # so we tell ASan to ignore leaks. | |
| run: | | |
| ASAN_OPTIONS=detect_leaks=0 \ | |
| LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) \ | |
| tox | |
| run-benchmarks: | |
| strategy: | |
| matrix: | |
| os: ['ubuntu-latest'] | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true | |
| if: github.event_name == 'pull_request' | |
| name: Run benchmarks on ${{ matrix.os }} | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.9' | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Install Linux dependencies | |
| if: runner.os == 'Linux' | |
| run: sudo apt-get update && sudo apt-get install -y pkg-config | |
| - name: Install dev dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install wheel | |
| pip install -r python/dev-requirements.txt | |
| - name: Run benchmarks | |
| run: | | |
| asv machine --yes | |
| asv continuous --sort name --no-only-changed refs/remotes/origin/main ${{ github.sha }} | tee >(sed '1,/All benchmarks:/d' > $GITHUB_STEP_SUMMARY) | |
| run-cpp-tests: | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true | |
| defaults: | |
| run: | |
| working-directory: cpp | |
| strategy: | |
| matrix: | |
| # TODO: Switch back to macos-latest once https://github.com/actions/python-versions/pull/114 is fixed | |
| os: | |
| - 'ubuntu-latest' | |
| # TODO: Fix failing CMake build on windows: | |
| # Error: `cl : command line error D8016: '/O2' and '/RTC1' command-line options are incompatible [D:\a\voyager\voyager\cpp\test\test.vcxproj]` | |
| # I've tried passing CXX flags, but windows doesn't seem to respect disabling runtime checks with /RTC1 or disabling optimizations with /O2 | |
| # - windows-latest | |
| - macos-12 | |
| name: Test C++ on ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install CMake (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' | |
| choco install ninja | |
| - name: Install CMake (MacOS) | |
| if: matrix.os == 'macos-12' | |
| run: brew install cmake | |
| - name: Install CMake (Ubuntu) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: sudo apt-get install -y cmake | |
| - name: Configure CMake | |
| run: cmake . | |
| - name: Build with CMake | |
| run: make | |
| - name: Run Tests (if any) | |
| run: cat Makefile | |
| build-python-wheels: | |
| needs: [run-python-tests, run-python-tests-with-address-sanitizer] | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: true | |
| if: true # (github.event_name == 'release' && github.event.action == 'published') || contains(github.event.pull_request.labels.*.name, 'Also Test Wheels') | |
| strategy: | |
| matrix: | |
| include: | |
| - { os: macos-12, build: cp37-macosx_x86_64 } | |
| - { os: macos-12, build: cp38-macosx_x86_64 } | |
| - { os: macos-12, build: cp39-macosx_x86_64 } | |
| - { os: macos-12, build: cp310-macosx_x86_64 } | |
| - { os: macos-12, build: cp311-macosx_x86_64 } | |
| - { os: macos-12, build: cp312-macosx_x86_64 } | |
| - { os: macos-12, build: cp38-macosx_universal2 } | |
| - { os: macos-12, build: cp39-macosx_universal2 } | |
| - { os: macos-12, build: cp310-macosx_universal2 } | |
| - { os: macos-12, build: cp311-macosx_universal2 } | |
| - { os: macos-12, build: cp312-macosx_universal2 } | |
| - { os: macos-12, build: cp38-macosx_arm64 } | |
| - { os: macos-12, build: cp39-macosx_arm64 } | |
| - { os: macos-12, build: cp310-macosx_arm64 } | |
| - { os: macos-12, build: cp311-macosx_arm64 } | |
| - { os: macos-12, build: cp312-macosx_arm64 } | |
| # - { os: macos-12, build: pp37-macosx_x86_64 } | |
| # - { os: macos-12, build: pp38-macosx_x86_64 } | |
| # - { os: macos-12, build: pp39-macosx_x86_64 } | |
| - { os: windows-latest, build: cp37-win_amd64 } | |
| - { os: windows-latest, build: cp38-win_amd64 } | |
| - { os: windows-latest, build: cp39-win_amd64 } | |
| - { os: windows-latest, build: cp310-win_amd64 } | |
| - { os: windows-latest, build: cp311-win_amd64 } | |
| - { os: windows-latest, build: cp312-win_amd64 } | |
| # - { os: windows-latest, build: pp37-win_amd64 } | |
| # - { os: windows-latest, build: pp38-win_amd64 } | |
| # - { os: windows-latest, build: pp39-win_amd64 } | |
| - { os: 'ubuntu-latest', build: cp37-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp37-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp38-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp38-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp39-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp39-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp310-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp310-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp311-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp311-manylinux_aarch64 } | |
| - { os: 'ubuntu-latest', build: cp312-manylinux_x86_64 } | |
| - { os: 'ubuntu-latest', build: cp312-manylinux_aarch64 } | |
| # - { os: 'ubuntu-latest', build: pp37-manylinux_x86_64 } | |
| # - { os: 'ubuntu-latest', build: pp37-manylinux_aarch64 } | |
| # - { os: 'ubuntu-latest', build: pp38-manylinux_x86_64 } | |
| # - { os: 'ubuntu-latest', build: pp38-manylinux_aarch64 } | |
| # - { os: 'ubuntu-latest', build: pp39-manylinux_x86_64 } | |
| # - { os: 'ubuntu-latest', build: pp39-manylinux_aarch64 } | |
| name: Build wheel for ${{ matrix.build }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # Used to host cibuildwheel, so version doesn't really matter | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Install cibuildwheel | |
| run: python -m pip install 'cibuildwheel>=2.11.0' | |
| - name: Copy README into place | |
| run: cp README.md python/ | |
| - name: Set up QEMU for aarch64 on Linux | |
| if: runner.os == 'Linux' | |
| uses: docker/setup-qemu-action@v1 | |
| with: | |
| platforms: all | |
| - name: Build wheels | |
| run: cp -r cpp python/cpp && cd python && python -m cibuildwheel --output-dir ../wheelhouse | |
| env: | |
| CIBW_TEST_REQUIRES: -r dev-requirements.txt | |
| CIBW_TEST_COMMAND: "pytest {project}/tests -x" | |
| CIBW_BUILD: ${{ matrix.build }} | |
| CIBW_ARCHS: auto64 # Only support building 64-bit wheels. It's 2022! | |
| CIBW_ARCHS_LINUX: auto64 aarch64 # Useful for building linux images with Apple Silicon | |
| CIBW_ARCHS_MACOS: x86_64 universal2 arm64 # Support Apple Silicon | |
| CIBW_ARCHS_WINDOWS: AMD64 | |
| # on macOS and with Python 3.10: building NumPy from source fails without these options: | |
| CIBW_ENVIRONMENT: NPY_BLAS_ORDER="" NPY_LAPACK_ORDER="" | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: pip install auditwheel-symbols && (auditwheel repair -w {dest_dir} {wheel} || auditwheel-symbols --manylinux 2010 {wheel}) | |
| # Use the minimum macOS deployment target that has C++17 support: | |
| CIBW_TEST_SKIP: "*aarch64" | |
| MACOSX_DEPLOYMENT_TARGET: "10.13" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: python-wheels | |
| path: ./wheelhouse/*.whl | |
| build-java-jars: | |
| continue-on-error: false | |
| name: Build JAR with on ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| working-directory: java | |
| strategy: | |
| matrix: | |
| os: ['ubuntu-latest', windows-latest, macos-12] | |
| java-version: ['11'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'corretto' | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Set up QEMU for aarch64 on Linux | |
| if: runner.os == 'Linux' | |
| uses: docker/setup-qemu-action@v1 | |
| with: | |
| platforms: all | |
| - uses: ilammy/msvc-dev-cmd@v1 | |
| if: runner.os == 'Windows' | |
| - name: Compile | |
| run: make | |
| - name: Package | |
| run: mvn --batch-mode package | |
| env: | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: java-${{ matrix.os }} | |
| path: java/target/voyager-*.jar | |
| combine-java-jars: | |
| name: Combine JARs | |
| runs-on: ubuntu-latest | |
| needs: build-java-jars | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Unpack and Repack JARs | |
| shell: bash | |
| working-directory: artifacts | |
| run: | | |
| rm -fv java-*/*-sources.jar | |
| rm -fv java-*/*-javadoc.jar | |
| for artifact in java-*; do unzip -o $artifact/*.jar -d combined-jar; done | |
| # Quick test that the JARs were built for the correct architectures | |
| echo "Checking linux-x64/libvoyager.so for x86 architecture..." | |
| file combined-jar/linux-x64/libvoyager.so | grep x86 | |
| echo "Checking mac-x64/libvoyager.dylib for x86 architecture..." | |
| file combined-jar/mac-x64/libvoyager.dylib | grep x86 | |
| echo "Checking linux-aarch64/libvoyager.so for aarch64 architecture..." | |
| file combined-jar/linux-aarch64/libvoyager.so | grep aarch64 | |
| echo "Checking mac-aarch64/libvoyager.dylib for arm64 architecture..." | |
| file combined-jar/mac-aarch64/libvoyager.dylib | grep arm64 | |
| echo "Checking the Linux JARs to ensure a sufficiently old required GLIBC and GLIBCXX version in each..." | |
| echo "Checking linux-aarch64/libvoyager.so (should require at most GLIBC 2.27...)" | |
| objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBC | grep -v GLIBCXX | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | cat - <(echo 2\.27) | sort -Vu | tail -n 1 | grep 2\.27 || (echo "Expected linux-aarch64 binary to require at most glibc 2.27. Ensure the dockcross container has not been updated." && false) | |
| echo "Checking linux-aarch64/libvoyager.so (should require at most GLIBCXX 3.4.22...)" | |
| objdump -T combined-jar/linux-aarch64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | cat - <(echo 3\.4\.22) | sort -Vu | tail -n 1 | grep 3\.4\.22 || (echo "Expected linux-aarch64 binary to require at most glibc++ 3.4.22. Ensure the dockcross container has not been updated." && false) | |
| echo "Checking linux-x64/libvoyager.so (should require at most GLIBC 2.27...)" | |
| objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBC | grep -v GLIBCXX | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | cat - <(echo 2\.27) | sort -Vu | tail -n 1 | grep 2\.27 || (echo "Expected linux-x64 binary to require at most glibc 2.28. Ensure the dockcross container has not been updated." && false) | |
| echo "Checking linux-x64/libvoyager.so (should require at most GLIBCXX 3.4.22...)" | |
| objdump -T combined-jar/linux-x64/libvoyager.so | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/\1/g' | cat - <(echo 3\.4\.22) | sort -Vu | tail -n 1 | grep 3\.4\.22 || (echo "Expected linux-x64 binary to require at most glibc++ 3.4.22. Ensure the dockcross container has not been updated." && false) | |
| zip -r $(ls java-* | grep jar | tail -n 1) combined-jar/* | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| if-no-files-found: error | |
| name: java-all-platforms | |
| path: artifacts/*.jar | |
| upload-pypi: | |
| defaults: | |
| run: | |
| working-directory: python | |
| needs: [build-python-wheels, run-python-tests-with-address-sanitizer, run-python-tests] | |
| runs-on: 'ubuntu-latest' | |
| name: "Upload wheels to PyPI" | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: python-wheels | |
| path: dist | |
| - uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_DEPLOY_TOKEN }} | |
| upload-maven: | |
| name: Upload to Maven Central | |
| runs-on: ubuntu-latest | |
| needs: combine-java-jars | |
| if: github.event_name == 'release' && github.event.action == 'published' | |
| defaults: | |
| run: | |
| working-directory: java | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: java-all-platforms | |
| path: combined-jar | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 11 | |
| distribution: 'corretto' | |
| server-id: ossrh | |
| server-username: MAVEN_USERNAME | |
| server-password: MAVEN_USER_TOKEN | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_PRIVATE_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| - name: Deploy | |
| # Note: this will re-run the build even though we already have the JAR file ready to deploy | |
| run: | | |
| # Copy the native artifacts built for each platform into the appropriate place: | |
| unzip ../combined-jar/*.jar -d ../ | |
| mkdir -p target/classes/ | |
| cp -r ../combined-jar/linux* target/classes/ | |
| cp -r ../combined-jar/mac* target/classes/ | |
| cp -r ../combined-jar/win* target/classes/ | |
| mvn --batch-mode deploy -Dmaven.test.skip=true | |
| env: | |
| MAVEN_USERNAME: ${{ secrets.SONATYPE_USERNAME }} | |
| MAVEN_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.SONATYPE_PASSPHRASE }} | |
| MAVEN_USER_TOKEN: ${{ secrets.SONATYPE_USER_TOKEN }} |