From dcc53db8ef28038e03aeeaa0c398e25b70625ae4 Mon Sep 17 00:00:00 2001 From: Richard Li Date: Thu, 10 Aug 2023 12:09:46 -0700 Subject: [PATCH] Add GitHub Actions config (#6112) Summary: Per Omkar's advice, I am learning from the migration of [this](https://github.com/pytorch/audio/pull/3391/files#diff-78a8a19706dbd2a4425dd72bdab0502ed7a2cef16365ab7030a5a0588927bf47L244-L269) to [this](https://github.com/pytorch/audio/blob/main/.github/workflows/unittest-linux-cpu.yml) Pull Request resolved: https://github.com/pytorch/glow/pull/6112 Test Plan: Documentation: [Optional Fixes #issue] Please see a detailed explanation of how to fill out the fields in the relevant sections in PULL_REQUEST.md. Reviewed By: osalpekar Differential Revision: D48221045 Pulled By: openrichardfb fbshipit-source-id: 48e5e789e146dd2964701e2a2dcb8031513864f3 --- .github/build.sh | 176 +++++++++++++++++++++++++++++++ .github/test.sh | 127 ++++++++++++++++++++++ .github/workflows/glow-build.yml | 55 ++++++++++ 3 files changed, 358 insertions(+) create mode 100644 .github/build.sh create mode 100644 .github/test.sh create mode 100644 .github/workflows/glow-build.yml diff --git a/.github/build.sh b/.github/build.sh new file mode 100644 index 0000000000..786650b627 --- /dev/null +++ b/.github/build.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +# This script setup an working environemnt for running glow tests and gtest driver. +# By default, we run with the enabled CPU backend and disabled OpenCL backend. +set -ex + +# Add support for https apt sources. +wget http://security.ubuntu.com/ubuntu/pool/main/a/apt/apt-transport-https_1.2.32ubuntu0.2_amd64.deb +echo "93475e4cc5e7a86de63fea0316f3f2cd8b791cf4d6ea50a6d63f5bd8e1da5726 apt-transport-https_1.2.32ubuntu0.2_amd64.deb" | sha256sum -c +sudo dpkg -i apt-transport-https_1.2.32ubuntu0.2_amd64.deb +rm apt-transport-https_1.2.32ubuntu0.2_amd64.deb + +export MAX_JOBS=8 +if [ "${CIRCLE_JOB}" != "COVERAGE" ]; then + if hash sccache 2>/dev/null; then + SCCACHE_BIN_DIR="/tmp/sccache" + mkdir -p "$SCCACHE_BIN_DIR" + for compiler in cc c++ gcc g++ x86_64-linux-gnu-gcc; do + ( + echo "#!/bin/sh" + echo "exec $(which sccache) $(which $compiler) \"\$@\"" + ) > "$SCCACHE_BIN_DIR/$compiler" + chmod +x "$SCCACHE_BIN_DIR/$compiler" + done + export PATH="$SCCACHE_BIN_DIR:$PATH" + fi +fi + +install_pocl() { + sudo apt-get install -y ocl-icd-opencl-dev clinfo libhwloc-dev opencl-headers + + git clone https://github.com/pocl/pocl.git + cd pocl && git checkout 4efafa82c087b5e846a9f8083d46b3cdac2f698b && cd ../ + mkdir build_pocl + cd build_pocl + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=/usr/bin/clang++-8 -DCMAKE_C_COMPILER=/usr/bin/clang-8 -DENABLE_ICD=ON ../pocl + make -j`nproc` + sudo make install + + sudo mkdir -p /etc/OpenCL/vendors/ + sudo cp /usr/local/etc/OpenCL/vendors/pocl.icd /etc/OpenCL/vendors/ + + clinfo + cd ../ +} + +install_fmt() { + git clone https://github.com/fmtlib/fmt --branch 7.1.3 + pushd fmt + mkdir build + cd build + cmake .. + make -j`nproc` + sudo make install + popd +} + +upgrade_python() { + echo "Removing old python..."; + sudo apt-get remove --purge -y python3.6 python3-pip libpython3-dev + sudo apt-get autoremove -y + + echo "Installing dependencies for new python..." + sudo apt-get update + sudo apt-get install -y libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev + + echo "Installing new python..." + mkdir python-src + pushd python-src + wget https://www.python.org/ftp/python/3.9.0/Python-3.9.0.tgz + tar xvf Python-3.9.0.tgz + cd Python-3.9.0 + ./configure --enable-shared + sudo make altinstall + popd + + echo "Adjusting system to recognize new python..." + sudo touch /etc/ld.so.conf.d/glowDevLibs.conf + echo "/usr/local/lib/" | sudo tee -a /etc/ld.so.conf.d/glowDevLibs.conf + sudo ldconfig + sudo rm /usr/local/bin/pip + sudo ln -s /usr/local/bin/pip3.9 /usr/local/bin/pip + + echo "Installing virtualenv..." + sudo pip3.9 install virtualenv +} + +GLOW_DEPS="libpng-dev libgoogle-glog-dev libboost-all-dev libdouble-conversion-dev libgflags-dev libjemalloc-dev libpthread-stubs0-dev libevent-dev libssl-dev" + +if [ "${CIRCLE_JOB}" == "CHECK_CLANG_AND_PEP8_FORMAT" ]; then + sudo apt-get update + upgrade_python +else + # Install Glow dependencies + sudo apt-get update + + # Redirect clang + sudo ln -s /usr/bin/clang-8 /usr/bin/clang + sudo ln -s /usr/bin/clang++-8 /usr/bin/clang++ + sudo ln -s /usr/bin/llvm-symbolizer-8 /usr/bin/llvm-symbolizer + sudo ln -s /usr/bin/llvm-config-8 /usr/bin/llvm-config-8.0 + + sudo apt-get install -y ${GLOW_DEPS} + install_fmt +fi + +# Since we are using llvm-7 in these two branches, we cannot use pip install cmake +if [ "${CIRCLE_JOB}" != "PYTORCH" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ]; then + sudo pip install cmake==3.17.3 +else + sudo apt-get install cmake +fi + +# Install ninja, (newest version of) autopep8 through pip +sudo pip install ninja +hash cmake ninja + +# Build glow +GLOW_DIR=$PWD +cd ${GLOW_DIR} +mkdir build && cd build + +CMAKE_ARGS=() + +CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-Werror") +CMAKE_ARGS+=("-DGLOW_WITH_CPU=ON") +CMAKE_ARGS+=("-DGLOW_WITH_HABANA=OFF") + +if [[ "${CIRCLE_JOB}" == "ASAN" ]]; then + CMAKE_ARGS+=("-DGLOW_USE_SANITIZER='Address;Undefined'") + CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=OFF") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") +elif [[ "$CIRCLE_JOB" == "COVERAGE" ]]; then + sudo apt-get install wget + sudo apt-get install -y lcov + sudo pip install awscli --upgrade + ../utils/install_protobuf.sh + CC=gcc-5 CXX=g++-5 cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug -DGLOW_WITH_OPENCL=OFF -DGLOW_WITH_CPU=ON \ + -DLLVM_DIR=/usr/lib/llvm-7/cmake \ + -DGLOW_USE_COVERAGE=ON \ + ../ +elif [[ "$CIRCLE_JOB" == "CHECK_CLANG_AND_PEP8_FORMAT" ]]; then + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-11 main" + sudo apt-get update + sudo apt-get install -y clang-format-11 + cd /tmp + python3.9 -m virtualenv venv + source venv/bin/activate + pip install black==22.3.0 + cd ${GLOW_DIR} +elif [[ "$CIRCLE_JOB" == "OPENCL" ]]; then + install_pocl + CMAKE_ARGS+=("-DGLOW_WITH_OPENCL=ON") +elif [[ "$CIRCLE_JOB" == "FEATURE_COMPILATION" ]]; then + CMAKE_ARGS+=("-DGLOW_USE_PNG_IF_REQUIRED=OFF") +elif [[ "$CIRCLE_JOB" == "32B_DIM_T" ]]; then + install_pocl + CMAKE_ARGS+=("-DTENSOR_DIMS_32_BITS=ON -DGLOW_WITH_OPENCL=ON") +else + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") + if [[ "${CIRCLE_JOB}" == "SHARED" ]]; then + CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=ON") + fi +fi + +if [ "${CIRCLE_JOB}" != "COVERAGE" ] && [ "${CIRCLE_JOB}" != "CHECK_CLANG_AND_PEP8_FORMAT" ] && [ "${CIRCLE_JOB}" != "PYTORCH" ]; then + cmake -GNinja ${CMAKE_ARGS[*]} ../ + ninja +fi + +# Report sccache hit/miss stats +if hash sccache 2>/dev/null; then + sccache --show-stats +fi diff --git a/.github/test.sh b/.github/test.sh new file mode 100644 index 0000000000..df19d028a4 --- /dev/null +++ b/.github/test.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# This script runs all tests in glow, including onnxifi gtests + +set -euxo pipefail + +export GLOW_SRC=$PWD +export GLOW_BUILD_DIR=${GLOW_SRC}/build +export LOADER=${GLOW_BUILD_DIR}/bin/image-classifier +export LSAN_OPTIONS="suppressions=$GLOW_SRC/.circleci/lsan_suppressions.txt" +export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer +export IMAGES_DIR=${GLOW_SRC}/tests/images/ + +# Pass in which tests to run (one of {test, test_unopt}). +run_unit_tests() { + CTEST_PARALLEL_LEVEL=4 GLOG_minloglevel=3 ninja "${1}" || ( cat Testing/Temporary/LastTest.log && exit 1 ) +} + +run_and_check_lenet_mnist_bundle() { + for q in "" "quantized_" + do + cd "${GLOW_BUILD_DIR}/bundles/${q}lenet_mnist/" + rm -f raw_results.txt + for f in ${IMAGES_DIR}/mnist/* + do + # Assume that there is only one file with this format (prepended with Quantized or not) + ./*LeNetMnistBundle ${f} | grep "Result: " >> raw_results.txt + done + diff raw_results.txt "${GLOW_SRC}/.ci/lenet_mnist_expected_output.txt" + cd - + done +} + +run_and_check_resnet50_bundle() { + for q in "" "quantized_" + do + cd "${GLOW_BUILD_DIR}/bundles/${q}resnet50/" + rm -f raw_results.txt + for f in ${IMAGES_DIR}/imagenet/* + do + # Assume that there is only one file with this format (prepended with Quantized or not) + ./*ResNet50Bundle ${f} | grep "Result: " >> raw_results.txt + done + diff raw_results.txt "${GLOW_SRC}/.ci/resnet50_expected_output.txt" + cd - + done +} + +run_and_check_bundle_instrument() { + cd "${GLOW_BUILD_DIR}/bundles/bundle_instrument/" + # Compare console output. + ./BundleInstrument ${IMAGES_DIR}/mnist/0_1009.png >> raw_results.txt + diff raw_results.txt "${GLOW_SRC}/.ci/bundle_instrument_expected_output.txt" + # Compare binary dumps between instrument-debug and instrument-ir. + for file in ./instrument-debug-data/*.bin + do + file_name=$(basename $file) + diff ./instrument-debug-data/${file_name} ./instrument-ir-data/${file_name} + done + cd - +} + +run_and_check_bundle_with_multiple_entries() { + cd "${GLOW_BUILD_DIR}/bundles/bundle_with_multiple_entries/" + # Compare console output. + ./bundle_with_multiple_entries >> raw_results.txt + diff raw_results.txt "${GLOW_SRC}/.ci/bundle_with_multiple_entries_expected_output.txt" + cd - +} + +run_and_check_bundle_with_extra_objects() { + cd "${GLOW_BUILD_DIR}/bundles/bundle_with_extra_objects/" + # Compare console output. + ./BundleWithExtraObjects >> raw_results.txt + diff raw_results.txt "${GLOW_SRC}/.ci/bundle_with_extra_objects_expected_output.txt" + cd - +} + +run_and_check_bundle_tflite_custom() { + cd "${GLOW_BUILD_DIR}/bundles/bundle_tflite_custom/" + # Compare console output. + ./BundleTFLiteCustom >> raw_results.txt + diff raw_results.txt "${GLOW_SRC}/.ci/bundle_tflite_custom_expected_output.txt" + cd - +} + +# Run unit tests and bundle tests. +cd "${GLOW_BUILD_DIR}" +case ${CIRCLE_JOB} in + ASAN) + run_unit_tests check + ;; + OPENCL) + run_unit_tests check + ;; + FEATURE_COMPILATION) + # FEATURE_COMPILATION is a compilation only CI job, thus tests + # are not requited. + ;; + DEBUG) + run_unit_tests check + run_unit_tests test_unopt + ;; + SHARED) + # No tests with shared libs; it's similar to DEBUG. + ;; + 32B_DIM_T) + # A lot of 32b dim_t issues are not revealed at build time, thus + # run the unit test suite also. + run_unit_tests check + ;; + COVERAGE) + cd "${GLOW_SRC}" + cd build + ../.circleci/run_coverage.sh + ;; + CHECK_CLANG_AND_PEP8_FORMAT) + cd "${GLOW_SRC}" + sudo ln -s /usr/bin/clang-format-11 /usr/bin/clang-format + source /tmp/venv/bin/activate + ./utils/format.sh check + ;; + *) + echo "Error, '${CIRCLE_JOB}' not valid mode; Please, check .circleci/test.sh for list of supported tests." + exit 1 + ;; +esac diff --git a/.github/workflows/glow-build.yml b/.github/workflows/glow-build.yml new file mode 100644 index 0000000000..5166e43360 --- /dev/null +++ b/.github/workflows/glow-build.yml @@ -0,0 +1,55 @@ +name: build glow + +on: + pull_request: + push: + branches: + - nightly + - main + - release/* + workflow_dispatch: + +jobs: + tests: + strategy: + matrix: + python_version: ["3.8"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + runner: linux.12xlarge + repository: pytorch/glow + timeout: 120 + script: | + # echo '::group::Setup Environment Variables' + # Mark Build Directory Safe + # git config --global --add safe.directory /__w/audio/audio + + # Set up Environment Variables + export PYTHON_VERSION="${{ matrix.python_version }}" + + # Set CHANNEL + if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then + export UPLOAD_CHANNEL=test + else + export UPLOAD_CHANNEL=nightly + fi + + original_pwd=$(pwd) + sudo apt-get update; + mkdir ~/tempdownload; + cd ~/tempdownload; + wget https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-8.1p1.tar.gz; + tar zxvf openssh-8.1p1.tar.gz; + cd openssh-8.1p1 && ./configure && make && sudo make install; + + cd $original_pwd + git submodule sync + git submodule update --recursive --init + + set -e + # Build + .github/build.sh + + # Test + .github/test.sh