Enable NeuralChat Unit Test process (#195)

.github/workflows/cpp-graph-test.yml

@@ -37,6 +37,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: "recursive"
+          fetch-tags: true
 
       - name: Env build
         run: |

.github/workflows/deploy-test.yml

@@ -65,6 +65,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: "recursive"
+          fetch-tags: true
       # We need this because GitHub needs to clone the branch to pipeline
       - name: Docker Build
         run: |

.github/workflows/llm-test.yml

@@ -41,6 +41,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: "recursive"
+          fetch-tags: true
 
       - name: Env build
         run: |

.github/workflows/optimize-test.yml

@@ -62,6 +62,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: "recursive"
+          fetch-tags: true
     # We need this because GitHub needs to clone the branch to pipeline
       - name: Docker Build
         run: |

.github/workflows/script/SparseLibCI/run_sparse_lib.sh

@@ -1,6 +1,4 @@
 #!/bin/bash
-export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.coveragerc"
-
 output_log_dir="/intel-extension-for-transformers/benchmark_log"
 WORKING_DIR="/intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/deprecated"
 mkdir ${output_log_dir}

.github/workflows/script/formatScan/pylint.sh

@@ -12,7 +12,6 @@ $BOLD_YELLOW && echo "---------------- run python setup.py sdist bdist_wheel ---
 export PYTHONPATH=`pwd`
 #$BOLD_YELLOW && echo "---------------- pip install binary -------------" && $RESET
 #pip install dist/intel_extension_for_transformers*.whl
-
 pip list
 
 cd /intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/

.github/workflows/script/unitTest/coverage/.engine-coveragerc

@@ -0,0 +1,14 @@
+[run]
+branch = True
+
+[report]
+omit =
+ */**/fake*yaml
+ */intel_extension_for_transformers/neural_chat/**
+ */intel_extension_for_transformers/transformers/**
+ */intel_extension_for_transformers/llm/evaluation/**
+ */intel_extension_for_transformers/llm/finetuning/**
+ */intel_extension_for_transformers/llm/inference/**
+ */intel_extension_for_transformers/llm/quantization/**
+exclude_lines =
+ pragma: no cover

.github/workflows/script/unitTest/coverage/.neural-chat-coveragerc

@@ -0,0 +1,24 @@
+[run]
+branch = True
+
+[report]
+omit =
+ */**/fake*yaml
+ */**/fake.py
+ */intel_extension_for_transformers/llm/amp/**
+ */intel_extension_for_transformers/llm/evaluation/**
+ */intel_extension_for_transformers/llm/quantization/**
+ */intel_extension_for_transformers/llm/library/**
+ */intel_extension_for_transformers/llm/operator/**
+ */intel_extension_for_transformers/llm/runtime/**
+ */intel_extension_for_transformers/transformers/**
+exclude_lines =
+ pragma: no cover
+ raise NotImplementedError
+ raise TypeError
+ if self.device == "gpu":
+ if device == "gpu":
+ except ImportError:
+ except Exception as e:
+ onnx_version < ONNX18_VERSION
+ onnx_version >= ONNX18_VERSION

.github/workflows/script/unitTest/coverage/calc_coverage.sh

@@ -87,18 +87,22 @@ function check_coverage_status() {
             $BOLD_RED && echo "Unit Test failed with ${item} coverage decrease ${decrease}%" && $RESET
         done
         $BOLD_RED && echo "compare coverage to give detail info" && $RESET
-        bash -x compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "FAILED" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
+        bash compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "FAILED" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
         exit 1
     else
         $BOLD_GREEN && echo "Unit Test success with coverage lines: ${coverage_PR_lines_rate}%, branches: ${coverage_PR_branches_rate}%" && $RESET
         $BOLD_GREEN && echo "compare coverage to give detail info" && $RESET
-        bash -x compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "SUCCESS" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
+        bash compare_coverage.sh ${coverage_compare} ${coverage_log_pr}/coverage.log ${coverage_log_base}/coverage.log "SUCCESS" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
     fi
 }
 
 function main() {
     compare_coverage
     check_coverage_status
+    $BOLD_BLUE && echo "PR lines coverage: $lines_PR_covered/$lines_PR_valid ($coverage_PR_lines_rate%)" && $RESET
+    $BOLD_BLUE && echo "PR branches coverage: $branches_PR_covered/$branches_PR_valid ($coverage_PR_branches_rate%)" && $RESET
+    $BOLD_BLUE && echo "BASE lines coverage: $lines_base_covered/$lines_base_valid ($coverage_base_lines_rate%)" && $RESET
+    $BOLD_BLUE && echo "BASE branches coverage: $branches_base_covered/$branches_base_valid ($coverage_base_branches_rate%)" && $RESET
 }
 
 main

.github/workflows/script/unitTest/run_unit_test_engine.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
-export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.coveragerc"
+export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.engine-coveragerc"
 LOG_DIR=/log_dir
 mkdir -p ${LOG_DIR}
 WORKING_DIR="/intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/deprecated"
@@ -86,7 +86,13 @@ function gtest() {
 
 function main() {
     bash /intel-extension-for-transformers/.github/workflows/script/unitTest/env_setup.sh
-    echo "test on ${test_name}"
+    cd ${WORKING_DIR}/test/pytest || exit 1
+    if [ -f "requirements.txt" ]; then
+        python -m pip install --default-timeout=100 -r requirements.txt
+        pip list
+    else
+        echo "Not found requirements.txt file."
+    fi
     if [[ $test_name == "PR-test" ]]; then
         pytest "${LOG_DIR}/coverage_pr"
         gtest

.github/workflows/script/unitTest/run_unit_test_neuralchat.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+export COVERAGE_RCFILE="/intel-extension-for-transformers/.github/workflows/script/unitTest/coverage/.neural-chat-coveragerc"
+LOG_DIR=/log_dir
+mkdir -p ${LOG_DIR}
+WORKING_DIR="/intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/tests"
+# get parameters
+PATTERN='[-a-zA-Z0-9_]*='
+PERF_STABLE_CHECK=true
+
+for i in "$@"; do
+    case $i in
+        --test_name=*)
+            test_name=`echo $i | sed "s/${PATTERN}//"`;;
+        *)
+            echo "Parameter $i not recognized."; exit 1;;
+    esac
+done
+
+function pytest() {
+    local coverage_log_dir=$1
+    mkdir -p ${coverage_log_dir}
+
+    cd ${WORKING_DIR} || exit 1
+    JOB_NAME=unit_test
+    ut_log_name=${LOG_DIR}/${JOB_NAME}.log
+    export GLOG_minloglevel=2
+
+    itrex_path=$(python -c 'import intel_extension_for_transformers; import os; print(os.path.dirname(intel_extension_for_transformers.__file__))')
+    find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${itrex_path}"' --append ,g' | sed 's/$/ --verbose/' >run.sh
+    coverage erase
+
+    # run UT
+    $BOLD_YELLOW && echo "cat run.sh..." && $RESET
+    cat run.sh | tee ${ut_log_name}
+    $BOLD_YELLOW && echo "------UT start-------" && $RESET
+    bash run.sh 2>&1 | tee -a ${ut_log_name}
+    $BOLD_YELLOW && echo "------UT end -------" && $RESET
+
+    # run coverage report
+    coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log_dir}/coverage.log
+    coverage html -d ${coverage_log_dir}/htmlcov --rcfile=${COVERAGE_RCFILE}
+    coverage xml -o ${coverage_log_dir}/coverage.xml --rcfile=${COVERAGE_RCFILE}
+
+    # check UT status
+    if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ]; then
+        $BOLD_RED && echo "Find errors in UT test, please check the output..." && $RESET
+        exit 1
+    fi
+    $BOLD_GREEN && echo "UT finished successfully! " && $RESET
+}
+
+function main() {
+    bash /intel-extension-for-transformers/.github/workflows/script/unitTest/env_setup.sh
+    wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
+    dpkg -i libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb
+    cd ${WORKING_DIR} || exit 1
+    if [ -f "requirements.txt" ]; then
+        python -m pip install --default-timeout=100 -r requirements.txt
+        pip list
+    else
+        echo "Not found requirements.txt file."
+    fi
+    echo "test on ${test_name}"
+    if [[ $test_name == "PR-test" ]]; then
+        pytest "${LOG_DIR}/coverage_pr"
+    elif [[ $test_name == "baseline" ]]; then
+        pytest "${LOG_DIR}/coverage_base"  
+    fi
+}
+main

.github/workflows/unit-test-engine.yml

@@ -56,6 +56,7 @@ jobs:
         with:
           submodules: "recursive"
           ref: ${{ matrix.test_branch }}
+          fetch-tags: true
 
       - name: Docker Build
         run: |

.github/workflows/unit-test-kernel.yml

@@ -47,6 +47,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: "recursive"
+          fetch-tags: true
 
       - name: Docker Build
         run: |

.github/workflows/unit-test-neuralchat.yml

@@ -0,0 +1,141 @@
+name: NeuralChat Unit Test
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - intel_extension_for_transformers/neural_chat/tests/**
+      - .github/workflows/unit-test-neuralchat.yml
+      - .github/workflows/script/unitTest/**
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DOCKER_CONFIG_NAME: "commonDockerConfig"
+  REPO_NAME: "intel-extension-for-transformers"
+  REPO_TAG: "py38"
+  DOCKER_FILE_NAME: "devel"
+  CONTAINER_NAME: "utTest"
+  EXTRA_CONTAINER_NAME: "modelTest"
+
+jobs:
+  unit-test:
+    runs-on: [self-hosted, Linux, X64, itrex-node]
+    strategy:
+      matrix:
+        include:
+          - test_branch: ${{ github.ref }}
+            test_name: "PR-test"
+          - test_branch: "main"
+            test_name: "baseline"
+    steps:
+      - name: Docker Clean Up
+        run: |
+          docker ps -a
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+              docker start ${{ env.CONTAINER_NAME }}
+              echo "remove left files through container ..."
+              docker exec ${{ env.CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+          if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}'$) ]]; then
+              docker start ${{ env.EXTRA_CONTAINER_NAME }}
+              echo "remove left files through container ..."
+              docker exec ${{ env.EXTRA_CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          ref: ${{ matrix.test_branch }}
+          fetch-tags: true
+
+      - name: Docker Build
+        run: |
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .
+
+      - name: Docker Run
+        run: |
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+            docker stop ${{ env.CONTAINER_NAME }}
+            docker rm -vf ${{ env.CONTAINER_NAME }} || true
+          fi
+          docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
+          -v ${{ github.workspace }}:/intel-extension-for-transformers \
+          ${{ env.REPO_NAME }}:${{ env.REPO_TAG }}
+
+      - name: Env build
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
+
+      - name: Binary build
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers/.github/workflows/script \
+          && bash install_binary.sh \
+          && pip install intel_extension_for_pytorch wget sentencepiece \
+          && pip install transformers>=4.32.0 \
+          && pip install peft torch num2words speechbrain paddlepaddle \
+          && pip install paddlespeech==1.4.1 shortuuid gptcache evaluate \
+          && pip install fschat pydub python-multipart PyPDF2  langchain \
+          && pip install python-docx scikit-learn farm-haystack librosa beautifulsoup4 \
+          && pip install InstructorEmbedding chromadb pydantic fastapi starlette \
+          && pip install yacs uvicorn optimum optimum[habana] \
+          && pip install sentence_transformers unstructured markdown rouge_score \
+          && pip install --upgrade accelerate"
+
+      - name: Run UT
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/unitTest \
+          && bash run_unit_test_neuralchat.sh --test_name=${{ matrix.test_name }}"
+
+      - name: Collect log
+        if: ${{ !cancelled() }}
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers && \
+                 mv /log_dir . "
+      
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: "UnitTest${{ matrix.test_name }}"
+          path: ${{ github.workspace }}/log_dir
+
+
+  Genreate-Report:
+    runs-on: itrex-node-spell
+    needs: [unit-test]
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Download UT PR Log
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ github.workspace }}/log_dir
+
+      - name: Display structure of downloaded files
+        run: cd ${{ github.workspace }}/log_dir && ls -R
+
+      - name: Calculte coverage
+        run: |
+          cd ${{ github.workspace }}/.github/workflows/script/unitTest/coverage
+          /usr/bin/bash calc_coverage.sh ${{ github.workspace }}/log_dir
+
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: Neural Chat Unit Test
+          path: ${{ github.workspace }}/log_dir
+          retention-days: 5

.github/workflows/unit-test-optimize.yml

@@ -56,6 +56,7 @@ jobs:
         with:
           submodules: "recursive"
           ref: ${{ matrix.test_branch }}
+          fetch-tags: true
 
       - name: Docker Build
         run: |
@@ -131,7 +132,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: ${{ !cancelled() }}
         with:
-          name: Engine Unit Test
+          name: Optimize Unit Test
           path: ${{ github.workspace }}/log_dir
           retention-days: 5
 

intel_extension_for_transformers/__init__.py

@@ -18,4 +18,4 @@
 try:
     from ._version import __version__  # load _version file generated by setuptools_scm 
 except ModuleNotFoundError:
-    __version__ = "0.0.0"
+    __version__ = "1.1"