Migrate pipeline linux test to docker image

.azure_pipelines/dockerfiles/linux-cpu.dockerfile

@@ -0,0 +1,21 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+FROM ubuntu:22.04
+
+ARG PYTHON_VERSION
+
+RUN apt-get update && \
+    apt-get install -y \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python${PYTHON_VERSION}-venv \
+    python3-pip \
+    unzip \
+    docker.io
+RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python
+
+COPY . /olive
+WORKDIR /olive
+RUN pip install -e .

.azure_pipelines/dockerfiles/linux-gpu.dockerfile

@@ -0,0 +1,41 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+
+ARG PYTHON_VERSION
+ARG TENSORRT_VERSION=10.0.1.6-1+cuda12.4
+
+RUN apt-get update && \
+    apt-get install -y \
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
+    python${PYTHON_VERSION}-venv \
+    python3-pip \
+    libnvinfer10=${TENSORRT_VERSION} \
+    libnvinfer-dev=${TENSORRT_VERSION} \
+    libnvinfer-plugin-dev=${TENSORRT_VERSION} \
+    libnvinfer-vc-plugin-dev=${TENSORRT_VERSION} \
+    libnvinfer-headers-plugin-dev=${TENSORRT_VERSION} \
+    libnvonnxparsers-dev=${TENSORRT_VERSION} \
+    libnvinfer-plugin10=${TENSORRT_VERSION} \
+    libnvinfer-vc-plugin10=${TENSORRT_VERSION} \
+    libnvonnxparsers10=${TENSORRT_VERSION} \
+    libnvinfer-headers-dev=${TENSORRT_VERSION} \
+    libnvinfer-lean10=${TENSORRT_VERSION} \
+    python3-libnvinfer-lean=${TENSORRT_VERSION} \
+    libnvinfer-dispatch10=${TENSORRT_VERSION} \
+    python3-libnvinfer-dispatch=${TENSORRT_VERSION} \
+    tensorrt-libs=${TENSORRT_VERSION} \
+    tensorrt-dev=${TENSORRT_VERSION} \
+    libnvinfer-lean-dev=${TENSORRT_VERSION} \
+    libnvinfer-dispatch-dev=${TENSORRT_VERSION} \
+    python3-libnvinfer=${TENSORRT_VERSION} \
+    unzip \
+    docker.io
+RUN ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python
+
+COPY . /olive
+WORKDIR /olive
+RUN pip install -e .

.azure_pipelines/job_templates/build-docker-image-template.yaml

@@ -0,0 +1,19 @@
+# Docker image build template
+
+parameters:
+  dockerfile: ''
+  python_version: ''
+  docker_image: ''
+
+steps:
+- script: |
+    docker login -u $(docker-username) -p $(docker-password)
+    docker build --build-arg PYTHON_VERSION=${{ parameters.python_version }} -t ${{ parameters.docker_image }} -f $(Build.SourcesDirectory)/${{ parameters.dockerfile }} .
+  displayName: Build Docker Image
+
+- script: |
+    docker version
+    docker image ls
+    docker system df
+    df -h
+  displayName: Check Docker Images

.azure_pipelines/job_templates/olive-example-linux-template.yaml

@@ -0,0 +1,78 @@
+# Olive Build and Test Pipeline template for examples on Azure DevOps
+
+parameters:
+  name: ''
+  pool: ''
+  python_version: '3.10'
+  device: 'cpu'
+  dockerfile: '.azure_pipelines/dockerfiles/linux-cpu.dockerfile'
+  docker_image: 'olive-pipeline:latest'
+  onnxruntime: 'onnxruntime'
+  subfolder: 'local'
+  torch: 'torch'
+  test_script: 'run_test.sh'
+  onnxruntime_nightly: false
+
+jobs:
+  - job: ${{ parameters.name }}_Test_Examples
+    timeoutInMinutes: 300
+    pool:
+      name: ${{ parameters.pool }}
+    strategy:
+      matrix:
+        ${{ insert }}: ${{ parameters.examples }}
+    variables:
+      PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
+      HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
+      OLIVE_TEMPDIR: $(Pipeline.Workspace)/.olive_tempdir
+
+    steps:
+    - template: build-docker-image-template.yaml
+      parameters:
+        python_version: ${{ parameters.python_version }}
+        dockerfile: ${{ parameters.dockerfile }}
+        docker_image: ${{ parameters.docker_image }}
+
+    # set exampleRequirements to requirements.txt if user does not specify
+    - script:
+        echo "##vso[task.setvariable variable=exampleRequirements]requirements.txt"
+      displayName: Set exampleRequirements
+      condition: eq(variables['exampleRequirements'], '')
+
+    - script: |
+        GPU_OPTION=""
+        if [ "${{ parameters.device }}" = "gpu" ]; then
+            GPU_OPTION="--gpus=all"
+        fi
+        docker run \
+        $GPU_OPTION \
+        -v $(Build.SourcesDirectory)/logs:/logs \
+        -e WORKSPACE_SUBSCRIPTION_ID=$(workspace-subscription-id) \
+        -e WORKSPACE_RESOURCE_GROUP=$(workspace-resource-group) \
+        -e WORKSPACE_NAME=$(workspace-name) \
+        -e MANAGED_IDENTITY_CLIENT_ID=$(olive-1es-identity-client-id) \
+        -e PIPELINE_TEST_ACCOUNT_NAME=$(pipeline-test-account-name) \
+        -e PIPELINE_TEST_CONTAINER_NAME=$(pipeline-test-container-name) \
+        -e KEYVAULT_NAME=$(keyvault-name) \
+        -e HF_TOKEN=$(hf_token) \
+        ${{ parameters.docker_image }} \
+        bash .azure_pipelines/scripts/${{ parameters.test_script }} \
+        ${{ parameters.torch }} \
+        ${{ parameters.onnxruntime }} \
+        ${{ parameters.onnxruntime_nightly }} \
+        examples/$(exampleFolder)/$(exampleRequirements) \
+        examples/test/${{ parameters.subfolder }}/test_$(exampleName).py
+      displayName: Run Tests in Docker
+
+    # Step 3: Publish test results
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFiles: '**/logs/test_examples-TestOlive.xml'
+        testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
+        failTaskOnFailedTests: true
+      displayName: Publish Test Results
+
+    - script: sudo git clean -dfX
+      condition: always()
+      displayName: Clean remaining artifacts

.azure_pipelines/job_templates/olive-example-template.yaml → .azure_pipelines/job_templates/olive-example-win-template.yaml

@@ -3,7 +3,9 @@
 parameters:
   name: ''
   pool: ''
-  python_version: '3.8'
+  test_type: ''
+  device: 'cpu'
+  python_version: '3.10'
   onnxruntime: 'onnxruntime'
   subfolder: 'local'
   torch: 'torch'

.azure_pipelines/job_templates/olive-setup-template.yaml

@@ -1,5 +1,5 @@
 parameters:
-  python_version: '3.8'
+  python_version: '3.10'
   onnxruntime: 'onnxruntime'
   torch: torch
 

.azure_pipelines/job_templates/olive-test-template.yaml → .azure_pipelines/job_templates/olive-test-cpu-template.yaml

@@ -3,8 +3,7 @@ parameters:
   pool: ''
   test_type: ''
   windows: False
-  device: 'cpu'
-  python_version: '3.8'
+  python_version: '3.10'
   onnxruntime: 'onnxruntime'
   torch: 'torch'
   requirements_file: 'requirements-test.txt'

.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml

@@ -0,0 +1,81 @@
+# Example Linux test template for Olive pipeline
+
+parameters:
+  name: ''
+  pool: ''
+  test_type: ''
+  device: 'cpu'
+  dockerfile: '.azure_pipelines/dockerfiles/linux-gpu.dockerfile'
+  docker_image: 'olive-pipeline:latest'
+  python_version: '3.10'
+  onnxruntime: 'onnxruntime'
+  torch: 'torch'
+  requirements_file: 'requirements-test.txt'
+  test_script: 'run_test.sh'
+  onnxruntime_nightly: false
+
+jobs:
+- job: ${{parameters.name}}
+  timeoutInMinutes: 300
+  pool:
+    name: ${{ parameters.pool}}
+  variables:
+    testType: ${{ parameters.test_type }}
+    python_version: ${{ parameters.python_version }}
+    requirements_file: ${{ parameters.requirements_file }}
+    PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
+    HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
+
+  steps:
+  - template: build-docker-image-template.yaml
+    parameters:
+      python_version: ${{ parameters.python_version }}
+      dockerfile: ${{ parameters.dockerfile }}
+      docker_image: ${{ parameters.docker_image }}
+
+  - script: |
+      docker run \
+      --gpus=all \
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      -v $(Build.SourcesDirectory)/logs:/logs \
+      -e WORKSPACE_SUBSCRIPTION_ID=$(workspace-subscription-id) \
+      -e WORKSPACE_RESOURCE_GROUP=$(workspace-resource-group) \
+      -e WORKSPACE_NAME=$(workspace-name) \
+      -e MANAGED_IDENTITY_CLIENT_ID=$(olive-1es-identity-client-id) \
+      ${{ parameters.docker_image }} \
+      bash .azure_pipelines/scripts/${{ parameters.test_script }} \
+      ${{ parameters.torch }} \
+      ${{ parameters.onnxruntime }} \
+      ${{ parameters.onnxruntime_nightly }} \
+      test/$(requirements_file) \
+      test/$(testType)
+    displayName: Run Tests in Docker
+
+  - task: CredScan@3
+    displayName: 'Run CredScan'
+    inputs:
+      debugMode: false
+    continueOnError: true
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/*TestOlive*.xml'
+      testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
+      failTaskOnFailedTests: true
+    displayName: Upload pipeline run test results
+
+  # Code coverage requires
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk 7.0.x'
+    inputs:
+      version: 7.0.x
+
+  - task: PublishCodeCoverageResults@2
+    inputs:
+      summaryFileLocation: '**/coverage.xml'
+    displayName: Publish code coverage results
+
+  - script: sudo git clean -dfX
+    condition: always()
+    displayName: Clean remaining artifacts

.azure_pipelines/olive-aml-ci.yaml

@@ -15,7 +15,7 @@ pr: none
 
 jobs:
 # Linux examples test
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-linux-template.yaml
   parameters:
     name: Linux_CI
     pool: $(OLIVE_POOL_UBUNTU2004)
@@ -37,7 +37,7 @@ jobs:
         exampleRequirements: requirements-pipeline.txt
 
 # Windows examples test
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-win-template.yaml
   parameters:
     name: Windows_CI
     pool: $(OLIVE_POOL_WIN2019)

.azure_pipelines/olive-ci.yaml

@@ -51,31 +51,34 @@ variables:
 
 jobs:
 # Linux unit tests
-- template: job_templates/olive-test-template.yaml
+- template: job_templates/olive-test-cpu-template.yaml
   parameters:
     name: Linux_CPU_CI_Unit_Test
     pool: $(OLIVE_POOL_UBUNTU2004)
+    onnxruntime: onnxruntime==1.19.2
     test_type: 'unit_test'
 
-- template: job_templates/olive-test-template.yaml
+- template: job_templates/olive-test-linux-gpu-template.yaml
   parameters:
     name: Linux_GPU_CI_Unit_Test
     pool: $(OLIVE_POOL_UBUNTU2004_GPU_V100)
     test_type: 'unit_test'
     device: 'gpu'
-    onnxruntime: onnxruntime-gpu
+    dockerfile: '.azure_pipelines/dockerfiles/linux-gpu.dockerfile'
+    onnxruntime: onnxruntime-gpu==1.19.2
     requirements_file: 'requirements-test-gpu.txt'
 
 # Windows unit tests
-- template: job_templates/olive-test-template.yaml
+- template: job_templates/olive-test-cpu-template.yaml
   parameters:
     name: Windows_CPU_CI_Unit_Test
     pool: $(OLIVE_POOL_WIN2019)
+    onnxruntime: onnxruntime==1.19.2
     test_type: 'unit_test'
     windows: True
 
 # Linux examples test
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-linux-template.yaml
   parameters:
     name: Linux_CI
     pool: $(OLIVE_POOL_UBUNTU2004)
@@ -95,7 +98,7 @@ jobs:
         exampleName: mobilenet_qnn_ep
 
 # Windows examples test
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-win-template.yaml
   parameters:
     name: Windows_CI
     pool: $(OLIVE_POOL_WIN2019)
@@ -115,11 +118,13 @@ jobs:
         exampleName: mobilenet_qnn_ep
 
 # Linux GPU examples testing.
-- template: job_templates/olive-example-template.yaml
+- template: job_templates/olive-example-linux-template.yaml
   parameters:
     name: Linux_GPU_CI
     pool: $(OLIVE_POOL_UBUNTU2004_GPU_V100)
+    device: 'gpu'
     onnxruntime: onnxruntime-gpu
+    dockerfile: '.azure_pipelines/dockerfiles/linux-gpu.dockerfile'
     examples:
       bert_cuda_gpu:
         exampleFolder: bert
@@ -128,29 +133,25 @@ jobs:
         exampleFolder: stable_diffusion
         exampleName: stable_diffusion_cuda_gpu
         exampleRequirements: requirements-common.txt
-      llama2:
-        exampleFolder: llama2
-        exampleName: llama2
-        exampleRequirements: requirements-pipeline.txt
 
 # these jobs need secrets not available in forks
 - ${{ if ne(variables['System.PullRequest.IsFork'], 'True') }}:
   # integration tests
-  - template: job_templates/olive-test-template.yaml
+  - template: job_templates/olive-test-cpu-template.yaml
     parameters:
       name: Linux_CPU_CI_Integration_Test
       pool: $(OLIVE_POOL_UBUNTU2004)
       test_type: 'integ_test'
 
-  - template: job_templates/olive-test-template.yaml
+  - template: job_templates/olive-test-cpu-template.yaml
     parameters:
       name: Windows_CPU_CI_Integration_Test
       pool: $(OLIVE_POOL_WIN2019)
       test_type: 'integ_test'
       windows: True
 
   # Multiple EP Linux testing
-  - template: job_templates/olive-test-template.yaml
+  - template: job_templates/olive-test-cpu-template.yaml
     parameters:
       name: Linux_CI_Multiple_EP_Test
       pool: $(OLIVE_POOL_UBUNTU2004)