[feature][refactor] Benchmark Reporting + Hub Mixin (#122)

.github/workflows/check_quality.yaml

@@ -18,10 +18,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install quality requirements
         run: |

.github/workflows/test_api_cpu.yaml

@@ -18,10 +18,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install dependencies
         run: |

.github/workflows/test_api_misc.yaml

@@ -18,10 +18,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install requirements
         run: |

.github/workflows/test_cpu_neural_compressor.yaml → .github/workflows/test_cli_cpu_neural_compressor.yaml

@@ -1,4 +1,4 @@
-name: CPU Intel Neural Compressor Tests
+name: CLI CPU Intel Neural Compressor Tests
 
 on:
   workflow_dispatch:
@@ -12,16 +12,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_neural_compressor_tests:
+  run_cli_cpu_neural_compressor_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install Intel Neural Compressor CPU requirements
         run: |

.github/workflows/test_cpu_onnxruntime.yaml → .github/workflows/test_cli_cpu_onnxruntime.yaml

@@ -1,4 +1,4 @@
-name: CPU OnnxRuntime Tests
+name: CLI CPU OnnxRuntime Tests
 
 on:
   workflow_dispatch:
@@ -12,16 +12,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_onnxruntime_tests:
+  run_cli_cpu_onnxruntime_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install requirements
         run: |

.github/workflows/test_cpu_openvino.yaml → .github/workflows/test_cli_cpu_openvino.yaml

@@ -1,4 +1,4 @@
-name: CPU OpenVINO Tests
+name: CLI CPU OpenVINO Tests
 
 on:
   workflow_dispatch:
@@ -12,16 +12,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_openvino_tests:
+  run_cli_cpu_openvino_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install requirements
         run: |

.github/workflows/test_cpu_pytorch.yaml → .github/workflows/test_cli_cpu_pytorch.yaml

@@ -1,4 +1,4 @@
-name: CPU Pytorch tests
+name: CLI CPU Pytorch tests
 
 on:
   workflow_dispatch:
@@ -12,16 +12,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run_cpu_pytorch_tests:
+  run_cli_cpu_pytorch_tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install requirements
         run: |

.github/workflows/test_cuda_onnxruntime_inference.yaml → .github/workflows/test_cli_cuda_onnxruntime.yaml

@@ -1,4 +1,4 @@
-name: CUDA OnnxRuntime Inference Tests
+name: CLI CUDA OnnxRuntime Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_cuda_onnxruntime_inference_tests:
+  build_image_and_run_cli_cuda_onnxruntime_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout
@@ -40,4 +40,4 @@ jobs:
           --workdir /workspace/optimum-benchmark
           --gpus '"device=0,1"'
           opt-bench-cuda:11.8.0
-          -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and cuda and onnxruntime and inference' -x"
+          -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and cuda and onnxruntime' -x"

.github/workflows/test_cuda_pytorch.yaml → .github/workflows/test_cli_cuda_pytorch.yaml

@@ -1,4 +1,4 @@
-name: CUDA Pytorch Tests
+name: CLI CUDA Pytorch Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_cuda_pytorch_tests:
+  build_image_and_run_cli_cuda_pytorch_tests:
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test_cuda_torch_ort_training.yaml → .github/workflows/test_cli_cuda_torch_ort.yaml

@@ -1,4 +1,4 @@
-name: CUDA Torch-ORT Training Tests
+name: CLI CUDA Torch-ORT Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_cuda_torch_ort_training_tests:
+  build_image_and_run_cli_cuda_torch_ort_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout
@@ -40,4 +40,4 @@ jobs:
           --workdir /workspace/optimum-benchmark
           --gpus '"device=0,1"'
           opt-bench-cuda:11.8.0
-          -c "pip install -e .[testing,torch-ort,peft] && python -m torch_ort.configure && pytest -k 'cli and cuda and torch_ort and training' -x"
+          -c "pip install -e .[testing,torch-ort,peft] && python -m torch_ort.configure && pytest -k 'cli and cuda and torch_ort' -x"

.github/workflows/test_cli_misc.yaml

@@ -18,10 +18,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install requirements
         run: |

.github/workflows/test_rocm_onnxruntime_inference.yaml → .github/workflows/test_cli_rocm_onnxruntime.yaml

@@ -1,4 +1,4 @@
-name: ROCm OnnxRuntime Inference Tests
+name: CLI ROCm OnnxRuntime Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_rocm_onnxruntime_inference_tests:
+  build_image_and_run_cli_rocm_onnxruntime_tests:
     runs-on: hf-amd-mi210-dev
     steps:
       - name: Checkout
@@ -51,4 +51,4 @@ jobs:
           --device /dev/dri/renderD129
           --entrypoint /bin/bash
           opt-bench-rocm-ort:5.7
-          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'cli and rocm and onnxruntime and inference' -x"
+          -c "pip install -e .[testing,timm,diffusers] && pytest -k 'cli and rocm and onnxruntime' -x"

.github/workflows/test_rocm_pytorch.yaml → .github/workflows/test_cli_rocm_pytorch.yaml

@@ -1,4 +1,4 @@
-name: ROCm Pytorch Tests
+name: CLI ROCm Pytorch Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_rocm_pytorch_tests:
+  build_image_and_run_cli_rocm_pytorch_tests:
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test_tensorrt_llm.yaml → .github/workflows/test_cli_tensorrt_llm.yaml

@@ -1,4 +1,4 @@
-name: TensorRT-LLM Tests
+name: CLI TensorRT-LLM Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  pull_image_and_run_tensorrt_llm_tests:
+  pull_image_and_run_cli_tensorrt_llm_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout

.github/workflows/test_tensorrt_onnxruntime_inference.yaml → .github/workflows/test_cli_tensorrt_onnxruntime.yaml

@@ -1,4 +1,4 @@
-name: TensorRT OnnxRuntime Inference Tests
+name: CLI TensorRT OnnxRuntime Tests
 
 on:
   workflow_dispatch:
@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_image_and_run_tensorrt_onnxruntime_tests:
+  build_image_and_run_cli_tensorrt_onnxruntime_tests:
     runs-on: hf-dgx-01
     steps:
       - name: Checkout
@@ -40,4 +40,4 @@ jobs:
           --gpus '"device=0,1"'
           --entrypoint /bin/bash
           opt-bench-tensorrt:22.12
-          -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and tensorrt and onnxruntime and inference' -x"
+          -c "pip install -e .[testing,onnxruntime-gpu,diffusers,timm] && pytest -k 'cli and tensorrt and onnxruntime' -x"

Makefile

@@ -12,8 +12,68 @@ style:
 install:
 	pip install -e .
 
-install_cpu_dev:
-	pip install -e .[quality,testing,openvino,onnxruntime,neural-compressor,diffusers,timm,peft]
+build_docker_cpu:
+	docker build -f docker/cuda.dockerfile  --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -t opt-bench-cpu:latest .
 
-install_gpu_dev:
-	pip install -e .[quality,testing,onnxruntime-gpu,deepspeed,diffusers,timm,peft]
+build_docker_cuda:
+	docker build -f docker/cuda.dockerfile  --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) --build-arg TORCH_CUDA=cu118 --build-arg CUDA_VERSION=11.8.0 -t opt-bench-cuda:11.8.0 . 
+
+build_docker_rocm:
+	docker build -f docker/rocm.dockerfile  --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) --build-arg TORCH_ROCM=rocm5.6 --build-arg ROCM_VERSION=5.6.1 -t opt-bench-rocm:5.6.1 . 
+
+test_cli_cpu_neural_compressor:
+	docker run \
+	--rm \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cpu:latest -c "pip install -e .[testing,neural-compressor] && pytest tests/ -k 'cli and cpu and neural_compressor' -x"
+
+test_cli_cpu_openvino:
+	docker run \
+	--rm \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cpu:latest -c "pip install -e .[testing,openvino,diffusers] && pytest tests/ -k 'cli and cpu and openvino' -x"
+
+test_cli_cpu_onnxruntime:
+	docker run \
+	--rm \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cpu:latest -c "pip install -e .[testing,onnxruntime,diffusers,timm] && pytest tests/ -k 'cli and cpu and onnxruntime' -x"
+
+test_cli_cpu_pytorch:
+	docker run \
+	--rm \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cpu:latest -c "pip install -e .[testing,diffusers,timm] && pytest tests/ -k 'cli and cpu and pytorch' -x"
+
+test_api_cpu:
+	docker run \
+	--rm \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cpu:latest -c "pip install -e .[testing,timm,diffusers,codecarbon] && pytest tests/ -k 'api and cpu' -x"
+
+test_api_cuda:
+	docker run \
+	--rm \
+	--gpus '"device=0,1"' \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cuda:11.8.0 -c "pip install -e .[testing,timm,diffusers,codecarbon] && pytest tests/ -k 'api and cuda' -x"
+
+test_api_misc:
+	docker run \
+	--rm \
+	--entrypoint /bin/bash \
+	--volume $(PWD):/workspace \
+	--workdir /workspace \
+	opt-bench-cpu:latest -c "pip install -e .[testing,timm,diffusers,codecarbon] && pytest tests/ -k 'api and not (cpu or cuda or rocm or tensorrt)' -x"