diff --git a/docs/benchmarks/image_classification/mobilenets.md b/docs/benchmarks/image_classification/mobilenets.md new file mode 100644 index 000000000..1907c2a41 --- /dev/null +++ b/docs/benchmarks/image_classification/mobilenets.md @@ -0,0 +1,59 @@ +# Image Classification using Mobilenet models + +Mobilenet models are not official MLPerf models and so cannot be used for a Closed division MLPerf inference submission. But since they can be run with Imagenet dataset, we are allowed to use them for Open division submission. Only CPU runs are supported now. + +## TFLite Backend + +=== "Mobilenet-V1" + ### Mobilenet V1 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v1 --adr.compiler.tags=gcc + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 --adr.compiler.tags=gcc + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 --adr.compiler.tags=gcc + ``` +=== "Mobilenets" + ### Mobilenet V1,V2,V3 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet --adr.compiler.tags=gcc + ``` +=== "Efficientnet" + ### Efficientnet + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_efficientnet --adr.compiler.tags=gcc + ``` + +## ARMNN Backend +=== "Mobilenet-V1" + ### Mobilenet V1 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v1 --adr.compiler.tags=gcc + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 --adr.compiler.tags=gcc + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 --adr.compiler.tags=gcc + ``` +=== "Mobilenets" + ### Mobilenet V1,V2,V3 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet --adr.compiler.tags=gcc + ``` +=== "Efficientnet" + ### Efficientnet + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_efficientnet --adr.compiler.tags=gcc + ``` + diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 1a77db65a..1d37dcf13 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -66,3 +66,5 @@ Get the Official MLPerf ResNet50 Model ### MLPerf Modular Implementation in C++ {{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }} + +Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification. diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md index 2e9c95c66..32173d49b 100644 --- a/docs/benchmarks/text_to_image/sdxl.md +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -37,13 +37,11 @@ Get the Official MLPerf Stable Diffusion Model === "Intel" ### Intel MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "sdxl", "intel") }} === "Qualcomm" ### Qualcomm AI100 MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "sdxl", "qualcomm") }} diff --git a/docs/submission/index.md b/docs/submission/index.md index b5ff53033..94287f5d2 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -1,4 +1,4 @@ -If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). +If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks/index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). ## Generate actual submission tree diff --git a/main.py b/main.py index a83315334..ccf76b3c1 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): pre_space += " " content="" + scenarios = [] if implementation == "reference": devices = [ "CPU", "CUDA", "ROCm" ] if model.lower() == "resnet50": @@ -23,6 +24,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): devices = [ "CUDA" ] frameworks = [ "TensorRT" ] elif implementation == "intel": + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]: + return pre_space+" WIP" devices = [ "CPU" ] frameworks = [ "Pytorch" ] elif implementation == "qualcomm": @@ -31,6 +34,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): elif implementation == "cpp": devices = [ "CPU", "CUDA" ] frameworks = [ "Onnxruntime" ] + elif implementation == "ctuning-cpp": + scenarios = [ "SingleStream" ] + devices = [ "CPU" ] + if model.lower() == "resnet50": + frameworks = [ "TFLite" ] + else: + frameworks = [] if model.lower() == "bert-99.9": categories = [ "Datacenter" ] @@ -40,11 +50,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): categories = [ "Edge", "Datacenter" ] for category in categories: - if category == "Edge": + if category == "Edge" and not scenarios: scenarios = [ "Offline", "SingleStream" ] if model.lower() in [ "resnet50", "retinanet" ]: - scenarios.append("Multistream") - elif category == "Datacenter": + scenarios.append("MultiStream") + elif category == "Datacenter" and not scenarios: scenarios = [ "Offline", "Server" ] content += f"{pre_space}=== \"{category.lower()}\"\n\n" @@ -69,7 +79,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space2}###### {device} device\n\n" content += f"{cur_space2}###### Docker Setup Command\n\n" - test_query_count=100 + test_query_count=get_test_query_count(model, implementation, device) + content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space2}
\n" @@ -101,6 +112,21 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): return content + def get_test_query_count(model, implementation, device, num_devices=1): + + if model == "resnet50": + p_range = 1000 + elif model in [ "retinanet", "bert-99", "bert-99.9" ]: + p_range = 100 + else: + p_range = 50 + if device == "cuda": + p_range *= 40 + p_range *= num_devices + + return p_range + + @env.macro def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False):