Update the docs

docs/benchmarks/image_classification/mobilenets.md

@@ -0,0 +1,59 @@
+# Image Classification using Mobilenet models
+
+Mobilenet models are not official MLPerf models and so cannot be used for a Closed division MLPerf inference submission. But since they can be run with Imagenet dataset, we are allowed to use them for Open division submission. Only CPU runs are supported now. 
+
+## TFLite Backend
+
+=== "Mobilenet-V1"
+    ### Mobilenet V1
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v1 --adr.compiler.tags=gcc
+    ```
+=== "Mobilenet-V2"
+    ### Mobilenet V2
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 --adr.compiler.tags=gcc
+    ```
+=== "Mobilenet-V2"
+    ### Mobilenet V2
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 --adr.compiler.tags=gcc
+    ```
+=== "Mobilenets"
+    ### Mobilenet V1,V2,V3
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_mobilenet --adr.compiler.tags=gcc
+    ```
+=== "Efficientnet"
+    ### Efficientnet
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_efficientnet --adr.compiler.tags=gcc
+    ```
+
+## ARMNN Backend
+=== "Mobilenet-V1"
+    ### Mobilenet V1
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v1 --adr.compiler.tags=gcc
+    ```
+=== "Mobilenet-V2"
+    ### Mobilenet V2
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 --adr.compiler.tags=gcc
+    ```
+=== "Mobilenet-V2"
+    ### Mobilenet V2
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 --adr.compiler.tags=gcc
+    ```
+=== "Mobilenets"
+    ### Mobilenet V1,V2,V3
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet --adr.compiler.tags=gcc
+    ```
+=== "Efficientnet"
+    ### Efficientnet
+    ```bash
+       cm run script --tags=run,mobilenet-models,_tflite,_armnn,_efficientnet --adr.compiler.tags=gcc
+    ```
+

docs/benchmarks/image_classification/resnet50.md

@@ -66,3 +66,5 @@ Get the Official MLPerf ResNet50 Model
     ### MLPerf Modular Implementation in C++
 
 {{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }}
+
+Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification.

docs/benchmarks/text_to_image/sdxl.md

@@ -37,13 +37,11 @@ Get the Official MLPerf Stable Diffusion Model
 
 === "Intel"
     ### Intel MLPerf Implementation
-    GPTJ-99
 {{ mlperf_inference_implementation_readme (4, "sdxl", "intel") }}
 
 
 === "Qualcomm"
     ### Qualcomm AI100 MLPerf Implementation
 
-    GPTJ-99
 {{ mlperf_inference_implementation_readme (4, "sdxl", "qualcomm") }}
 

docs/submission/index.md

@@ -1,4 +1,4 @@
-If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1).
+If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks/index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1).
 
 ## Generate actual submission tree
 

main.py

@@ -9,6 +9,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation):
      pre_space += " "
 
      content=""
+     scenarios = []
      if implementation == "reference":
        devices = [ "CPU", "CUDA", "ROCm" ]
        if model.lower() == "resnet50":
@@ -23,6 +24,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation):
        devices = [ "CUDA" ]
        frameworks = [ "TensorRT" ]
      elif implementation == "intel":
+       if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]:
+            return pre_space+"    WIP"
        devices = [ "CPU" ]
        frameworks = [ "Pytorch" ]
      elif implementation == "qualcomm":
@@ -31,6 +34,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation):
      elif implementation == "cpp":
        devices = [ "CPU", "CUDA" ]
        frameworks = [ "Onnxruntime" ]
+     elif implementation == "ctuning-cpp":
+       scenarios = [ "SingleStream" ]
+       devices = [ "CPU" ]
+       if model.lower() == "resnet50":
+         frameworks = [ "TFLite" ]
+       else:
+         frameworks = []
 
      if model.lower() == "bert-99.9":
        categories = [ "Datacenter" ]
@@ -40,11 +50,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation):
        categories = [ "Edge", "Datacenter" ]
 
      for category in categories:
-       if category == "Edge":
+       if category == "Edge" and not scenarios:
          scenarios = [ "Offline", "SingleStream" ]
          if model.lower() in [ "resnet50", "retinanet" ]:
-           scenarios.append("Multistream")
-       elif category == "Datacenter":
+           scenarios.append("MultiStream")
+       elif category == "Datacenter" and not scenarios:
          scenarios = [ "Offline", "Server" ] 
 
        content += f"{pre_space}=== \"{category.lower()}\"\n\n"
@@ -69,7 +79,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation):
            content += f"{cur_space2}###### {device} device\n\n"
 
            content += f"{cur_space2}###### Docker Setup Command\n\n"
-           test_query_count=100
+           test_query_count=get_test_query_count(model, implementation, device)
+
            content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True)
            content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n"
            content += f"{cur_space2}<details>\n"
@@ -101,6 +112,21 @@ def mlperf_inference_implementation_readme(spaces, model, implementation):
 
      return content
 
+   def get_test_query_count(model, implementation, device, num_devices=1):
+
+       if model == "resnet50":
+           p_range = 1000
+       elif model in [ "retinanet", "bert-99", "bert-99.9" ]:
+           p_range = 100
+       else:
+           p_range = 50
+       if device == "cuda":
+           p_range *= 40
+       p_range *= num_devices
+
+       return p_range
+
+
 
    @env.macro
    def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False):