Update data_axs.json for bert,resnet50,retinanet to support validate_…

…accuracy. (#39) * Update data_axs.json for bert,resnet50,retinanet to support validate_accuracy function. * Add function to return sdk_ver * Update code_axs.py for base_loadgen_program * Update accuracy_range_dict for gptj --------- Co-authored-by: krai <[email protected]>
krai · Mar 15, 2024 · fe52dc2 · fe52dc2
1 parent 4564802
commit fe52dc2
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 7 deletions.
diff --git a/base_bert_loadgen_experiment/data_axs.json b/base_bert_loadgen_experiment/data_axs.json
@@ -49,5 +49,6 @@
      ]], {} ],
     "f1": [ "^^" , "dig","accuracy_dict.f1" ],
     "exact_match": [ "^^" , "dig","accuracy_dict.exact_match" ],
-    "accuracy": [ "^^" , "get", "f1" ]
+    "accuracy": [ "^^" , "get", "f1" ],
+    "accuracy_range_dict": { "f1": [89.965, null], "exact_match": [null, null]}
 }
diff --git a/base_gptj_loadgen_experiment/data_axs.json b/base_gptj_loadgen_experiment/data_axs.json
@@ -61,5 +61,6 @@
     "rougeL": [ "^^" , "dig","accuracy_dict.rougeL" ],
     "rougeLsum": [ "^^" , "dig","accuracy_dict.rougeLsum" ],
     "gen_len": [ "^^" , "dig","accuracy_dict.gen_len" ],
-    "gen_num": [ "^^" , "dig","accuracy_dict.gen_num" ]
+    "gen_num": [ "^^" , "dig","accuracy_dict.gen_num" ],
+    "accuracy_range_dict": { "rouge1": [ 42.9865, null ], "rouge2": [ 20.1235, null ], "rougeL": [ 29.9881, null ], "gen_len": [ 4016878, null ] }
 }
diff --git a/base_image_classification_loadgen_experiment/data_axs.json b/base_image_classification_loadgen_experiment/data_axs.json
@@ -17,11 +17,13 @@
                 "python_deps": [ [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=numpy" ] ],
                 "abs_script_path": ["^^", "get", "abs_script_path"],
                 "script_extra_params": [ "^^", "substitute", "--mlperf-accuracy-file \"#{abs_accuracy_log_path}#\" --imagenet-val-file \"#{ground_truth_path}#\"" ],
-                "capture_output": true
-            } ]
+                "capture_output": true,
+		"split_to_lines": true
+            } ],
+        [ "__getitem__", 0 ]
     ]], {} ],
 
-    "accuracy": [ "^^", "execute", [[
+        "accuracy": [ "^^", "execute", [[
         [ "get", "accuracy_report" ],
         ["split", ","],
         [ "__getitem__", 0 ],
@@ -30,5 +32,8 @@
         0,
         [ "func", "float"]
         ]]
-    ]
+    ],
+
+    	"accuracy_dict": {"accuracy": [ "^^" , "get", "accuracy" ] },
+	"accuracy_range_dict": { "accuracy": [75.695, null] }
 }
diff --git a/base_loadgen_program/code_axs.py b/base_loadgen_program/code_axs.py
@@ -101,3 +101,7 @@ def generate_current_timestamp(used_for="unknown"):
     timestamp = datetime.datetime.now().strftime("%Y.%m.%dT%H:%M:%S")
     #print(f"GENERATING TIMESTAMP: {timestamp} used for: {used_for}")
     return timestamp
+
+def list_to_str(sdk_ver_list):
+    sdk_ver_list_stringified = [ str(elem) for elem in sdk_ver_list ]
+    return ".".join(sdk_ver_list_stringified)
diff --git a/base_object_detection_loadgen_experiment/data_axs.json b/base_object_detection_loadgen_experiment/data_axs.json
@@ -43,5 +43,7 @@
         [ "func", "float"]
     ]], {} ],
 
-    "accuracy": [ "^^", "get", "mAP" ]
+    "accuracy": [ "^^", "get", "mAP" ],
+    "accuracy_dict": { "mAP": [ "^^", "get", "accuracy" ] },
+    "accuracy_range_dict": { "mAP": [37.175, null] }
 }