diff --git a/base_bert_loadgen_experiment/data_axs.json b/base_bert_loadgen_experiment/data_axs.json index 3b733ff..c4de3b2 100644 --- a/base_bert_loadgen_experiment/data_axs.json +++ b/base_bert_loadgen_experiment/data_axs.json @@ -49,5 +49,6 @@ ]], {} ], "f1": [ "^^" , "dig","accuracy_dict.f1" ], "exact_match": [ "^^" , "dig","accuracy_dict.exact_match" ], - "accuracy": [ "^^" , "get", "f1" ] + "accuracy": [ "^^" , "get", "f1" ], + "accuracy_range_dict": { "f1": [89.965, null], "exact_match": [null, null]} } diff --git a/base_gptj_loadgen_experiment/data_axs.json b/base_gptj_loadgen_experiment/data_axs.json index b39bc5c..86022dd 100644 --- a/base_gptj_loadgen_experiment/data_axs.json +++ b/base_gptj_loadgen_experiment/data_axs.json @@ -61,5 +61,6 @@ "rougeL": [ "^^" , "dig","accuracy_dict.rougeL" ], "rougeLsum": [ "^^" , "dig","accuracy_dict.rougeLsum" ], "gen_len": [ "^^" , "dig","accuracy_dict.gen_len" ], - "gen_num": [ "^^" , "dig","accuracy_dict.gen_num" ] + "gen_num": [ "^^" , "dig","accuracy_dict.gen_num" ], + "accuracy_range_dict": { "rouge1": [ 42.9865, null ], "rouge2": [ 20.1235, null ], "rougeL": [ 29.9881, null ], "gen_len": [ 4016878, null ] } } diff --git a/base_image_classification_loadgen_experiment/data_axs.json b/base_image_classification_loadgen_experiment/data_axs.json index fdf304d..61f0f4b 100644 --- a/base_image_classification_loadgen_experiment/data_axs.json +++ b/base_image_classification_loadgen_experiment/data_axs.json @@ -17,11 +17,13 @@ "python_deps": [ [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=numpy" ] ], "abs_script_path": ["^^", "get", "abs_script_path"], "script_extra_params": [ "^^", "substitute", "--mlperf-accuracy-file \"#{abs_accuracy_log_path}#\" --imagenet-val-file \"#{ground_truth_path}#\"" ], - "capture_output": true - } ] + "capture_output": true, + "split_to_lines": true + } ], + [ "__getitem__", 0 ] ]], {} ], - "accuracy": [ "^^", "execute", [[ + "accuracy": [ "^^", "execute", [[ [ "get", "accuracy_report" ], ["split", ","], [ "__getitem__", 0 ], @@ -30,5 +32,8 @@ 0, [ "func", "float"] ]] - ] + ], + + "accuracy_dict": {"accuracy": [ "^^" , "get", "accuracy" ] }, + "accuracy_range_dict": { "accuracy": [75.695, null] } } diff --git a/base_loadgen_program/code_axs.py b/base_loadgen_program/code_axs.py index d33f2bb..a20168c 100644 --- a/base_loadgen_program/code_axs.py +++ b/base_loadgen_program/code_axs.py @@ -101,3 +101,7 @@ def generate_current_timestamp(used_for="unknown"): timestamp = datetime.datetime.now().strftime("%Y.%m.%dT%H:%M:%S") #print(f"GENERATING TIMESTAMP: {timestamp} used for: {used_for}") return timestamp + +def list_to_str(sdk_ver_list): + sdk_ver_list_stringified = [ str(elem) for elem in sdk_ver_list ] + return ".".join(sdk_ver_list_stringified) diff --git a/base_object_detection_loadgen_experiment/data_axs.json b/base_object_detection_loadgen_experiment/data_axs.json index 9cc5da6..d542b49 100644 --- a/base_object_detection_loadgen_experiment/data_axs.json +++ b/base_object_detection_loadgen_experiment/data_axs.json @@ -43,5 +43,7 @@ [ "func", "float"] ]], {} ], - "accuracy": [ "^^", "get", "mAP" ] + "accuracy": [ "^^", "get", "mAP" ], + "accuracy_dict": { "mAP": [ "^^", "get", "accuracy" ] }, + "accuracy_range_dict": { "mAP": [37.175, null] } }