From 1551d272e4c52fab874f1b4139d22e5ff121e498 Mon Sep 17 00:00:00 2001 From: Ben Stewart Date: Wed, 7 Aug 2024 17:43:40 +0100 Subject: [PATCH 1/3] A more detailed summary of performance run available with 'get performance' command --- base_loadgen_experiment/code_axs.py | 49 ++++++++++++++++++++------- base_loadgen_experiment/data_axs.json | 25 ++++++++------ 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/base_loadgen_experiment/code_axs.py b/base_loadgen_experiment/code_axs.py index be58fdd..70fee06 100755 --- a/base_loadgen_experiment/code_axs.py +++ b/base_loadgen_experiment/code_axs.py @@ -14,6 +14,10 @@ def parse_summary(abs_log_summary_path): k = k.replace(' ', '_').replace('/', '_').replace('*', '').replace(')', '').replace('(', '') parsed_summary[k] = to_num_or_not_to_num(v) + + return parsed_summary + +def beautify_summary(parsed_summary): ureg = UnitRegistry() @@ -42,7 +46,7 @@ def parse_summary(abs_log_summary_path): v = (v*unit).to_compact() - if v.u == ureg.us: + if v.u == ureg.us: v.ito(ureg.ms) # Keep everything in milliseconds kv_with_units[k] = v @@ -61,21 +65,40 @@ def parse_summary(abs_log_summary_path): return beautified_summary -def parse_performance(summary, scenario_performance_map, raw=False): +def calc_latency_cutoff_ratio(parsed_summary): + + scenario = parsed_summary["Scenario"] + + if scenario == "Server": + return parsed_summary["99.00_percentile_latency_ns"]/parsed_summary["target_latency_ns"] + +#returns list of formatted performance metrics (as strings) for given experiment +def parse_performance(beautified_summary, latency_cutoff_ratio, scenario_performance_map, raw=False): - scenario = summary["Scenario"] - validity = summary["Result_is"] + scenario = beautified_summary["Scenario"] + validity = beautified_summary["Result_is"] if raw and validity == "INVALID": - return None + return None - key_name, multiplier, formatting, units = scenario_performance_map[scenario][validity] - if raw: - return summary[key_name] - else: - formatted_value = ('{:'+formatting+'}').format(summary[key_name]*multiplier) - display_key_name = key_name.replace('_ns', '') - return '{} : {}={}{}'.format(validity, display_key_name, formatted_value, units) + performance_metrics = scenario_performance_map[scenario][validity] + formatted_performance_metrics = ['{} :'.format(validity)] # set first element + + for key_name in performance_metrics: + + if raw: + if key_name == "latency_cutoff_ratio": + formatted_performance_metrics.append(latency_cutoff_ratio) + else: + formatted_performance_metrics.append(beautified_summary[key_name]) + + else: #no need for multiplier, formatting, units in scenario_performance_map - the beautify_summary function does all of this already + if key_name == "latency_cutoff_ratio": + formatted_performance_metrics.append('{}={}'.format(key_name, latency_cutoff_ratio)) + else: + formatted_performance_metrics.append('{}={}'.format(key_name, beautified_summary[key_name])) + + return formatted_performance_metrics def unpack_accuracy_log(raw_accuracy_log): @@ -124,7 +147,7 @@ def guess_command(tags, framework, loadgen_scenario, loadgen_mode, model_name, l return "axs byquery "+','.join(terms_list) -def validate_accuracy(accuracy_dict, accuracy_range_dict ): +def validate_accuracy(accuracy_dict, accuracy_range_dict): result_list = [] for key in accuracy_dict: if key not in accuracy_range_dict: diff --git a/base_loadgen_experiment/data_axs.json b/base_loadgen_experiment/data_axs.json index c048674..a371d92 100644 --- a/base_loadgen_experiment/data_axs.json +++ b/base_loadgen_experiment/data_axs.json @@ -9,28 +9,32 @@ "rel_log_summary_path": "mlperf_log_summary.txt", "abs_log_summary_path": [ "^^", "get_path_from", "rel_log_summary_path" ], - "summary": [ "^^", "parse_summary" ], + "parsed_summary": [ "^^", "parse_summary" ], + + "beautified_summary": [ "^^", "beautify_summary" ], + + "latency_cutoff_ratio": [ "^^", "calc_latency_cutoff_ratio" ], "scenario_performance_map": { "Offline": { - "VALID": ["Samples_per_second", 1, ".3f", ""], - "INVALID": ["Samples_per_second", 1, ".3f", ""] + "VALID": ["Samples_per_second", "target_qps"], + "INVALID": ["Samples_per_second", "target_qps"] }, "SingleStream": { - "VALID": ["_Early_stopping_90th_percentile_estimate", 1e-6, ".3f", " (milliseconds)"], - "INVALID": ["90th_percentile_latency_ns", 1e-6, ".3f", " (milliseconds)"] + "VALID": ["90th_percentile_latency", "_Early_stopping_90th_percentile_estimate"], + "INVALID": ["90th_percentile_latency", "_Early_stopping_90th_percentile_estimate"] }, "MultiStream": { - "VALID": ["_Early_stopping_99th_percentile_estimate", 1e-6, ".3f", " (milliseconds)"], - "INVALID": ["99th_percentile_latency_ns", 1e-6, ".3f", " (milliseconds)"] + "VALID": ["99th_percentile_latency", "_Early_stopping_99th_percentile_estimate"], + "INVALID": ["99th_percentile_latency", "_Early_stopping_99th_percentile_estimate"] }, "Server": { - "VALID": ["Scheduled_samples_per_second", 1, ".3f", ""], - "INVALID": ["99.00_percentile_latency_ns", 1e-6, ".3f", " (milliseconds)"] + "VALID": ["target_qps", "99.00_percentile_latency", "target_latency", "latency_cutoff_ratio", "Completed_samples_per_second"], + "INVALID": ["target_qps", "99.00_percentile_latency", "target_latency", "latency_cutoff_ratio", "Completed_samples_per_second"] } }, - "performance": ["^^", "parse_performance"], + "performance": ["^^", "parse_performance"], "accuracy_report": [ "^^", "execute", [[ [ "plant", ["accuracy_report", [ "^^", "get", "extract_accuracy_report" ]] ], @@ -45,6 +49,7 @@ 0, [ "func", "ufun.load_json" ] ]] ], + "readable_accuracy_log": [ "^^", "unpack_accuracy_log" ], "accuracy_range_dict": {}, From 257c627a5d59a7fc07faf917154ed9979add0e2d Mon Sep 17 00:00:00 2001 From: Ben Stewart Date: Fri, 9 Aug 2024 15:50:05 +0100 Subject: [PATCH 2/3] Removed unsightly colon from validity message --- base_loadgen_experiment/code_axs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/base_loadgen_experiment/code_axs.py b/base_loadgen_experiment/code_axs.py index 70fee06..44aa355 100755 --- a/base_loadgen_experiment/code_axs.py +++ b/base_loadgen_experiment/code_axs.py @@ -17,6 +17,7 @@ def parse_summary(abs_log_summary_path): return parsed_summary + def beautify_summary(parsed_summary): ureg = UnitRegistry() @@ -71,7 +72,8 @@ def calc_latency_cutoff_ratio(parsed_summary): if scenario == "Server": return parsed_summary["99.00_percentile_latency_ns"]/parsed_summary["target_latency_ns"] - + + #returns list of formatted performance metrics (as strings) for given experiment def parse_performance(beautified_summary, latency_cutoff_ratio, scenario_performance_map, raw=False): @@ -82,7 +84,7 @@ def parse_performance(beautified_summary, latency_cutoff_ratio, scenario_perform return None performance_metrics = scenario_performance_map[scenario][validity] - formatted_performance_metrics = ['{} :'.format(validity)] # set first element + formatted_performance_metrics = ['{}'.format(validity)] # set first element for key_name in performance_metrics: From e0e929ce1d071224541b7fbefb8cffb0062b493c Mon Sep 17 00:00:00 2001 From: Ben Stewart Date: Wed, 14 Aug 2024 09:41:02 +0100 Subject: [PATCH 3/3] Latency_cutoff_ratio now to 2dp --- base_loadgen_experiment/code_axs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_loadgen_experiment/code_axs.py b/base_loadgen_experiment/code_axs.py index 44aa355..b05317d 100755 --- a/base_loadgen_experiment/code_axs.py +++ b/base_loadgen_experiment/code_axs.py @@ -96,7 +96,7 @@ def parse_performance(beautified_summary, latency_cutoff_ratio, scenario_perform else: #no need for multiplier, formatting, units in scenario_performance_map - the beautify_summary function does all of this already if key_name == "latency_cutoff_ratio": - formatted_performance_metrics.append('{}={}'.format(key_name, latency_cutoff_ratio)) + formatted_performance_metrics.append('{}={:.2f}'.format(key_name, latency_cutoff_ratio)) else: formatted_performance_metrics.append('{}={}'.format(key_name, beautified_summary[key_name]))