Supported of TEST06 for llama2 in submitter

krai · Mar 26, 2024 · c65b1a3 · c65b1a3
1 parent d46c75b
commit c65b1a3
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 14 deletions.
diff --git a/submitter/code_axs.py b/submitter/code_axs.py
@@ -7,6 +7,22 @@
 from ufun import save_json
 from tabulate import tabulate
 
+def create_run_verification_input_dict(task, tmp_dir, verify_script_path, results_dir, compliance_dir, output_dir, scenario, dtype=None):
+    result_dict = {
+        "in_dir": tmp_dir,
+        "verify_script_path": verify_script_path,
+        "compliance_dir": compliance_dir,
+        "output_dir": output_dir
+    }
+    if task == "llm":
+        result_dict [ "scenario" ] = scenario
+        result_dict [ "dtype" ] = dtype
+    else:
+        result_dict [ "results_dir" ] = results_dir
+    print("result_dict = ", result_dict)
+    return result_dict
+
+
 def scenarios_from_sut_type_and_task(sut_system_type, task):
 
     if sut_system_type == "edge":
@@ -49,7 +65,8 @@ def list_experiment_entries( power, sut_name, sut_system_type, task, division, e
             "object_detection":     [ 'TEST01', 'TEST05' ],
             "bert":                 [ 'TEST01', 'TEST05' ],
             "gptj":                 [ ],
-            "text_to_image":        [ ]
+            "text_to_image":        [ ],
+            "llm":                  [ 'TEST06' ]
         }[task]
 
         for compliance_test_name in compliance_test_list:
@@ -251,7 +268,7 @@ def lay_out(experiment_entries, division, submitter, log_truncation_script_path,
             if not with_power:
                 results_path_syll.append( 'run_1' )
 
-        if mode=='performance' and compliance_test_name in [ "TEST01", "TEST04", "TEST05" ]:
+        if mode=='performance' and compliance_test_name in [ "TEST01", "TEST04", "TEST05", "TEST06" ]:
             results_path_syll.extend(( mode, 'run_1' ))
 
         results_path = make_local_dir( results_path_syll, submitted_tree_path )
@@ -312,7 +329,7 @@ def lay_out(experiment_entries, division, submitter, log_truncation_script_path,
             shutil.copytree( src_images_dir, results_images_path, dirs_exist_ok=True)
 
         # -------------------------------[ compliance , verification ]--------------------------------------
-        if compliance_test_name in [ "TEST01", "TEST04", "TEST05" ]:
+        if compliance_test_name in [ "TEST01", "TEST04", "TEST05", "TEST06" ]:
             compliance_path_test = make_local_dir( [ division, submitter, 'compliance', sut_name , mlperf_model_name, scenario, compliance_test_name ], submitted_tree_path )
 
             ("Verification for ", compliance_test_name)
@@ -322,13 +339,11 @@ def lay_out(experiment_entries, division, submitter, log_truncation_script_path,
             compliance_dir = src_dir
             output_dir = os.path.join(submitter_path ,'compliance', sut_name , mlperf_model_name, scenario)
             verify_script_path =  os.path.join(compliance_path,compliance_test_name, "run_verification.py")
-            result_verify =  __entry__.call('get', 'run_verify', {
-                    "in_dir": tmp_dir,
-                    "verify_script_path": verify_script_path,
-                    "results_dir": results_dir,
-                    "compliance_dir": compliance_dir,
-                    "output_dir": output_dir
-                        } )
+            if task == "llm":
+                dtype = experiment_entry['dtype']
+            else:
+                dtype = ""
+            result_verify =  __entry__.call('get', 'run_verify', create_run_verification_input_dict(task, tmp_dir, verify_script_path, results_dir, compliance_dir, output_dir, target_scenario, dtype))
             if result_verify == "":
                 shutil.rmtree(tmp_dir, ignore_errors=True)
             else:
@@ -380,7 +395,6 @@ def run_checker(submitted_tree_path, division, submitter, submission_checker_pat
 
 def full_run(experiment_entries, division, submitter, log_truncation_script_path, submission_checker_path, checker_log_path, sut_path, compliance_path, scenarios, power=False, infer_from_ss=False, model_meta_data=None, submitted_tree_path=None, __entry__=None):
 
-    print("DEBUG:full run entry ", __entry__)
     if os.path.exists(submitted_tree_path):
         print("The path " + submitted_tree_path + " exists, skipping lay_out()")
     else:

diff --git a/submitter/data_axs.json b/submitter/data_axs.json
@@ -78,6 +78,22 @@
 
     "desired_python_version": ["^", "kernel_python_major_dot_minor"],
     "python_tool_entry": [ "^", "byquery", [[ "shell_tool", "can_python", ["desired_python_version", ["^^", "get", "desired_python_version"]] ]], {}, ["desired_python_version"] ],
+    "scenario_cmd": [ "^^", "case",  [ ["^^", "get", "task"],
+        "llm" , [ "^^", "substitute", "--scenario #{scenario}#"] ],
+        { "default_value": ""  }
+    ],
+
+    "result_dir_cmd": [ "^^", "case",  [ ["^^", "get", "task"],
+        "llm" , "" ],
+        { "default_value": [ "^^", "substitute", "--results_dir #{results_dir}#" ] }
+    ],
+
+
+    "dtype_cmd": [ "^^", "case",  [ ["^^", "get", "task"],
+        "llm" , [ "^^", "substitute", "--dtype #{dtype}#"] ],
+        { "default_value": "" }
+    ],
+
 
     "run_verify": [ "^^", "execute", [[
         [ "get_kernel" ],
@@ -88,12 +104,14 @@
                 [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=numpy" ]
             ],
             "abs_script_path": ["^^", "get", "verify_script_path"],
-            "script_extra_params": [ "^^", "substitute", "--results_dir \"#{results_dir}#\" --compliance_dir \"#{compliance_dir}#\" --output_dir \"#{output_dir}#\" " ],
+            "script_extra_params": [ "^^", "substitute", "#{result_dir_cmd}# --compliance_dir \"#{compliance_dir}#\" --output_dir \"#{output_dir}#\" #{scenario_cmd}# #{dtype_cmd}#" ],
             "capture_output": false,
             "capture_stderr": true,
-            "results_dir": [ "^^", "get", "results_dir" ],
+            "results_dir_cmd": [ "^^", "get", "results_dir_cmd" ],
             "compliance_dir": [ "^^", "get", "compliance_dir" ],
-            "output_dir": [ "^^", "get", "output_dir" ]
+            "output_dir": [ "^^", "get", "output_dir" ],
+            "scenario_cmd": [ "^^", "get", "scenario_cmd" ],
+            "dtype_cmd": [ "^^", "get", "dtype_cmd" ]
         } ]
     ]] ],