Feature/67 update test suite results

test_suite/README.md

@@ -47,6 +47,12 @@ The `process_test_suite.py` script provides various command-line options to cust
   **Usage:** `--limit <number>`  
   **Default:** `0` (no limit)
 
+ `--cases`  
+  **Description:** list specific cases to process
+  **Usage:** `--cases <list>`  
+  **Default:** `[]` (no limit)
+
+
 - `--suite-path`  
   **Description:** Specifies the path to the directory containing the test suite files.  
   **Usage:** `--suite-path <path>`  

test_suite/process_test_suite.py

@@ -45,6 +45,15 @@ def parse_arguments():
         help="Limit to the first n test cases, 0 means no limit",
     )
 
+    parser.add_argument(
+        "--cases",
+        action="extend",
+        nargs="+",
+        type=str,
+        default=[],
+        help="Limit to the cases listed in the file. Empty list means no limit",
+    )
+
     parser.add_argument(
         "--suite-path",
         action="store",
@@ -116,8 +125,8 @@ def process_cases(args):
     matplotlib.use("agg") 
     # Suppress specific UserWarning caused by matplotlib (required to suppress interactive plots)
     warnings.filterwarnings("ignore", category=UserWarning, message="FigureCanvasAgg is non-interactive, and thus cannot be shown")
-    subfolders = os.listdir(suite_path_abs) if args.limit == 0 else os.listdir(suite_path_abs)[:args.limit]    
-    
+    subfolders = os.listdir(suite_path_abs) if args.limit == 0 else os.listdir(suite_path_abs)[:args.limit]   
+
     for subfolder in subfolders:
         # if sbml_level_version is empty string (default), find the highest level and version in the folder
         if args.sbml_level_version == "highest":

test_suite/test_01186/01186-sbml-l3v2-sedml.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Written by libSedML v1.1.5419.25885 see http://libsedml.sf.net -->
+<sedML level="1" version="2" xmlns="http://sed-ml.org/sed-ml/level1/version2">
+  <listOfSimulations>
+    <steadyState id="simulation_1">
+      <algorithm kisaoID="KISAO:0000437" />
+    </steadyState>
+  </listOfSimulations>
+  <listOfModels>
+    <model id="model_1" language="urn:sedml:language:sbml" source="01186-sbml-l3v2.xml" />
+  </listOfModels>
+  <listOfTasks>
+    <task id="task_1" modelReference="model_1" simulationReference="simulation_1" />
+  </listOfTasks>
+  <listOfDataGenerators>
+    <dataGenerator id="R01_1" name="R01">
+      <listOfVariables>
+        <variable id="R01" name="R01" taskReference="task_1" target="/sbml:sbml/sbml:model/sbml:listOfReactions/sbml:reaction[@id='R01']" />
+      </listOfVariables>
+      <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <ci> R01 </ci>
+      </math>
+    </dataGenerator>
+    <dataGenerator id="R26_1" name="R26">
+      <listOfVariables>
+        <variable id="R26" name="R26" taskReference="task_1" target="/sbml:sbml/sbml:model/sbml:listOfReactions/sbml:reaction[@id='R26']" />
+      </listOfVariables>
+      <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <ci> R26 </ci>
+      </math>
+    </dataGenerator>
+    <dataGenerator id="OBJF_1" name="OBJF">
+      <listOfVariables>
+        <variable id="OBJF" name="OBJF" taskReference="task_1" target="/sbml:sbml/sbml:model/fbc:listOfObjectives/fbc:objective[@fbc:id='OBJF']" />
+      </listOfVariables>
+      <math xmlns="http://www.w3.org/1998/Math/MathML">
+        <ci> OBJF </ci>
+      </math>
+    </dataGenerator>
+  </listOfDataGenerators>
+  <listOfOutputs>
+    <report id="report_1" name="Report for test 01186">
+      <listOfDataSets>
+        <dataSet id="ds_R01_1" dataReference="R01_1" label="R01" />
+        <dataSet id="ds_R26_1" dataReference="R26_1" label="R26" />
+        <dataSet id="ds_OBJF_1" dataReference="OBJF_1" label="OBJF" />
+      </listOfDataSets>
+    </report>
+  </listOfOutputs>
+</sedML>

test_suite/test_test_suite_compatibility_biosimulators.py

@@ -15,7 +15,7 @@
 import shutil
 sys.path.append("..")
 import utils
-engines = utils.engines
+engines = utils.ENGINES
 
 
 md_description = \
@@ -44,6 +44,15 @@ def parse_arguments():
         help="Limit to the first n test cases, 0 means no limit",
     )
 
+    parser.add_argument(
+        "--cases",
+        action="extend",
+        nargs="+",
+        type=str,
+        default=[],
+        help="Limit to the cases listed in the file. Empty list means no limit",
+    )
+
     parser.add_argument(
         "--suite-path",
         action="store",
@@ -93,10 +102,15 @@ def process_cases(args):
     os.chdir(args.suite_path) # change to test suite directory
     suite_path_abs = os.getcwd() # absolute path to test suite
 
-    subfolders = os.listdir(suite_path_abs) if args.limit == 0 else os.listdir(suite_path_abs)[:args.limit]   
+    if args.cases != []:
+        subfolders = args.cases
+    else:
+        subfolders = os.listdir(suite_path_abs) if args.limit == 0 else os.listdir(suite_path_abs)[:args.limit]   
+
     print(f"Processing {len(subfolders)} subfolders in {args.suite_path}") 
     test_folder = 'tests'
 
+
     for subfolder in subfolders:
         # create an equivalently named folder in the starting directory
         os.chdir(args.suite_path)
@@ -129,16 +143,19 @@ def process_cases(args):
 
         os.chdir(new_directory)
         print(f"Changed to {new_directory}")
+
+        engine_list = list(engines.keys()) 
 
-        utils.run_biosimulators_remotely_and_locally(os.path.basename(sedml_file_path), 
+        utils.run_biosimulators_remotely_and_locally(engine_list,
+                                 os.path.basename(sedml_file_path), 
                                  os.path.basename(sbml_file_path),
                                  os.path.join(test_folder,'d1_plots_remote'), 
                                  os.path.join(test_folder,'d1_plots_local'),
                                  test_folder=test_folder)
 
 
 if __name__ == "__main__":
-    args = parse_arguments()
+    args = parse_arguments()    
 
     process_cases(args)
 

test_suite/test_test_suite_results_table.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+"""
+produce a markdown table of the results of running various tests on the SBML Test Suite
+
+get this version of the test suite that includes sedml versions or the sedml validation will fail:
+https://github.com/sbmlteam/sbml-test-suite/releases/download/3.4.0/semantic_tests_with_sedml_and_graphs.v3.4.0.zip
+"""
+
+import os
+import sys
+import json
+sys.path.append("..")
+import utils
+engines = utils.ENGINES
+
+
+def create_test_suite_results_tables():
+    """
+    Create a table of results for each test case in the test suite
+
+    """
+    starting_dir = os.getcwd() 
+    print('Current working directory:', starting_dir)
+    subfolders = [f for f in os.listdir(starting_dir) if os.path.isdir(f) and f.startswith('test_')]
+
+    print(subfolders)
+
+    for subfolder in subfolders:
+        subfolder_path = os.path.join(starting_dir, subfolder)
+        os.chdir(subfolder_path)
+        print(f"Changed to {os.getcwd()}")
+
+        test_folder = 'tests'
+
+        results_paths = {
+            "local": os.path.join(subfolder_path, test_folder, 'results_local.json'),
+            "remote": os.path.join(subfolder_path, test_folder, 'results_remote.json')
+        }
+
+        results = {}
+        for key, path in results_paths.items():
+            with open(path, 'r') as f:
+                results[key] = json.load(f)
+
+        d1_plot_paths = {
+            "local": os.path.join(subfolder_path, test_folder, 'd1_plots_local'),
+            "remote": os.path.join(subfolder_path, test_folder, 'd1_plots_remote')
+        }
+
+        # get sbml filename with reg ex (it should contain sbml but not sedml)
+        sbml_file_name = [f for f in os.listdir(subfolder_path) if 'sbml' in f and 'sedml' not in f][0]
+        # also sbml file does not end with omex
+        sbml_file_name = [f for f in os.listdir(subfolder_path) if 'sbml' in f and 'sedml' not in f and not f.endswith('omex')][0]
+        sedml_file_name = [f for f in os.listdir(subfolder_path) if 'sedml' in f and not f.endswith('omex')][0]
+
+        results_table = utils.create_combined_results_table(results["remote"], 
+                                        results["local"], 
+                                        sedml_file_name, 
+                                        sbml_file_name, 
+                                        d1_plot_paths["local"], 
+                                        d1_plot_paths["remote"],
+                                        test_folder='tests')
+
+
+        print(results_table)
+
+
+if __name__ == "__main__":
+    create_test_suite_results_tables()

utils/__init__.py

@@ -21,6 +21,7 @@
 from pyneuroml import biosimulations
 import pandas as pd
 from requests.exceptions import HTTPError 
+import json
 
 ENGINES = {
     'amici': {
@@ -397,20 +398,17 @@ def ansi_to_html(text):
         if len(text_message) > 0:
             text = text_message
             text = bytes(text[0], "utf-8").decode("unicode_escape")
-        elif 'The COMBINE/OMEX did not execute successfully:' in text:
-            text = text # to deal with remote error message
-        else:
-            text = text.replace('|', '')
-            return text
+        # elif 'The COMBINE/OMEX did not execute successfully:' in text:
+        #     text = text # to deal with remote error message
 
         text = text.replace('|', '')
 
         # # for any text with "<*>" remove "<" as well as ">" but leave wildcard text *
         text = re.sub(r'<([^>]*)>', r'\1', text)
 
         # replace color codes with html color codes
-        text = text.replace("\x1b[33m",'<span style="color:darkorange;">')
-        text = text.replace("\x1b[31m",'<span style="color:red;">')
+        text = text.replace("\x1b[33m","")
+        text = text.replace("\x1b[31m","")
 
         # # remove .\x1b[0m
         text = text.replace("\x1b[0m", "")
@@ -429,28 +427,32 @@ def ansi_to_html(text):
         text = text.replace('BioSimulatorsWarning:', '<br><br>BioSimulatorsWarning:<br><br>')
         text = text.replace('warnings.warn(termcolor.colored(message, Colors.warning.value), category)', '<br>')
 
-        # if text includes The COMBINE/OMEX did not execute successfully: make everyhting from that point red
-        text = text.replace('The COMBINE/OMEX did not execute successfully:', '<span style="color:red;">The COMBINE/OMEX did not execute successfully:')
-    return text
+        return text
 
 def check_file_compatibility_test(engine, model_filepath, experiment_filepath):
     '''
     Check if the file extensions suggest the file types are compatible with the engine.
-    This is done by comparing the file extensions of the model and experiment files with the file types supported by the engine.
-    For SED-ML files, the expected file extension is '.sedml'. For SBML files, the expected file extension is '.sbml'.
+    Only .sedml and .sbml files, and .xml files with 'sedml' and/or 'sbml' in their filename
+    are considered at this moment. It can be extended to other cases if needed in the future.
     '''
-    input_filetypes_tuple = get_filetypes(model_filepath, experiment_filepath)
+    file_extensions = get_filetypes(model_filepath, experiment_filepath)
     engine_filetypes_tuple_list = ENGINES[engine]['formats']
     flat_engine_filetypes_tuple_list = [item for sublist in engine_filetypes_tuple_list for item in sublist if sublist != 'unclear']
     compatible_filetypes = [TYPES[i] for i in flat_engine_filetypes_tuple_list if i in list(TYPES.keys())]
 
-    if input_filetypes_tuple in engine_filetypes_tuple_list:
-        file_types = [TYPES[i] for i in input_filetypes_tuple]
-        return 'pass', (f"The file extensions {input_filetypes_tuple} suggest the input file types are '{file_types}'. {compatible_filetypes} are compatible with {engine}")
-    if 'xml' in input_filetypes_tuple:
-        return 'unsure', (f"The file extensions of the input files are '{input_filetypes_tuple}'. These may be compatible with {engine}. {compatible_filetypes} are compatible with {engine}")
+    if file_extensions in engine_filetypes_tuple_list:
+        file_types = [TYPES[i] for i in file_extensions]
+        return 'pass', (f"The file extensions {file_extensions} suggest the input file types are '{file_types}'. {compatible_filetypes} are compatible with {engine}.")
+    if 'xml' in file_extensions:
+            if 'sbml' in model_filepath and 'sedml' not in model_filepath:
+                if 'sbml' in experiment_filepath and 'sedml' in experiment_filepath:
+                    file_types = ('sbml', 'sedml')
+                    if file_types in engine_filetypes_tuple_list:
+                        return 'pass', (f"The filenames '{model_filepath}' and '{experiment_filepath}' suggest the input files are {[TYPES[i] for i in file_types]} which is compatible with {engine}.<br><br>{compatible_filetypes} are compatible with {engine}.")
+                    else: 
+                        return 'unsure', (f"The filenames '{model_filepath}' and '{experiment_filepath}' suggest the input files are {[TYPES[i] for i in file_types]} which is not compatible with {engine}.<br><br>{compatible_filetypes} are compatible with {engine}.")
     else:
-        return 'FAIL', (f"The file extensions {input_filetypes_tuple} suggest the input file types are not compatibe with {engine}. {compatible_filetypes} are compatible with {engine}")
+        return 'unsure', (f"The file extensions {file_extensions} suggest the input file types may not be compatibe with {engine}.<br><br>{compatible_filetypes} are compatible with {engine}.")
 
 
 def collapsible_content(content, title='Details'):
@@ -463,25 +465,37 @@ def collapsible_content(content, title='Details'):
         return f'<details><summary>{title}</summary>{content}</details>'
     else:
         return f'{title}'
-    
+
 def get_filetypes(model_filepath, simulation_filepath):
     """
     Get the filetypes of the model and simulation files
 
     Input: model_filepath, simulation_filepath
     Output: tuple of filetypes
     """
-    if model_filepath.endswith(".sbml") and simulation_filepath.endswith(".sedml"):
-        filetypes = ('sbml', 'sedml')
-    elif model_filepath.endswith(".xml") and simulation_filepath.endswith(".xml"):
-        filetypes = ('xml', 'xml')
-    elif model_filepath.endswith(".xml") and simulation_filepath.endswith(".sedml"):
-        filetypes = ('xml', 'sedml')
-    elif model_filepath.endswith(".sbml") and simulation_filepath.endswith(".xml"):
-        filetypes = ('sbml', 'xml')
-    else:
-        filetypes = "other"
-    return filetypes
+    model_ext = os.path.splitext(model_filepath)[-1].lstrip('.')
+    simulation_ext = os.path.splitext(simulation_filepath)[-1].lstrip('.')
+
+    return (model_ext, simulation_ext)
+
+# def get_filetypes(model_filepath, simulation_filepath):
+#     """
+#     Get the filetypes of the model and simulation files
+
+#     Input: model_filepath, simulation_filepath
+#     Output: tuple of filetypes
+#     """
+#     if model_filepath.endswith(".sbml") and simulation_filepath.endswith(".sedml"):
+#         filetypes = ('sbml', 'sedml')
+#     elif model_filepath.endswith(".xml") and simulation_filepath.endswith(".xml"):
+#         filetypes = ('xml', 'xml')
+#     elif model_filepath.endswith(".xml") and simulation_filepath.endswith(".sedml"):
+#         filetypes = ('xml', 'sedml')
+#     elif model_filepath.endswith(".sbml") and simulation_filepath.endswith(".xml"):
+#         filetypes = ('sbml', 'xml')
+#     else:
+#         filetypes = "other"
+#     return filetypes
 
 def delete_output_folder(output_dir):
     '''
@@ -1038,6 +1052,7 @@ def create_results_table(results, sbml_filepath, sedml_filepath, output_dir):
     pass_html = "&#9989; PASS"
     fail_html = "&#10060; FAIL"
     warning_html = "&#9888; WARNING"
+    unsure_html = "&#10067; UNSURE"
     xfail_html = "&#9888; XFAIL"
 
     for e in results.keys():
@@ -1076,7 +1091,7 @@ def create_results_table(results, sbml_filepath, sedml_filepath, output_dir):
         if compatibility_content[0] == 'pass':
             results_table.loc[results_table[ENGINE] == e, COMPAT] = collapsible_content(compatibility_content[1], title=f'{pass_html}')
         elif compatibility_content[0] == 'unsure':
-            results_table.loc[results_table[ENGINE] == e, COMPAT] = collapsible_content(compatibility_content[1], title=f'{warning_html}')
+            results_table.loc[results_table[ENGINE] == e, COMPAT] = collapsible_content(compatibility_content[1], title=f'{unsure_html}')
         else:
             results_table.loc[results_table[ENGINE] == e, COMPAT] = collapsible_content(compatibility_content[1], title=f'{fail_html}')
 
@@ -1233,6 +1248,15 @@ def create_combined_results_table(results_remote,
 
     suffix_remote = ' (R)'
     suffix_local = ' (L)'
+
+    # save results_remote and results_local as json files with dicts
+    path_to_results_remote = os.path.join(test_folder, 'results_remote.json')
+    path_to_results_local = os.path.join(test_folder, 'results_local.json')
+
+    with open(path_to_results_remote, 'w') as f:
+        json.dump(results_remote, f, indent=4)
+    with open(path_to_results_local, 'w') as f:
+        json.dump(results_local, f, indent=4)
 
     # Create results tables for remote and local results
     results_table_remote = create_results_table(results_remote, sbml_file_name, sedml_file_name, d1_plots_remote_dir)