merge integration to validation

docs/source/conf.py

@@ -19,7 +19,6 @@
 copyright = '2024, Sostrades Developers'
 author = 'Sostrades Developers'
 
-# release = '0.1'
 # version = '0.1.0'
 
 # -- General configuration

parameters_glossary.csv

@@ -1,6 +1,6 @@
 id,label,unit,definition,definitionSource,ACLTag
 algo,Optimization Algorithm,,Optimization algorithm from GEMSEO library among the available list,,
-acceleration,MDA Acceleration Method,,Acceleration method for MDA algorithms,,
+acceleration_method,MDA Acceleration Method,,Acceleration method for MDA algorithms,,
 algo_options,Algorithm options,,Options for the chosen optimization algorithm ,,
 builder_mode,Builder Model,-,,,
 approximate_jacobian,approximate_jacobian,[-],,,
@@ -66,7 +66,7 @@ parallel_options,Parallel Options,,Parallel options of the optimization problem
 power,Power,,,,
 Product_list,List of Products,,List of Products,,
 propagate_cache_to_children,Propagate Cache,,Propagate Cache to children disciplines,,
-relax_factor,Relaxation Factor,,,,
+over_relaxation_factor,Relaxation Factor,,,,
 repo_of_sub_processes,Repository of subprocesses,,Folder root of the sub processes to be nested inside the DoE. If 'None' then it uses the sos_processes python for doe creation.,,
 resource_list,Resource List,,"coal, gas, oil, uranium",,
 sale_price_dict,Sale price per program,,Sale price of an aircraft per program,,CC - Costs
@@ -89,7 +89,7 @@ scenario_name,Scenario Name,,,,
 scenario_name_dict,Scenarii Dict,,,,
 scenario_name_trade,Scenario Name Trade,,,,
 sensitivity_outputs,Sensitivity Outputs,,,,D - Nominal property
-sub_mda_class,Sub MDA Class,,The class to instantiate for sub Multi-disciplinary Design Analysis,,D - Nominal property
+inner_mda_name,Sub MDA Class,,The class to instantiate for sub Multi-disciplinary Design Analysis,,D - Nominal property
 sub_process_folder_name,Folder name of the subprocess,,Selected process folder name to be nested inside the DoE.If 'None' then it uses the sos_processes python for doe creation.,,
 subsystems_list,SubSystems List,,,,
 tolerance,Tolerance,,"Tolerance of the iterative direct coupling solver, norm of the current residuals divided by initial residuals norm shall be lower than the tolerance to stop iterating",,

requirements.in

@@ -1,6 +1,6 @@
 # Project libraries
 # gemseo
-
+gemseo[all]@git+https://gitlab.com/gemseo/dev/gemseo@07dac5f70ad7cc3ebd65d4e5e61c835f028aec59
 # External libraries
 black==24.4.2
 chaospy==4.3.15
@@ -15,15 +15,17 @@ matplotlib==3.9.2
 numpy==1.24.4
 openturns==1.23
 pandas==2.2.2
-plotly==5.22.0
+plotly==5.24.0
 pyarrow==16.1.0
 pycryptodome==3.20.0
 python-arango==8.0.0
 PyYAML==6.0.1
-scipy==1.10.1
+scipy==1.13.1
 six==1.16.0
-sympy==1.13.0
-tqdm==4.66.4
+sympy==1.13.2
+tqdm==4.66.5
+future==1.0.0
+numpoly==1.2.14  # last version with wheel
 
 # Development requirements
 # breaking change in 8.2.0 https://docs.pytest.org/en/stable/changelog.html#pytest-8-2-0-2024-04-27
@@ -32,4 +34,4 @@ pytest==8.1.2
 pytest-cov==5.0.0
 pytest-xdist==3.6.1
 pytest-durations==1.2.0
-ruff==0.5.0
+ruff==0.6.8

sostrades_core/__init__.py

@@ -16,5 +16,15 @@
 '''
 
 import logging
+import os
+from os.path import dirname, join
 
 logging.basicConfig(level=logging.INFO)
+
+# set-up the folder where GEMSEO will look-up for new wrapps (solvers, grammars etc)
+logging.getLogger('gemseo').setLevel('DEBUG')
+parent_dir = dirname(__file__)
+GEMSEO_ADDON_DIR = "gemseo_addon"
+EXEC_ENGINE = "execution_engine"
+
+os.environ["GEMSEO_PATH"] = join(parent_dir, EXEC_ENGINE, GEMSEO_ADDON_DIR)

sostrades_core/datasets/dataset_info/dataset_info_factory.py

@@ -73,7 +73,7 @@ def get_dataset_info_version(cls, dataset_mapping_key: str) -> DatasetInfoSerial
             DatasetInfoSerializerVersion: The corresponding DatasetInfoSerializerVersion enum value.
         """
         # check if the key starts with V0 or V1 (or v0 or v1)
-        version_pattern = r"^([Vv][0-9])\|"
+        version_pattern = r"^([Vv][\d])\|"
         match = re.match(version_pattern, dataset_mapping_key)
         version = DatasetInfoSerializerVersion.V0
         if match:

sostrades_core/datasets/datasets_connectors/abstract_datasets_connector.py

@@ -57,9 +57,11 @@ class AbstractDatasetsConnector(abc.ABC):
     BOOL_VALUE = "parameter_bool_value"
     VALUE_KEYS.update({STRING_VALUE, INT_VALUE, FLOAT_VALUE, BOOL_VALUE, PARAMETER_NAME})
 
+    DATASET_FOLDER = "Dataset folder"
     # list of compatible version of dataset info (V0, V1...)
     compatible_dataset_info_version = [VERSION_V0]
 
+
     def check_dataset_info_version(self, dataset_identifier: AbstractDatasetInfo) -> None:
         """
         Check that the version of the dataset info is compatible with the version of the dataset Connector

sostrades_core/datasets/datasets_connectors/arango_datasets_connector.py

@@ -74,7 +74,7 @@ def __init__(self, connector_id: str, host: str, db_name: str, username: str, pa
             self.db = client.db(name=db_name, username=username, password=password)
 
             if not self.db.has_collection(name=datasets_descriptor_collection_name):
-                raise Exception(f"Expected to find collection {datasets_descriptor_collection_name} describing datasets")
+                raise DatasetGenericException(f"Expected to find collection {datasets_descriptor_collection_name} describing datasets")
             self.datasets_descriptor_collection_name = datasets_descriptor_collection_name
 
         except Exception as exc:

sostrades_core/datasets/datasets_connectors/bigquery_datasets_connector.py

@@ -246,7 +246,7 @@ def _write_dataset(
                 # if dataset doesn't exists, create the dataset
                 # Construct a full Dataset object to send to the API.
                 dataset = bigquery.Dataset(dataset_id)
-                # TODO: Specify the geographic location where the dataset should reside.
+                # specify the geographic location where the dataset should reside.
                 dataset.location = "europe-west1"
 
                 # Send the dataset to the API for creation, with an explicit timeout.
@@ -374,7 +374,7 @@ def __load_descriptor_index_tables(self, dataset_id: str) -> tuple[dict, dict, s
         table_descriptor_id = "{}.{}".format(dataset_id, self.DESCRIPTOR_TABLE_NAME)
         table_descriptor_exists = True
         try:
-             table = self.client.get_table(table_descriptor_id)  # Make an API request.
+            _ = self.client.get_table(table_descriptor_id)  # Make an API request.
         except:
             table_descriptor_exists = False
             self.__logger.debug(f"create table for descriptor:{table_descriptor_id}")
@@ -390,7 +390,7 @@ def __load_descriptor_index_tables(self, dataset_id: str) -> tuple[dict, dict, s
         table_index_id = "{}.{}".format(dataset_id, self.COL_NAME_INDEX_TABLE_NAME)
         table_index_exists = True
         try:
-             table = self.client.get_table(table_index_id)  # Make an API request.
+            _ = self.client.get_table(table_index_id)  # Make an API request.
         except:
             table_index_exists = False
             self.__logger.debug(f"create table for descriptor:{table_index_id}")

sostrades_core/datasets/datasets_connectors/local_filesystem_datasets_connector/local_filesystem_datasets_connectorV0.py

@@ -86,7 +86,7 @@ def _get_values(self, dataset_identifier: DatasetInfoV0, data_to_get: dict[str,
         dataset_directory = os.path.join(self._root_directory_path, dataset_identifier.dataset_id)
         dataset_descriptor_path = os.path.join(dataset_directory, self.DESCRIPTOR_FILE_NAME)
 
-        self._datasets_serializer.check_path_exists("Dataset folder", dataset_directory)
+        self._datasets_serializer.check_path_exists(self.DATASET_FOLDER, dataset_directory)
 
         self._datasets_serializer.set_dataset_directory(dataset_directory)
 
@@ -133,7 +133,7 @@ def _write_values(self, dataset_identifier: DatasetInfoV0, values_to_write: dict
         dataset_directory = os.path.join(self._root_directory_path, dataset_identifier.dataset_id)
         dataset_descriptor_path = os.path.join(dataset_directory, self.DESCRIPTOR_FILE_NAME)
 
-        self._datasets_serializer.check_path_exists("Dataset folder", dataset_directory)
+        self._datasets_serializer.check_path_exists(self.DATASET_FOLDER, dataset_directory)
 
         self._datasets_serializer.set_dataset_directory(dataset_directory)
 
@@ -169,7 +169,7 @@ def _get_values_all(self, dataset_identifier: DatasetInfoV0, data_types_dict: di
         dataset_directory = os.path.join(self._root_directory_path, dataset_identifier.dataset_id)
         dataset_descriptor_path = os.path.join(dataset_directory, self.DESCRIPTOR_FILE_NAME)
 
-        self._datasets_serializer.check_path_exists("Dataset folder", dataset_directory)
+        self._datasets_serializer.check_path_exists(self.DATASET_FOLDER, dataset_directory)
 
         self._datasets_serializer.set_dataset_directory(dataset_directory)
 

sostrades_core/datasets/datasets_connectors/local_filesystem_datasets_connector/local_filesystem_datasets_connectorV1.py

@@ -86,7 +86,7 @@ def __load_dataset_descriptor(self, dataset_id: str) -> dict[str, dict[str, Any]
         dataset_directory = self.__build_dataset_path(dataset_id)
         dataset_descriptor_path = self.__build_descriptor_file_path(dataset_id)
 
-        self._datasets_serializer.check_path_exists("Dataset folder", dataset_directory)
+        self._datasets_serializer.check_path_exists(self.DATASET_FOLDER, dataset_directory)
 
         # Load the descriptor, the serializer loads the pickle if it exists
         return self._datasets_serializer.read_descriptor_file(dataset_id, dataset_descriptor_path)

sostrades_core/datasets/datasets_connectors/sospickle_datasets_connector.py

@@ -190,7 +190,7 @@ def _get_values_all(self, dataset_identifier: AbstractDatasetInfo) -> dict[str,
 
         dataset_keys = []
         for key in self.__pickle_data:
-            dataset_id, data_name = self.__get_dataset_id_and_data_name(key)
+            dataset_id, _ = self.__get_dataset_id_and_data_name(key)
             if dataset_id == dataset_identifier.dataset_id:
                 dataset_keys.append(key)
 

sostrades_core/datasets/datasets_serializers/filesystem_datasets_serializer.py

@@ -66,7 +66,6 @@ class FileSystemDatasetsSerializer(JSONDatasetsSerializer):
     """
     Specific dataset serializer for dataset
     """
-    # TYPE_IN_FILESYSTEM_PREFIXES = ['@dataframe@', '@array@']  # TODO: discuss
     TYPE_IN_FILESYSTEM_PARTICLE = '@'
     TYPE_DATAFRAME = 'dataframe'
     TYPE_ARRAY = 'array'
@@ -254,7 +253,6 @@ def __clean_from_pickle_data(self, data_name: str) -> None:
         Args:
             data_name (str): The name of the data to clear.
         """
-        # TODO: [discuss] is this necessary ?
         if data_name in self.__pickle_data:
             del self.__pickle_data[data_name]
 
@@ -276,7 +274,7 @@ def convert_from_dataset_data(self, data_name: str, data_value: Any, data_types_
         if data_name in data_types_dict:
             # unknown data type is handled in mother class method
             data_type = data_types_dict[data_name]
-            # TODO[discuss]: when the data is in the pickle pre-fill with @object@ and will be overwritten by connector
+            # when the data is in the pickle, pre-fill with @object@ and will be overwritten by connector
             if filesystem_type == self.TYPE_OBJECT:
                 self.__logger.debug(f"{data_name} with filesystem descriptor {data_value} is loaded from"
                                     f"non-serializable types pickle file.")

sostrades_core/datasets_database/datasets/default_numerical_parameters/descriptor.json

@@ -6,12 +6,12 @@
     "value": ""
   },
   "cache_type": {
-    "value": "None"
+    "value": ""
   },
   "cache_file_path": {
     "value": ""
   },
-  "sub_mda_class": {
+  "inner_mda_name": {
     "value": "MDAJacobi"
   },
   "max_mda_iter": {

sostrades_core/execution_engine/archi_builder.py

@@ -119,6 +119,9 @@ def __init__(self, sos_name, ee, architecture_df, cls_builder=None, associated_n
         self.get_children_list_by_vb(self.builder_dict)
 
     def prepare_execution(self):
+        """
+        Purely a configuration discipline it does not prepare execution.
+        """
         pass
 
     @property
@@ -180,7 +183,7 @@ def check_architecture(self, archi_df):
                 f'The architecture dataframe must have 5 columns named : {self.ARCHI_COLUMNS}'
             )
 
-        if not archi_df[self.ACTIVATION].dtype == 'bool':
+        if archi_df[self.ACTIVATION].dtype != 'bool':
             raise ArchiBuilderException(
                 'The architecture dataframe must contains bouleans in Activation column'
             )
@@ -285,12 +288,6 @@ def check_activation_df(self):
                                 ~modified_activation_df[colname], children_names
                             ] = False
 
-            # if len(rows_to_delete) > 0:
-            #     # remove rows with values not among possible_values
-            #     self.get_data_in()[self.ACTIVATION_DF][self.VALUE] = modified_activation_df.drop(
-            #         rows_to_delete
-            #     )
-
     def get_children_names(self, parent_name, architecture):
         """
         Recursive method to get children names for parent name by reading architecture_df
@@ -548,7 +545,7 @@ def build_action_from_builder_dict(self, builder_dict, archi_df):
                     builder_name = namespace
                     if not builder.sos_name.endswith('@archi_node'):
                         old_builder_name = builder.sos_name
-                        builder.set_disc_name(f'{builder.sos_name}@archi_node')
+                        builder.set_disc_name(f'{old_builder_name}@archi_node')
                         # self.ee.ns_manager.add_display_ns_to_builder(builder, old_builder_name)
                 action, args = self.get_action_builder(namespace, archi_df)
 
@@ -608,7 +605,7 @@ def build_action_from_builder_dict(self, builder_dict, archi_df):
 
                             if isinstance(args[1], tuple):
                                 # get builder of scatter of scatter
-                                raise Exception('Nested scatters in architecture builder not implemented')
+                                raise ArchiBuilderException('Nested scatters in architecture builder not implemented')
                             else:
                                 # get builder of scatter
                                 scatter_builder = self.get_builder_from_factory(
@@ -759,8 +756,6 @@ def send_children_to_father(self):
         attribute children_list
         """
         for ns, disc_list in self.archi_disciplines.items():
-            # scatter_in_node = False
-            # scattered_disciplines = {}
             for disc in disc_list:
                 disc_children_list = [
                     child
@@ -776,10 +771,10 @@ def setup_sos_disciplines(self):
         """
         Set samples_df value by reading activation_df input
         """
-        dynamic_outputs = {}
+        dynamic_inputs = {}
         for driver_name, input_name in self.driver_input_to_fill.items():
 
-            if f'{driver_name}.samples_df' in self.get_data_out():
+            if f'{driver_name}.samples_df' in self.get_data_in():
                 activation_df = deepcopy(self.get_sosdisc_inputs(self.ACTIVATION_DF))
 
                 if driver_name == 'driver':
@@ -806,20 +801,24 @@ def setup_sos_disciplines(self):
                 indexes = np.unique(input_value, return_index=True)[1]
                 input_value = [input_value[index] for index in sorted(indexes)]
 
-                scenario_full_name = self.get_var_full_name(f'{driver_name}.samples_df', self.get_data_out())
+                scenario_full_name = self.get_var_full_name(f'{driver_name}.samples_df', self.get_data_in())
                 self.dm.set_data(scenario_full_name, 'value', pd.DataFrame({self.SCENARIO_NAME: input_value,
                                                                             self.SELECTED_SCENARIO: True}),
                                  check_value=False)
-            dynamic_outputs.update(
+            dynamic_inputs.update(
                 {f'{driver_name}.samples_df': {'type': 'dataframe',
                                                'default': pd.DataFrame(
-                                                   columns=(self.SCENARIO_NAME, self.SELECTED_SCENARIO))},
+                                                   columns=(self.SCENARIO_NAME, self.SELECTED_SCENARIO)),
+                                               'structuring': True,
+                                               'editable': False
+                                               }
+
                  # f'{driver_name}.builder_mode': {'type': 'string',
                  #                                 'value': 'multi_instance'}
                  },
             )
 
-        self.add_outputs(dynamic_outputs)
+        self.add_inputs(dynamic_inputs, clean_inputs=False)
 
     def get_scatter_builder(
             self,
@@ -845,8 +844,6 @@ def get_scatter_builder(
             builder_name = namespace
             driver_name = f'{builder_name}.driver'
 
-        # TODO: check it is OK to always go multi-instance
-        # FIXME: issue with the treeviews...
         if isinstance(builder, list):
             builder_scatter = self.ee.factory.create_multi_instance_driver(driver_name, builder)
         else:
@@ -863,7 +860,6 @@ def get_scatter_builder(
             self.ee.ns_manager.add_display_ns_to_builder(
                 builder_scatter[0], self.get_disc_full_name().replace(self.sos_name, namespace))
         else:
-            # builder_name = f"{namespace.split('.', 1)[1]}"
             self.ee.ns_manager.add_display_ns_to_builder(
                 builder_scatter[0], f'{self.get_disc_full_name()}.{builder_name}')
         result_builder_list.extend(builder_scatter)
@@ -1091,7 +1087,7 @@ def remove_discipline_list(self, disc_list):
         for disc in disc_list:
             #             if isinstance(disc, SoSDisciplineScatter):
             #                 disc.clean_scattered_disciplines([])
-            self.ee.root_process.sos_disciplines.remove(disc)
+            self.ee.root_process.proxy_disciplines.remove(disc)
             disc.clean_dm_from_disc()
             self.ee.ns_manager.remove_dependencies_after_disc_deletion(
                 disc, self.disc_id