Merge pull request #890 from Ultimaker/CURA-10722_plugins_can_has_set…

…tings

[CURA-10722] Add 'additional settings providers' (Engine Plugins)

UM/Settings/AdditionalSettingDefinitionAppender.py

@@ -0,0 +1,133 @@
+# Copyright (c) 2023 UltiMaker.
+# Uranium is released under the terms of the LGPLv3 or higher.
+
+import json
+from pathlib import Path
+import os.path
+from typing import Any, Dict, List
+
+from UM.Logger import Logger
+from UM.PluginObject import PluginObject
+from UM.Settings.SettingDefinition import DefinitionPropertyType, SettingDefinition
+
+
+class AdditionalSettingDefinitionsAppender(PluginObject):
+    """
+    This class is a way for plugins to append additional settings, not defined by/for the main program itself.
+
+    Each plugin needs to register as either a 'setting_definitions_appender' or 'backend_plugin'.
+
+    Any implementation also needs to fill 'self.definition_file_paths' with a list of files with setting definitions.
+    Each file should be a json list of setting categories, either matching existing names, or be a new category.
+    Each category and setting has the same json structure as the main settings otherwise.
+
+    It's also possible to set the 'self.appender_type', if there are many kinds of plugins to implement this,
+    in order to prevent name-clashes.
+
+    Lastly, if setting-definitions are to be made on the fly by the plugin, override 'getAdditionalSettingDefinitions',
+    instead of providing the files. This should then return a dict, as if parsed by json.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.appender_type = "PLUGIN"
+        self.definition_file_paths: List[Path] = []
+
+    def getAppenderType(self) -> str:
+        """
+        Return an extra identifier prepended to the setting internal id, to prevent name-clashes with other plugins.
+        """
+        return self.appender_type
+
+    def getAdditionalSettingDefinitions(self) -> Dict[str, Dict[str, Any]]:
+        """
+        Return the settings added by this plugin in json format.
+        Put values in self.definition_file_paths if you wish to load from files, or override this function otherwise.
+
+        The settings should be divided by category (either existing or new ones).
+        Settings in existing categories will be appended, new categories will be created.
+
+        Setting names (not labels) will be post-processed ('mangled') internally to prevent name-clashes.
+        NOTE: The 'mangled' names are also the ones send out to any backend!
+        (See the _prependIdToSettings function below for a more precise explanation.)
+
+        :return: Dictionary of settings-categories, containing settings-definitions (with post-processed names).
+        """
+        result = {}
+        for path in self.definition_file_paths:
+            if not os.path.exists(path):
+                Logger.error(f"File {path} with additional settings for '{self.getId()}' doesn't exist.")
+                continue
+            try:
+                with open(path, "r", encoding = "utf-8") as definitions_file:
+                    result.update(json.load(definitions_file))
+            except OSError as oex:
+                Logger.error(f"Could not read additional settings file for '{self.getId()}' because: {str(oex)}")
+                continue
+            except json.JSONDecodeError as jex:
+                Logger.error(f"Could not parse additional settings provided by '{self.getId()}' because: {str(jex)}")
+                continue
+        return self._prependIdToSettings(result)
+
+    def _prependIdToSettings(self, settings: Dict[str, Any]) -> Dict[str, Any]:
+        """ This takes the whole (extra) settings-map as defined by the provider, and returns a tag-renamed version.
+
+        Additional (appended) settings will need to be prepended with (an) extra identifier(s)/namespaces to not collide.
+        This is done for when there are multiple additional settings appenders that might not know about each other.
+        This includes any formulas, which will also be included in the renaming process.
+
+        Appended settings may not be the same as 'baseline' (so any 'non-appended' settings) settings.
+        (But may of course clash between different providers and versions, that's the whole point of this function...)
+        Furthermore, it's assumed that formulas within the appended settings will only use settings either;
+         - as defined within the baseline, or;
+         - any other settings defined _by the provider itself_.
+
+        For each key that is renamed, this results in a mapping <key> -> _<provider_type>__<id*>__<version>__<key>
+         where '<id*>' is the version of the provider, but converted from using points to using underscores.
+        Example: 'tapdance_factor' might become '_plugin__dancingprinter__1_2_99__tapdance_factor'
+        Also note that all the tag_... parameters will be forced to lower-case.
+
+        :param tag_type: Type of the additional settings appender, for example; "PLUGIN".
+        :param tag_id: ID of the provider. Should be unique.
+        :param tag_version: Version of the provider. Points will be replaced by underscores.
+        :param settings: The settings as originally provided.
+
+        :returns: Remapped settings, where each settings-name is properly tagged/'namespaced'.
+        """
+        tag_type = self.getAppenderType().lower()
+        tag_id = self.getId().lower()
+        tag_version = self.getVersion().lower().replace(".", "_")
+
+        # First get the mapping, so that both the 'headings' and formula's can be renamed at the same time later.
+        def _getMapping(values: Dict[str, Any]) -> Dict[str, str]:
+            result = {}
+            for key, value in values.items():
+                mapped_key = key
+                if isinstance(value, dict):
+                    if "type" in value and value["type"] != "category":
+                        mapped_key = f"_{tag_type}__{tag_id}__{tag_version}__{key}"
+                    result.update(_getMapping(value))
+                result[key] = mapped_key
+            return result
+
+        key_map = _getMapping(settings)
+
+        # Get all values that can be functions, so it's known where to replace.
+        function_type_names = set(SettingDefinition.getPropertyNames(DefinitionPropertyType.Function))
+
+        # Replace all, both as key-names and their use in formulas.
+        def _doReplace(values: Dict[str, Any]) -> Dict[str, str]:
+            result = {}
+            for key, value in values.items():
+                if key in function_type_names and isinstance(value, str):
+                    # Replace key-names in the specified settings-function.
+                    for original, mapped in key_map.items():
+                        value = value.replace(original, mapped)
+                elif isinstance(value, dict):
+                    # Replace key-name 'heading'.
+                    key = key_map.get(key, key)
+                    value = _doReplace(value)
+                result[key] = value
+            return result
+
+        return _doReplace(settings)

UM/Settings/ContainerRegistry.py

@@ -17,6 +17,7 @@
 from UM.Settings.EmptyInstanceContainer import EmptyInstanceContainer
 from UM.Settings.ContainerFormatError import ContainerFormatError
 from UM.Settings.ContainerProvider import ContainerProvider
+from UM.Settings.AdditionalSettingDefinitionAppender import AdditionalSettingDefinitionsAppender
 from UM.Settings.constant_instance_containers import empty_container
 from . import ContainerQuery
 from UM.Settings.ContainerStack import ContainerStack
@@ -59,6 +60,9 @@ def __init__(self, application: "QtApplication") -> None:
         self._providers = []  # type: List[ContainerProvider]
         PluginRegistry.addType("container_provider", self.addProvider)
 
+        self._additional_setting_definitions_list: List[Dict[str, Dict[str, Any]]] = []
+        PluginRegistry.addType("setting_definitions_appender", self.addAdditionalSettingDefinitionsAppender)
+
         self.metadata = {}  # type: Dict[str, metadata_type]
         self._containers = {}  # type: Dict[str, ContainerInterface]
         self._wrong_container_ids = set() # type: Set[str]  # Set of already known wrong containers that must be skipped
@@ -115,6 +119,11 @@ def addProvider(self, provider: ContainerProvider) -> None:
         # Re-sort every time. It's quadratic, but there shouldn't be that many providers anyway...
         self._providers.sort(key = lambda provider: PluginRegistry.getInstance().getMetaData(provider.getPluginId())["container_provider"].get("priority", 0))
 
+    def addAdditionalSettingDefinitionsAppender(self, appender: AdditionalSettingDefinitionsAppender) -> None:
+        """Adds a provider for additional setting definitions to append to each definition-container."""
+
+        self._additional_setting_definitions_list.append(appender.getAdditionalSettingDefinitions())
+
     def findDefinitionContainers(self, **kwargs: Any) -> List[DefinitionContainerInterface]:
         """Find all DefinitionContainer objects matching certain criteria.
 
@@ -607,6 +616,12 @@ def addContainer(self, container: ContainerInterface) -> bool:
             self.source_provider[container_id] = None  # Added during runtime.
         self._clearQueryCacheByContainer(container)
 
+        for additional_setting_definitions in self._additional_setting_definitions_list:
+            if container.getMetaDataEntry("type") == "extruder" or not isinstance(container, DefinitionContainer):
+                continue
+            container = cast(DefinitionContainer, container)
+            container.appendAdditionalSettingDefinitions(additional_setting_definitions)
+
         # containerAdded is a custom signal and can trigger direct calls to its subscribers. This should be avoided
         # because with the direct calls, the subscribers need to know everything about what it tries to do to avoid
         # triggering this signal again, which eventually can end up exceeding the max recursion limit.

UM/Settings/DefinitionContainer.py

@@ -332,16 +332,60 @@ def deserialize(self, serialized: str, file_name: Optional[str] = None) -> str:
         self._metadata["version"] = self.Version #Guaranteed to be equal to what's in the parsed data by the validation.
         self._metadata["container_type"] = DefinitionContainer
 
-        for key, value in parsed["settings"].items():
-            definition = SettingDefinition(key, self, None, self._i18n_catalog)
+        self._deserializeDefinitions(parsed["settings"])
+        return serialized
+
+    def _deserializeDefinitions(self, settings_dict: Dict[str, Any], force_category: Optional[str] = None) -> None:
+
+        # When there is a forced category (= parent) present, find the category parent, create it if it doesn't exist.
+        category_parent = None
+        if force_category:
+            category_parent = self.findDefinitions(key = force_category)
+            category_parent = category_parent[0] if len(category_parent) > 0 else None
+
+        for key, value in settings_dict.items():
+            definition = SettingDefinition(key, self, category_parent, self._i18n_catalog)
             self._definition_cache[key] = definition
             definition.deserialize(value)
-            self._definitions.append(definition)
+            if category_parent:
+                # Forced category; these are then children of that category, instead of full categories on their own.
+                category_parent.children.append(definition)
+            else:
+                self._definitions.append(definition)
 
         for definition in self._definitions:
             self._updateRelations(definition)
 
-        return serialized
+    def appendAdditionalSettingDefinitions(self, additional_settings: Dict[str, Dict[str, Any]]) -> None:
+        """
+        Appends setting-definitions not defined for/by the main program (for example, a plugin) to this container.
+
+        Additional settings are always assumed to come in the form of categories with child-settings.
+        See also the Settings.AdditionalSettingDefinitionAppender class.
+
+        :param additional_settings: A dictionary of category-name to categories, each containing setting-definitions.
+        """
+        try:
+            merge_with_existing_categories = {}
+            create_new_categories = {}
+
+            for category, values in additional_settings.items():
+                if len(self.findDefinitions(key = category)) > 0:
+                    merge_with_existing_categories[category] = values
+                else:
+                    create_new_categories[category] = values
+
+            if len(create_new_categories) > 0:
+                self._deserializeDefinitions(create_new_categories)
+            for category, values in merge_with_existing_categories.items():
+                if "children" in values:
+                    for key, value in values["children"].items():
+                        self._deserializeDefinitions({key: value}, category)
+                else:
+                    self._deserializeDefinitions(values, category)
+
+        except Exception as ex:
+            Logger.error(f"Failed to append additional settings from external source because: {str(ex)}")
 
     @classmethod
     def deserializeMetadata(cls, serialized: str, container_id: str) -> List[Dict[str, Any]]:
@@ -429,6 +473,7 @@ def _resolveInheritance(self, file_name: str) -> Dict[str, Any]:
         json_dict = self._loadFile(file_name)
 
         if "inherits" in json_dict:
+            # NOTE: Since load-file isn't cached, this will load base definitions multiple times!
             inherited = self._resolveInheritance(json_dict["inherits"])
             json_dict = self._mergeDicts(inherited, json_dict)
 

tests/Settings/TestAppendAdditionalSettings.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2023 UltiMaker
+# Uranium is released under the terms of the LGPLv3 or higher.
+
+import os
+import pytest
+from unittest.mock import MagicMock, patch
+
+from UM.Settings.AdditionalSettingDefinitionAppender import AdditionalSettingDefinitionsAppender
+from UM.Settings.DefinitionContainer import DefinitionContainer
+from UM.VersionUpgradeManager import VersionUpgradeManager
+
+
+class PluginTestClass(AdditionalSettingDefinitionsAppender):
+    def __init__(self) -> None:
+        super().__init__()
+        self._plugin_id = "RealityPerforator"
+        self._version = "7.8.9"
+        self.appender_type = "CLOWNS"
+        self.definition_file_paths = [os.path.join(os.path.dirname(os.path.abspath(__file__)), "additional_settings", "append_extra_settings.def.json")]
+
+
+def test_AdditionalSettingNames():
+    plugin = PluginTestClass()
+    settings = plugin.getAdditionalSettingDefinitions()
+
+    assert "test_setting" in settings
+    assert "category_too" in settings
+    assert "children" in settings["test_setting"]
+    assert "children" in settings["category_too"]
+
+    assert "_clowns__realityperforator__7_8_9__glombump" in settings["test_setting"]["children"]
+    assert "_clowns__realityperforator__7_8_9__zharbler" in settings["category_too"]["children"]
+
+
+def test_AdditionalSettingContainer(upgrade_manager: VersionUpgradeManager):
+    plugin = PluginTestClass()
+    settings = plugin.getAdditionalSettingDefinitions()
+
+    definition_container = DefinitionContainer("TheSunIsADeadlyLazer")
+    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "definitions", "children.def.json"), encoding = "utf-8") as data:
+        definition_container.deserialize(data.read())
+    definition_container.appendAdditionalSettingDefinitions(settings)
+
+    # 'merged' setting-categories should definitely be in the relevant container (as well as the original ones):
+    assert len(definition_container.findDefinitions(key="test_setting")) == 1
+    kid_keys = [x.key for x in definition_container.findDefinitions(key="test_setting")[0].children]
+    assert "test_child_0" in kid_keys
+    assert "test_child_1" in kid_keys
+    assert "_clowns__realityperforator__7_8_9__glombump" in kid_keys
+
+    # other settings (from new categories) are added 'dry' to the container:
+    assert "_clowns__realityperforator__7_8_9__zharbler" in definition_container.getAllKeys()

tests/Settings/additional_settings/append_extra_settings.def.json

@@ -0,0 +1,44 @@
+{
+  "test_setting":
+  {
+    "children":
+    {
+      "glombump":
+      {
+        "label": "Snosmozea",
+        "description": "Sudoriferous.",
+        "unit": "mm/s",
+        "type": "float",
+        "minimum_value": "0.1",
+        "maximum_value_warning": "150",
+        "maximum_value": "500",
+        "default_value": 60,
+        "settable_per_mesh": true
+      }
+    }
+  },
+  "category_too":
+  {
+    "label": "Warble!",
+    "type": "category",
+    "description": "Blomblimg",
+    "icon": "Printer",
+    "children":
+    {
+      "zharbler":
+      {
+        "label": "Frumg",
+        "description": "Pleonastic Pellerator.",
+        "unit": "mm/s",
+        "type": "float",
+        "minimum_value": "0.1",
+        "maximum_value_warning": "150",
+        "maximum_value": "500",
+        "default_value": 60,
+        "value": "glombump * 1.2",
+        "settable_per_mesh": false
+      }
+    }
+  }
+}
+

tests/TestTrust.py

@@ -1,3 +1,6 @@
+# Copyright (c) 2023 UltiMaker
+# Uranium is released under the terms of the LGPLv3 or higher.
+
 import copy
 import json
 from unittest.mock import MagicMock, patch