#2072: add configuration option to configure pre-defined "extraFields"

base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java

@@ -527,6 +527,41 @@ public enum DittoHeaderDefinition implements HeaderDefinition {
             JsonObject.class,
             false,
             true,
+            HeaderValueValidators.getJsonObjectValidator()),
+
+    /**
+     * Internal header containing the pre-defined configured {@code extraFields} as list of jsonPointers for the
+     * emitted thing event.
+     *
+     * @since 3.7.0
+     */
+    PRE_DEFINED_EXTRA_FIELDS("ditto-pre-defined-extra-fields",
+            JsonArray.class,
+            false,
+            false,
+            HeaderValueValidators.getJsonArrayValidator()),
+
+    /**
+     * Internal header containing the pre-defined configured {@code extraFields} as keys and the allowed "read subjects"
+     * as array of stings - defining which "auth subjects" are allowed to read which pre-defined extra field.
+     *
+     * @since 3.7.0
+     */
+    PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT("ditto-pre-defined-extra-fields-read-grant",
+            JsonObject.class,
+            false,
+            false,
+            HeaderValueValidators.getJsonObjectValidator()),
+
+    /**
+     * Internal header containing pre-defined {@code extraFields} as JSON object sent along for emitted thing event.
+     *
+     * @since 3.7.0
+     */
+    PRE_DEFINED_EXTRA_FIELDS_OBJECT("ditto-pre-defined-extra-fields-object",
+            JsonObject.class,
+            false,
+            false,
             HeaderValueValidators.getJsonObjectValidator());
 
     /**

base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java

@@ -137,6 +137,16 @@ public final class ImmutableDittoHeadersTest {
             .set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar")
             .build();
 
+    private static final JsonArray KNOWN_PRE_DEFINED_EXTRA_FIELDS = JsonArray.newBuilder()
+            .add("foo:bar:123")
+            .build();
+    private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT = JsonObject.newBuilder()
+            .set("/definition", "known:subject")
+            .build();
+    private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT = JsonObject.newBuilder()
+            .set("definition", "foo:bar:123")
+            .build();
+
 
     static {
         KNOWN_METADATA_HEADERS = MetadataHeaders.newInstance();
@@ -205,6 +215,12 @@ public void settingAllKnownHeadersWorksAsExpected() {
                 .putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION))
                 .putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP))
                 .putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString())
+                .putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
+                        KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString())
+                .putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
+                        KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString())
+                .putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
+                        KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString())
                 .build();
 
         assertThat(underTest).isEqualTo(expectedHeaderMap);
@@ -535,6 +551,11 @@ public void toJsonReturnsExpected() {
                 .set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION)
                 .set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString())
                 .set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS)
+                .set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(), KNOWN_PRE_DEFINED_EXTRA_FIELDS)
+                .set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
+                        KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT)
+                .set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
+                        KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT)
                 .build();
 
         final Map<String, String> allKnownHeaders = createMapContainingAllKnownHeaders();
@@ -774,6 +795,12 @@ private static Map<String, String> createMapContainingAllKnownHeaders() {
         result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION));
         result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP));
         result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString());
+        result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
+                KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString());
+        result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
+                KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString());
+        result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
+                KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString());
 
         return result;
     }

bom/pom.xml

@@ -27,7 +27,7 @@
 
     <properties>
         <scala.version>2.13</scala.version> <!-- for scala libraries the scala version is used in their artifactId -->
-        <scala.full.version>2.13.15</scala.full.version>
+        <scala.full.version>2.13.16</scala.full.version>
         <scala-parser-combinators.version>1.1.2</scala-parser-combinators.version>
         <scala-java8-compat.version>1.0.2</scala-java8-compat.version>
 

deployment/helm/ditto/Chart.yaml

@@ -16,7 +16,7 @@ description: |
   A digital twin is a virtual, cloud based, representation of his real world counterpart
   (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
 type: application
-version: 3.6.9  # chart version is effectively set by release-job
+version: 3.6.10  # chart version is effectively set by release-job
 appVersion: 3.6.9
 keywords:
   - iot-chart

deployment/helm/ditto/service-config/things-extension.conf.tpl

@@ -44,6 +44,46 @@ ditto {
           "{{$header}}"
         {{- end }}
         ]
+        pre-defined-extra-fields = [
+        {{- range $index, $extraFieldConfig := .Values.things.config.event.preDefinedExtraFields }}
+          {
+            namespaces = [
+            {{- range $index, $namespace := $extraFieldConfig.namespaces }}
+              "{{$namespace}}"
+            {{- end }}
+            ]
+            {{- if $extraFieldConfig.condition }}
+            condition = "{{$extraFieldConfig.condition}}"
+            {{- end }}
+            extra-fields = [
+            {{- range $index, $extraField := $extraFieldConfig.extraFields }}
+              "{{$extraField}}"
+            {{- end }}
+            ]
+          }
+        {{- end }}
+        ]
+      }
+      message {
+        pre-defined-extra-fields = [
+        {{- range $index, $extraFieldConfig := .Values.things.config.message.preDefinedExtraFields }}
+          {
+            namespaces = [
+            {{- range $index, $namespace := $extraFieldConfig.namespaces }}
+              "{{$namespace}}"
+            {{- end }}
+            ]
+            {{- if $extraFieldConfig.condition }}
+            condition = "{{$extraFieldConfig.condition}}"
+            {{- end }}
+            extra-fields = [
+            {{- range $index, $extraField := $extraFieldConfig.extraFields }}
+              "{{$extraField}}"
+            {{- end }}
+            ]
+          }
+        {{- end }}
+        ]
       }
     }
 

deployment/helm/ditto/values.yaml

@@ -1059,6 +1059,27 @@ things:
         interval: 15m
         # the threshold after how many changes to a Thing to do a snapshot
         threshold: 50
+    # event contains configuration related to e.g. publishing of thing events
+    event:
+      # preDefinedExtraFields contains pre-defined (configured) extraFields to send along all thing (change) events
+      preDefinedExtraFields: []
+      # - namespaces:
+      #     - "org.eclipse.ditto*"
+      #   condition: "eq(attributes/foo,'RQL condition')"
+      #   extraFields:
+      #     - "definition"
+      #     - "attributes/serial"
+    # message contains configuration related to distributing thing messages
+    message:
+      # preDefinedExtraFields contains pre-defined (configured) extraFields to send along all thing messages
+      preDefinedExtraFields: []
+      # - namespaces:
+      #     - "namespace1"
+      #     - "namespace2"
+      #   condition: "eq(attributes/foo,'RQL condition')"
+      #   extra-fields:
+      #     - "definition"
+      #     - "attributes/serial"
     # entityCreation by default, Ditto allows anyone to create a new entity (thing in this case) in any namespace.
     #  However, this behavior can be customized, and the ability to create new entities can be restricted:
     entityCreation:

documentation/src/main/resources/pages/ditto/installation-operating.md

@@ -385,6 +385,86 @@ entities (things/policies) and no-one other:
 
 These system properties would have to be configured for the "things" and "policies" services.
 
+## Configuring pre-defined extra fields
+
+Starting with Ditto 3.7.0, it is possible to configure [enrichment of `extraFields`](basic-enrichment.html) statically 
+via the configuration of the Ditto "things" service.
+
+The benefit of doing this statically is a Ditto internal optimization to reduce internal traffic between Ditto services.  
+By default, Ditto will internally do an additional roundtrip from an "edge" service ("gateway" or "connectivity") to the
+"things" service in order to retrieve configured `extraFields` (of a [managed connection](basic-connections.html#target-topics-and-enrichment) or 
+e.g. a [WebSocket session](httpapi-protocol-bindings-websocket.html#enrichment)).
+
+Those retrieved `extraFields` are additionally cached, so also require some memory as well.
+
+If for a Ditto installation the `extraFields` are known upfront and will not change dynamically, it is possible to configure
+them in the [things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf).
+
+This configuration is something for power operators of Ditto, needing to reduce resources and improving resiliency by
+reducing internal lookups.
+
+### Pre-defined extra fields configuration
+
+The configuration can be done for:
+* events: Thing Events emitted to subscriber
+* messages: Thing Messages forwarded by Ditto to message subscribers
+
+Available options:
+* `pre-defined-extra-fields`: a list of pre-defined extra fields configurations
+  * `namespaces`: a list of namespaces for which the configuration applies
+    * if this list is empty, the configuration applies to all namespaces
+    * the entries support wildcards (`*` matches any number of characters, `?` matches exactly one character)
+  * `condition`: a [RQL condition](basic-rql.html) to check if the extra fields should be added
+  * `extra-fields`: a list of extra fields (as JsonPointers) to proactively add for all matching `namespaces` and `condition` combinations
+
+Example configuration:
+```hocon
+ditto {
+  things {
+    thing {
+      event {
+        pre-defined-extra-fields = [
+          {
+            namespaces = []
+            condition = "exists(definition)"
+            extra-fields = [
+              "definition"
+            ]
+          },
+          {
+            namespaces = [
+              "org.eclipse.ditto.lamps"
+            ]
+            extra-fields = [
+              "attributes/manufacturer",
+              "attributes/serial"
+            ]
+          }
+        ]
+      }
+
+      message {
+        pre-defined-extra-fields = [
+          {
+            namespaces = []
+            condition = "exists(definition)"
+            extra-fields = [
+              "definition"
+            ]
+          }
+        ]
+    //...
+}
+```
+
+The above example configuration would always proactively send the `definition` of all Things (if it exists) in the published events
+and all messages.  
+If a consumer of events or messages is interested in this `extraField`, this would not lead to an additional internal 
+lookup in Ditto and save an internal roundtrip + caching of the result.
+
+For the namespace `org.eclipse.ditto.lamps` there would even be some defined attributes pre-defined to be available as
+`extraFields` without the need for another internal lookup.
+
 ## Limiting Indexed Fields
 
 The default behavior of Ditto is to index the complete JSON of a thing, which includes all its attributes and features.  This may not be desired behavior for certain use cases: