Update ScorpioBroker to the v4

Signed-off-by: Meric Feyzullahoglu <[email protected]>
IndustryFusion · May 31, 2024 · e4716f1 · e4716f1
1 parent 68995fa
commit e4716f1
Show file tree

Hide file tree

Showing 24 changed files with 118 additions and 49 deletions.
diff --git a/AllInOneRunner/pom.xml b/AllInOneRunner/pom.xml
@@ -9,17 +9,17 @@
 	<artifactId>all-in-one-runner</artifactId>
 	<name>AllInOneRunner</name>
 	<dependencies>
-		<dependency>
+		<!-- <dependency>
 			<groupId>eu.neclab.ngsildbroker</groupId>
 			<artifactId>at-context-server</artifactId>
 			<version>${parent.version}</version>
-		</dependency>
+		</dependency> -->
 		<dependency>
 			<groupId>eu.neclab.ngsildbroker</groupId>
 			<artifactId>entity-manager</artifactId>
 			<version>${parent.version}</version>
 		</dependency>
-		<dependency>
+		<!-- <dependency>
 			<groupId>eu.neclab.ngsildbroker</groupId>
 			<artifactId>history-entity-manager</artifactId>
 			<version>${parent.version}</version>
@@ -28,7 +28,7 @@
 			<groupId>eu.neclab.ngsildbroker</groupId>
 			<artifactId>history-query-manager</artifactId>
 			<version>${parent.version}</version>
-		</dependency>
+		</dependency> -->
 		<dependency>
 			<groupId>eu.neclab.ngsildbroker</groupId>
 			<artifactId>query-manager</artifactId>
@@ -75,7 +75,7 @@
 						</executions>
 						<configuration>
 							<dockerfile>src/main/resources/dockerfiles/dockerfile-${os}</dockerfile>
-							<repository>scorpiobroker/${project.artifactId}</repository>
+							<repository>ibn40/scorpio-${project.artifactId}</repository>
 							<tag>${docker-tag}</tag>
 							<buildArgs>
 								<FILE_BUILD>${project.build.finalName}-runner${nativeext}</FILE_BUILD>

diff --git a/AllInOneRunner/src/main/resources/application-kafka.properties b/AllInOneRunner/src/main/resources/application-kafka.properties
@@ -1,5 +1,5 @@
-mysettings.kafka.bootstrap.host=${bushost:localhost}
-mysettings.kafka.bootstrap.port=${busport:9092}
+mysettings.kafka.bootstrap.host=${KAFKA_BOOTSTRAP:localhost}
+mysettings.kafka.bootstrap.port=${KAFKA_PORT:9092}
 bootstrap.servers=${mysettings.kafka.bootstrap.host}:${mysettings.kafka.bootstrap.port}
 client-id-prefix=$[quarkus.uuid}
 group.id=$[quarkus.application.name}$[quarkus.uuid}
@@ -33,8 +33,8 @@ mp.messaging.incoming.entitybatchretrieve.auto.offset.reset=latest
 mp.messaging.incoming.entitybatchretrieve.broadcast=true
 #mp.messaging.incoming.entitybatchretrieve.value.deserializer=eu.neclab.ngsildbroker.commons.serialization.messaging.BatchRequestDeserializer
 #readability block###########.
-mp.messaging.outgoing.history.connector=smallrye-kafka
-mp.messaging.outgoing.history.topic=${scorpio.topics.temporal}
+# mp.messaging.outgoing.history.connector=smallrye-kafka
+# mp.messaging.outgoing.history.topic=${scorpio.topics.temporal}
 #readability block###########
 mp.messaging.outgoing.isubs.topic=${scorpio.topics.internalregsub}
 mp.messaging.outgoing.isubs.connector=smallrye-kafka
@@ -89,10 +89,10 @@ mp.messaging.incoming.regsubsyncretrieve.topic=${scorpio.topics.regsubsync}
 mp.messaging.incoming.regsubsyncretrieve.auto.offset.reset=latest
 mp.messaging.incoming.regsubsyncretrieve.broadcast=true
 #mp.messaging.incoming.regsubsyncretrieve.value.deserializer=eu.neclab.ngsildbroker.commons.serialization.messaging.SyncMessageDeserializer
-mp.messaging.outgoing.histsync.connector=smallrye-kafka
-mp.messaging.outgoing.histsync.topic=${scorpio.topics.histsync}
-#mp.messaging.outgoing.regsubsync.value.serializer=eu.neclab.ngsildbroker.commons.serialization.messaging.SyncMessageSerializer
-mp.messaging.incoming.histsyncretrieve.connector=smallrye-kafka
-mp.messaging.incoming.histsyncretrieve.topic=${scorpio.topics.histsync}
-mp.messaging.incoming.histsyncretrieve.auto.offset.reset=latest
-mp.messaging.incoming.histsyncretrieve.broadcast=true
+# mp.messaging.outgoing.histsync.connector=smallrye-kafka
+# mp.messaging.outgoing.histsync.topic=${scorpio.topics.histsync}
+# #mp.messaging.outgoing.regsubsync.value.serializer=eu.neclab.ngsildbroker.commons.serialization.messaging.SyncMessageSerializer
+# mp.messaging.incoming.histsyncretrieve.connector=smallrye-kafka
+# mp.messaging.incoming.histsyncretrieve.topic=${scorpio.topics.histsync}
+# mp.messaging.incoming.histsyncretrieve.auto.offset.reset=latest
+# mp.messaging.incoming.histsyncretrieve.broadcast=true
diff --git a/AllInOneRunner/src/main/resources/application.properties b/AllInOneRunner/src/main/resources/application.properties
@@ -12,14 +12,14 @@ quarkus.vertx.prefer-native-transport=true
 #quarkus.http.tcp-quick-ack=true
 #quarkus.http.tcp-cork=true
 #quarkus.http.tcp-fast-open=true
-mysettings.postgres.host=${dbhost:localhost}
+mysettings.postgres.host=${POSTGRES_SERVICE:localhost}
 mysettings.postgres.port=${dbport:5432}
 mysettings.postgres.username=${dbuser:ngb}
-mysettings.postgres.password=${dbpass:ngb}
-mysettings.postgres.database-name=${dbname:ngb}
+mysettings.postgres.password=${POSTGRES_PASSWORD:ngb}
+mysettings.postgres.database-name=${POSTGRES_DBNAME:ngb}
 mysettings.gateway.host=${gateway.host:localhost}
 mysettings.gateway.port=${gateway.port:9090}
-jdbcurl=jdbc:postgresql://${mysettings.postgres.host}:${mysettings.postgres.port}/${mysettings.postgres.database-name}?ApplicationName=${quarkus.application.name}
+jdbcurl=jdbc:postgresql://${mysettings.postgres.host}:${mysettings.postgres.port}/${mysettings.postgres.database-name}?ApplicationName=${quarkus.application.name}&sslmode=require
 scorpio.messaging.duplicate=true
 scorpio.gatewayurl=http://localhost:9090
 atcontext.url=http://localhost:9090/ngsi-ld/v1/jsonldContexts/
@@ -48,17 +48,17 @@ scorpio.registry.batch-operations.upsert.max=1000
 scorpio.registry.batch-operations.update.max=1000
 scorpio.registry.batch-operations.delete.max=1000
 scorpio.registry.batch-operations.query.max=1000
-scorpio.topics.entity=ENTITY
-scorpio.topics.entitybatch=ENTITYBATCH
-scorpio.topics.registry=REGISTRY
-scorpio.topics.temporal=TEMPORAL
-scorpio.topics.internalnotification=I_NOTIFY
-scorpio.topics.internalregsub=I_REGSUB
-scorpio.topics.subalive=SUB_ALIVE
-scorpio.topics.subsync=SUB_SYNC
-scorpio.topics.regsubalive=REG_SUB_ALIVE
-scorpio.topics.regsubsync=REG_SUB_SYNC
-scorpio.topics.histsync=HIST_SUB_SYNC
+scorpio.topics.entitybatch=${ENTITYBATCH_TOPIC:ENTITYBATCH}
+scorpio.topics.entity=${ENTITY_TOPIC:ENTITY}
+scorpio.topics.registry=${REGISTRY_TOPIC:REGISTRY}
+scorpio.topics.temporal=${TEMPORAL_TOPIC:TEMPORAL}
+scorpio.topics.internalnotification=${INTERNAL_NOTIFY_TOPIC:I_NOTIFY}
+scorpio.topics.internalregsub=${INTERNAL_REGISTRYSUB_TOPIC:I_REGSUB}
+scorpio.topics.subalive=${SUB_ALIVE_TOPIC:SUB_ALIVE}
+scorpio.topics.subsync=${SUB_SYNC_TOPIC:SUB_SYNC}
+scorpio.topics.regsubalive=${REGISTRYSUB_ALIVE_TOPIC:REG_SUB_ALIVE}
+scorpio.topics.regsubsync=${REGISTRYSUB_SYNC_TOPIC:REG_SUB_SYNC}
+# scorpio.topics.histsync=HIST_SUB_SYNC not used without history manager
 scorpio.entitymap.cleanup.ttl=30 sec
 scorpio.entitymap.cleanup.schedule=10s
 scorpio.startupdelay=5s
@@ -70,11 +70,12 @@ quarkus.datasource.db-kind=postgresql
 quarkus.datasource.username=${mysettings.postgres.username}
 quarkus.datasource.password=${mysettings.postgres.password}
 quarkus.datasource.jdbc.url=${jdbcurl}
-quarkus.datasource.reactive.url=postgresql://${mysettings.postgres.host}:${mysettings.postgres.port}/${mysettings.postgres.database-name}
+quarkus.datasource.reactive.url=postgresql://${mysettings.postgres.host}:${mysettings.postgres.port}/${mysettings.postgres.database-name}?sslmode=require
 quarkus.datasource.reactive.shared=true
 quarkus.datasource.reactive.cache-prepared-statements=true
 quarkus.datasource.reactive.max-size=50
 quarkus.datasource.reactive.idle-timeout=15s
+quarkus.datasource.reactive.trust-all=true
 #quarkus.datasource.reactive.event-loop-size=100
 quarkus.flyway.migrate-at-start=true
 quarkus.flyway.baseline-on-migrate=true
@@ -87,4 +88,8 @@ quarkus.rest-client.atcontext-service.url=http://localhost:9090
 scorpio.registry.subscription.checkinterval=30s
 scorpio.subscription.checkinterval=2s
 scorpio.fedupdaterate=600s
-quarkus.http.limits.max-initial-line-length=50000
+quarkus.http.limits.max-initial-line-length=50000
+#Keycloak
+quarkus.oidc.auth-server-url=${KEYCLOAK_SERVER_URL}realms/${REALM}
+quarkus.oidc.client-id=${CLIENT_ID}
+quarkus.oidc.tls.verification=none
diff --git a/AllInOneRunner/src/main/resources/db/migration/V20190604.1__entity.sql b/AllInOneRunner/src/main/resources/db/migration/V20190604.1__entity.sql
@@ -54,4 +54,6 @@ CREATE INDEX i_entity_operationspace ON entity USING GIST (operationspace);
 
 -- to check if this index will be used by the database optimizer, or if it should be applied only for for certain keys
 -- check https://www.postgresql.org/docs/current/static/datatype-json.html
-CREATE INDEX i_entity_data ON entity USING GIN (data); 
+CREATE INDEX i_entity_data ON entity USING GIN (data);
+
+ALTER TABLE public.entity REPLICA IDENTITY FULL;
diff --git a/AllInOneRunner/src/main/resources/dockerfiles/dockerfile-java b/AllInOneRunner/src/main/resources/dockerfiles/dockerfile-java
@@ -1,6 +1,4 @@
 FROM openjdk:22-ea-21-jdk-slim
 WORKDIR /usr/src/scorpio
 COPY target/quarkus-app ./
-ENV DBHOST=postgres
-ENV BUSHOST=kafka
 CMD java $MY_JAVA_SETTINGS -jar quarkus-run.jar 
diff --git a/BrokerParent/pom.xml b/BrokerParent/pom.xml
@@ -207,6 +207,18 @@
 			<groupId>io.quarkus</groupId>
 			<artifactId>quarkus-smallrye-reactive-messaging-kafka</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>io.quarkus</groupId>
+			<artifactId>quarkus-oidc</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>io.quarkus</groupId>
+			<artifactId>quarkus-keycloak-authorization</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>jakarta.annotation</groupId>
+			<artifactId>jakarta.annotation-api</artifactId>
+		</dependency>		
 	</dependencies>
 	<profiles>
 		<profile>

diff --git a/EntityManager/pom.xml b/EntityManager/pom.xml
@@ -150,7 +150,7 @@
 						</executions>
 						<configuration>
 							<dockerfile>src/main/resources/dockerfiles/dockerfile-${os}</dockerfile>
-							<repository>scorpiobroker/${project.artifactId}</repository>
+							<repository>ibn40/scorpio-${project.artifactId}</repository>
 							<tag>${docker-tag}</tag>
 							<buildArgs>
 								<FILE_BUILD>${project.build.finalName}-runner${nativeext}</FILE_BUILD>

diff --git a/.../src/main/java/eu/neclab/ngsildbroker/entityhandler/controller/EntityBatchController.java b/.../src/main/java/eu/neclab/ngsildbroker/entityhandler/controller/EntityBatchController.java
@@ -14,6 +14,7 @@
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.QueryParam;
+import jakarta.annotation.security.RolesAllowed;
 import static eu.neclab.ngsildbroker.commons.tools.EntityTools.noConcise;
 import eu.neclab.ngsildbroker.commons.datatypes.results.NGSILDOperationResult;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
@@ -60,6 +61,7 @@ public class EntityBatchController {
 
 	@POST
 	@Path("/create")
+	@RolesAllowed({"Factory-Admin", "Factory-Writer"})
 	public Uni<RestResponse<Object>> createMultiple(HttpServerRequest request,
 			List<Map<String, Object>> compactedEntities, @QueryParam("localOnly") boolean localOnly) {
 		List<Uni<Tuple2<String, Object>>> unis = Lists.newArrayList();
@@ -119,6 +121,7 @@ public Uni<RestResponse<Object>> createMultiple(HttpServerRequest request,
 
 	@POST
 	@Path("/upsert")
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> upsertMultiple(HttpServerRequest request,
 			List<Map<String, Object>> compactedEntities, @QueryParam(value = "options") String options,
 			@QueryParam("localOnly") boolean localOnly) {
@@ -186,6 +189,7 @@ public Uni<RestResponse<Object>> upsertMultiple(HttpServerRequest request,
 	 */
 	@POST
 	@Path("/update")
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> appendMultiple(HttpServerRequest request,
 			List<Map<String, Object>> compactedEntities, @QueryParam(value = "options") String options,
 			@QueryParam("localOnly") boolean localOnly) {
@@ -237,6 +241,7 @@ public Uni<RestResponse<Object>> appendMultiple(HttpServerRequest request,
 
 	@POST
 	@Path("/delete")
+	@RolesAllowed("Factory-Admin")
 	public Uni<RestResponse<Object>> deleteMultiple(HttpServerRequest request, String entityIdsStr,
 			@QueryParam("localOnly") boolean localOnly) {
 		List<String> entityIds;

diff --git a/...nager/src/main/java/eu/neclab/ngsildbroker/entityhandler/controller/EntityController.java b/...nager/src/main/java/eu/neclab/ngsildbroker/entityhandler/controller/EntityController.java
@@ -30,6 +30,7 @@
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.PathParam;
 import jakarta.ws.rs.QueryParam;
+import jakarta.annotation.security.RolesAllowed;
 /**
  * 
  * @version 1.0
@@ -58,6 +59,7 @@ public class EntityController {// implements EntityHandlerInterface {
 	 */
 	@Path("/entities")
 	@POST
+	@RolesAllowed({"Factory-Admin", "Factory-Writer"})
 	public Uni<RestResponse<Object>> createEntity(HttpServerRequest req, String bodyStr) {
 		Map<String,Object> body;
 		try {
@@ -88,6 +90,7 @@ public Uni<RestResponse<Object>> createEntity(HttpServerRequest req, String body
 
 	@PATCH
 	@Path("/entities/{entityId}/attrs")
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> updateEntity(HttpServerRequest req, @PathParam("entityId") String entityId,
 			String bodyStr) {
 		Map<String,Object> body;
@@ -117,6 +120,7 @@ public Uni<RestResponse<Object>> updateEntity(HttpServerRequest req, @PathParam(
 
 	@POST
 	@Path("/entities/{entityId}/attrs")
+	@RolesAllowed({"Factory-Admin", "Factory-Writer"})
 	public Uni<RestResponse<Object>> appendEntity(HttpServerRequest req, @PathParam("entityId") String entityId,
 			String bodyStr, @QueryParam("options") String options) {
 		Map<String,Object> body;
@@ -147,6 +151,7 @@ public Uni<RestResponse<Object>> appendEntity(HttpServerRequest req, @PathParam(
 	 */
 	@PATCH
 	@Path("/entities/{entityId}/attrs/{attrId}")
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> partialUpdateAttribute(HttpServerRequest req,
 			@PathParam("entityId") String entityId, @PathParam("attrId") String attrib, String bodyStr) {
 		Map<String,Object> body;
@@ -188,6 +193,7 @@ public Uni<RestResponse<Object>> partialUpdateAttribute(HttpServerRequest req,
 
 	@DELETE
 	@Path("/entities/{entityId}/attrs/{attrId}")
+	@RolesAllowed("Factory-Admin")
 	public Uni<RestResponse<Object>> deleteAttribute(HttpServerRequest request, @PathParam("entityId") String entityId,
 			@PathParam("attrId") String attrId, @QueryParam("datasetId") String datasetId,
 			@QueryParam("deleteAll") boolean deleteAll) {
@@ -218,6 +224,7 @@ public Uni<RestResponse<Object>> deleteAttribute(HttpServerRequest request, @Pat
 	 */
 	@DELETE
 	@Path("/entities/{entityId}")
+	@RolesAllowed("Factory-Admin")
 	public Uni<RestResponse<Object>> deleteEntity(HttpServerRequest request, @PathParam("entityId") String entityId) {
 		try {
 			HttpUtils.validateUri(entityId);
@@ -234,6 +241,7 @@ public Uni<RestResponse<Object>> deleteEntity(HttpServerRequest request, @PathPa
 
 	@PATCH
 	@Path("/entities/{entityId}")
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> mergePatch(HttpServerRequest request, @PathParam("entityId") String entityId,
 			String bodyStr) {
 		Map<String,Object> body;
@@ -259,6 +267,7 @@ public Uni<RestResponse<Object>> mergePatch(HttpServerRequest request, @PathPara
 
 	@Path("/entities/{entityId}")
 	@PUT
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> replaceEntity(@PathParam("entityId") String entityId, HttpServerRequest request,
 			String bodyStr) {
 		logger.debug("replacing entity");
@@ -289,6 +298,7 @@ public Uni<RestResponse<Object>> replaceEntity(@PathParam("entityId") String ent
 
 	@Path("/entities/{entityId}/attrs/{attrId}")
 	@PUT
+	@RolesAllowed({"Factory-Admin", "Factory-Editor", "Factory-Writer"})
 	public Uni<RestResponse<Object>> replaceAttribute(@PathParam("attrId") String attrId,
 			@PathParam("entityId") String entityId, HttpServerRequest request, String bodyStr) {
 		logger.debug("replacing Attrs");

diff --git a/EntityManager/src/main/resources/db/migration/V20190604.1__entity.sql b/EntityManager/src/main/resources/db/migration/V20190604.1__entity.sql
@@ -54,4 +54,6 @@ CREATE INDEX i_entity_operationspace ON entity USING GIST (operationspace);
 
 -- to check if this index will be used by the database optimizer, or if it should be applied only for for certain keys
 -- check https://www.postgresql.org/docs/current/static/datatype-json.html
-CREATE INDEX i_entity_data ON entity USING GIN (data); 
+CREATE INDEX i_entity_data ON entity USING GIN (data);
+
+ALTER TABLE public.entity REPLICA IDENTITY FULL;
diff --git a/QueryManager/pom.xml b/QueryManager/pom.xml
@@ -144,7 +144,7 @@
 						</executions>
 						<configuration>
 							<dockerfile>src/main/resources/dockerfiles/dockerfile-${os}</dockerfile>
-							<repository>scorpiobroker/${project.artifactId}</repository>
+							<repository>ibn40/scorpio-${project.artifactId}</repository>
 							<tag>${docker-tag}</tag>
 							<buildArgs>
 								<FILE_BUILD>${project.build.finalName}-runner${nativeext}</FILE_BUILD>

diff --git a/.../java/eu/neclab/ngsildbroker/queryhandler/controller/EntityOperationsQueryController.java b/.../java/eu/neclab/ngsildbroker/queryhandler/controller/EntityOperationsQueryController.java
@@ -11,6 +11,7 @@
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.QueryParam;
+import jakarta.annotation.security.RolesAllowed;
 
 import org.apache.commons.lang3.RandomStringUtils;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
@@ -60,6 +61,7 @@ public class EntityOperationsQueryController {
 
 	@Path("/query")
 	@POST
+	@RolesAllowed({"Factory-Admin", "Factory-Reader", "Factory-Editor"})
 	public Uni<RestResponse<Object>> postQuery(HttpServerRequest request, String bodyStr,
 			@QueryParam(value = "limit") Integer limit, @QueryParam(value = "offset") int offset,
 			@QueryParam(value = "options") String options, @QueryParam(value = "count") boolean count,