From 41e4af9b8cf25cdc5ce16025f8a2cc10c4a2ed8e Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Thu, 3 Oct 2024 10:38:36 +0200 Subject: [PATCH 01/19] wip: not tested merge entities function --- .../search/csr/service/ContextSourceUtils.kt | 130 ++++++++++++++++++ .../search/entity/web/EntityHandler.kt | 5 +- 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt new file mode 100644 index 000000000..cc3884ff9 --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -0,0 +1,130 @@ +package com.egm.stellio.search.csr.service + +import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.shared.model.* +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_COMPACTED_ENTITY_CORE_MEMBERS +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM +import com.egm.stellio.shared.util.JsonLdUtils.logger +import com.egm.stellio.shared.util.isDate +import org.json.XMLTokener.entity +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.awaitExchange +import sun.jvm.hotspot.oops.CellTypeState.value +import java.net.URI +import java.time.LocalDateTime +import kotlin.random.Random.Default.nextBoolean + +typealias SingleAttribute = Map // todo maybe use the actual attribute type +typealias CompactedAttribute = List + +object ContextSourceUtils { + + suspend fun call( + httpHeaders: HttpHeaders, + csr: ContextSourceRegistration, + path: URI, + body: String, + method: HttpMethod + ): Any { + val uri = csr.endpoint + val request = + WebClient.create("$uri/$path") + .method(method) + .headers { newHeader -> httpHeaders.entries.forEach { (key, value) -> newHeader[key] = value } } + val response: Any = request + .bodyValue(body) + .awaitExchange { response -> + logger.info( + "The csr request has return a ${response.statusCode()}}" + ) + } + return response + } + + /** + * Implements 4.5.5 - Multi-Attribute Support + */ + fun mergeEntity( + entities: List, + auxiliaryEntities: List = listOf() + ): CompactedEntity? { + val initialEntity = entities.getOrNull(0) ?: auxiliaryEntities.getOrNull(0) + if ((entities.size + auxiliaryEntities.size) <= 1) { + return initialEntity + } + val mergedEntity = initialEntity!!.toMutableMap() + entities.forEach { + entity -> + entity.entries.forEach { + (key, value) -> + when { + JSONLD_COMPACTED_ENTITY_CORE_MEMBERS.contains(key) -> {} + !mergedEntity.containsKey(key) -> mergedEntity[key] = value + else -> mergedEntity[key] = mergeAttribute(mergedEntity[key]!!, value) + } + } + } + auxiliaryEntities.forEach { entity -> + entity.entries.forEach { (key, value) -> + when { + JSONLD_COMPACTED_ENTITY_CORE_MEMBERS.contains(key) -> {} + !mergedEntity.containsKey(key) -> mergedEntity[key] = value + else -> mergedEntity[key] = mergeAttribute(mergedEntity[key]!!, value, true) + } + } + } + + return mergedEntity + } + + fun mergeAttribute( + attribute1: Any, + attribute2: Any, + auxiliary: Boolean = false + ): CompactedAttribute { + val mergeMap = attributeToDatasetIdMap(attribute1).toMutableMap() + val attribute2Map = attributeToDatasetIdMap(attribute2) + attribute2Map.entries.forEach { (datasetId, value) -> + when { + mergeMap[datasetId] == null -> mergeMap[datasetId] = value + auxiliary -> {} + mergeMap[datasetId]!!.isBefore(value, NGSILD_OBSERVED_AT_TERM) -> mergeMap[datasetId] = value + value.isBefore(mergeMap[datasetId]!!, NGSILD_OBSERVED_AT_TERM) -> {} + mergeMap[datasetId]!!.isBefore(value, NGSILD_MODIFIED_AT_TERM) -> mergeMap[datasetId] = value + value.isBefore(mergeMap[datasetId]!!, NGSILD_MODIFIED_AT_TERM) -> {} + nextBoolean() -> mergeMap[datasetId] = value + else -> {} + } + } + return mergeMap.values.toList() + } + + private fun attributeToDatasetIdMap(attribute: Any): Map> = when (attribute) { + is Map<*, *> -> { + attribute as SingleAttribute + mapOf(attribute[NGSILD_DATASET_ID_TERM] as String to attribute) + } + is List<*> -> { + attribute as CompactedAttribute + attribute.associate { + it[NGSILD_DATASET_ID_TERM] as String to it + } + } + + else -> throw InternalErrorException( + "the attribute is nor a list nor a map, check that you have excluded the CORE Members" + ) + } + private fun SingleAttribute.isBefore( + attr: SingleAttribute, + property: String + ): Boolean = ( + (this[property] as? String)?.isDate() == true && + (attr[property] as? String)?.isDate() == true && + LocalDateTime.parse(this[property] as String) < LocalDateTime.parse(attr[property] as String) + ) +} diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 793539e56..2fae59613 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -182,7 +182,7 @@ class EntityHandler( suspend fun getByURI( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, - @RequestParam params: MultiValueMap + @RequestParam params: MultiValueMap, ): ResponseEntity<*> = either { val mediaType = getApplicableMediaType(httpHeaders).bind() val sub = getSubFromSecurityContext() @@ -204,6 +204,9 @@ class EntityHandler( val compactedEntity = compactEntity(filteredExpandedEntity, contexts) val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) + + // todo first CSR working + prepareGetSuccessResponseHeaders(mediaType, contexts) .body(serializeObject(compactedEntity.toFinalRepresentation(ngsiLdDataRepresentation))) }.fold( From c7bff1e0549eea2e70ef23aa90ff1862379033d1 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 7 Oct 2024 14:32:56 +0200 Subject: [PATCH 02/19] wip: not tested merge entities function --- .../stellio/search/csr/model/CSRFilters.kt | 20 +++++++++++++++++++ .../ContextSourceRegistrationService.kt | 18 +++++++++++++---- .../entity/service/EntityQueryService.kt | 1 + .../search/entity/web/EntityHandler.kt | 17 +++++++++++++--- 4 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt new file mode 100644 index 000000000..cfd567c01 --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt @@ -0,0 +1,20 @@ +package com.egm.stellio.search.csr.model + +import java.net.URI + +data class CSRFilters( // todo use a combination of EntitiesQuery TemporalQuery (when we implement all operations) + val ids: Set = emptySet(), + +) { + fun buildWHEREStatement(): String { + val idsMatcher = if (ids.isNotEmpty()) "(" + + "entity_info.id is null" + + " OR entity_info.id in ('${ids.joinToString("', '")}')" + + ") AND (" + + "entity_info.\"idPattern\" is null OR " + + ids.map { "'$it' ~ entity_info.\"idPattern\"" }.joinToString(" OR ") + + ")" + else "true" + return idsMatcher + } +} diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index 9543612ac..20e08fac7 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -3,6 +3,7 @@ package com.egm.stellio.search.csr.service import arrow.core.* import arrow.core.raise.either import com.egm.stellio.search.common.util.* +import com.egm.stellio.search.csr.model.CSRFilters import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.ContextSourceRegistration.RegistrationInfo import com.egm.stellio.search.csr.model.ContextSourceRegistration.TimeInterval @@ -156,8 +157,11 @@ class ContextSourceRegistrationService( suspend fun getContextSourceRegistrations( limit: Int, offset: Int, - sub: Option + sub: Option, + filters: CSRFilters = CSRFilters() ): List { + val filterQuery = filters.buildWHEREStatement() + val selectStatement = """ SELECT id, @@ -171,12 +175,18 @@ class ContextSourceRegistrationService( management_interval_end, created_at, modified_at - FROM context_source_registration - WHERE sub = :sub + FROM context_source_registration as csr + LEFT JOIN jsonb_to_recordset(information) + as information(entities jsonb,propertyNames text[],relationshipNames text[] ) on true + LEFT JOIN jsonb_to_recordset(entities) + as entity_info(id text,"idPattern" text,"type" text) on true + WHERE sub = :sub AND $filterQuery ORDER BY id LIMIT :limit - OFFSET :offset) + OFFSET :offset + GROUP BY csr.id """.trimIndent() + println(selectStatement) return databaseClient.sql(selectStatement) .bind("limit", limit) .bind("offset", offset) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt index e7961bc90..87a2f33e0 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt @@ -137,6 +137,7 @@ class EntityQueryService( else null val formattedType = entitiesQuery.typeSelection?.let { "(" + buildTypeQuery(it) + ")" } val formattedAttrs = + if (entitiesQuery.attrs.isNotEmpty()) entitiesQuery.attrs.joinToString( separator = ",", diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 2fae59613..f4be1b8b8 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -4,6 +4,8 @@ import arrow.core.getOrElse import arrow.core.left import arrow.core.raise.either import arrow.core.right +import com.egm.stellio.search.csr.model.CSRFilters +import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.entity.service.EntityQueryService import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.entity.util.composeEntitiesQuery @@ -34,7 +36,8 @@ import java.util.Optional class EntityHandler( private val applicationProperties: ApplicationProperties, private val entityService: EntityService, - private val entityQueryService: EntityQueryService + private val entityQueryService: EntityQueryService, + private val contextSourceRegistrationService: ContextSourceRegistrationService ) : BaseHandler() { /** @@ -194,6 +197,16 @@ class EntityHandler( contexts ).bind() + // todo first CSR working + val csrFilters = CSRFilters(setOf(entityId)) + + contextSourceRegistrationService.getContextSourceRegistrations( + limit = Int.MAX_VALUE, + offset = 0, + sub, + csrFilters + ) + val expandedEntity = entityQueryService.queryEntity(entityId, sub.getOrNull()).bind() expandedEntity.checkContainsAnyOf(queryParams.attrs).bind() @@ -205,8 +218,6 @@ class EntityHandler( val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) - // todo first CSR working - prepareGetSuccessResponseHeaders(mediaType, contexts) .body(serializeObject(compactedEntity.toFinalRepresentation(ngsiLdDataRepresentation))) }.fold( From cc6de358d4fddbd94645c00cec27b920402fc50f Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Tue, 8 Oct 2024 17:51:52 +0200 Subject: [PATCH 03/19] feat: retrieve Entity Working --- context-source-docker-compose.yml | 54 ++++++++++++ .../ContextSourceRegistrationService.kt | 6 +- .../search/csr/service/ContextSourceUtils.kt | 83 +++++++++---------- .../search/entity/web/EntityHandler.kt | 44 +++++++--- 4 files changed, 126 insertions(+), 61 deletions(-) create mode 100644 context-source-docker-compose.yml diff --git a/context-source-docker-compose.yml b/context-source-docker-compose.yml new file mode 100644 index 000000000..9ca09790c --- /dev/null +++ b/context-source-docker-compose.yml @@ -0,0 +1,54 @@ +services: + context-source-postgres: + image: stellio/stellio-timescale-postgis:16-2.16.0-3.3 + container_name: context-source-stellio-postgres + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASS=${POSTGRES_PASS} + - POSTGRES_DBNAME=${POSTGRES_DBNAME} + - POSTGRES_MULTIPLE_EXTENSIONS=postgis,timescaledb,pgcrypto + - ACCEPT_TIMESCALE_TUNING=TRUE + ports: + - "65432:5432" + volumes: + - context-source-stellio-postgres-storage:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -h localhost -U stellio"] + interval: 10s + timeout: 5s + retries: 20 + start_period: 10s + context-source-api-gateway: + container_name: context-source-stellio-api-gateway + image: stellio/stellio-api-gateway:${STELLIO_DOCKER_TAG} + environment: + - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} + ports: + - "18080:8080" + context-source-search-service: + container_name: context-source-stellio-search-service + image: stellio/stellio-search-service:${STELLIO_DOCKER_TAG} + environment: + - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} + - SPRING_R2DBC_URL=r2dbc:pool:postgresql://context-source-postgres:5432/${STELLIO_SEARCH_DB_DATABASE} + - SPRING_FLYWAY_URL=jdbc:postgresql://context-source-postgres:5432/${STELLIO_SEARCH_DB_DATABASE} + - SPRING_R2DBC_USERNAME=${POSTGRES_USER} + - SPRING_R2DBC_PASSWORD=${POSTGRES_PASS} + - APPLICATION_AUTHENTICATION_ENABLED=${STELLIO_AUTHENTICATION_ENABLED} + - APPLICATION_TENANTS_0_ISSUER=${APPLICATION_TENANTS_0_ISSUER} + - APPLICATION_TENANTS_0_NAME=${APPLICATION_TENANTS_0_NAME} + - APPLICATION_TENANTS_0_DBSCHEMA=${APPLICATION_TENANTS_0_DBSCHEMA} + - APPLICATION_PAGINATION_LIMIT-DEFAULT=${APPLICATION_PAGINATION_LIMIT_DEFAULT} + - APPLICATION_PAGINATION_LIMIT-MAX=${APPLICATION_PAGINATION_LIMIT_MAX} + - APPLICATION_PAGINATION_TEMPORAL-LIMIT=${APPLICATION_PAGINATION_TEMPORAL_LIMIT} + + ports: + - "18083:8083" + depends_on: + context-source-postgres: + condition: service_healthy + + + +volumes: + context-source-stellio-postgres-storage: diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index 20e08fac7..4e81ad4a1 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -164,7 +164,7 @@ class ContextSourceRegistrationService( val selectStatement = """ - SELECT id, + SELECT csr.id, endpoint, mode, information, @@ -181,10 +181,10 @@ class ContextSourceRegistrationService( LEFT JOIN jsonb_to_recordset(entities) as entity_info(id text,"idPattern" text,"type" text) on true WHERE sub = :sub AND $filterQuery - ORDER BY id + GROUP BY csr.id + ORDER BY csr.id LIMIT :limit OFFSET :offset - GROUP BY csr.id """.trimIndent() println(selectStatement) return databaseClient.sql(selectStatement) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index cc3884ff9..4b0fe7ed9 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -8,80 +8,71 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.logger import com.egm.stellio.shared.util.isDate -import org.json.XMLTokener.entity import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.awaitBody import org.springframework.web.reactive.function.client.awaitExchange -import sun.jvm.hotspot.oops.CellTypeState.value -import java.net.URI import java.time.LocalDateTime import kotlin.random.Random.Default.nextBoolean typealias SingleAttribute = Map // todo maybe use the actual attribute type typealias CompactedAttribute = List +typealias CompactedEntityWithIsAuxiliary = Pair object ContextSourceUtils { suspend fun call( httpHeaders: HttpHeaders, csr: ContextSourceRegistration, - path: URI, - body: String, - method: HttpMethod - ): Any { - val uri = csr.endpoint - val request = - WebClient.create("$uri/$path") - .method(method) - .headers { newHeader -> httpHeaders.entries.forEach { (key, value) -> newHeader[key] = value } } - val response: Any = request - .bodyValue(body) - .awaitExchange { response -> - logger.info( - "The csr request has return a ${response.statusCode()}}" - ) - } - return response + method: HttpMethod, + path: String, + body: String? = null + ): CompactedEntity? { + val uri = "${csr.endpoint}$path" + val request = WebClient.create(uri) + .method(method) + .headers { newHeader -> "Link" to httpHeaders["Link"] } + body?.let { request.bodyValue(it) } + val (statusCode, response) = request + .awaitExchange { response -> response.statusCode() to response.awaitBody() } + return if (statusCode.is2xxSuccessful) { + logger.info("Successfully received Informations from CSR at : $uri") + + response + } else { + logger.info("Error contacting CSR at : $uri") + logger.info("Error contacting CSR at : $response") + null + } } - /** - * Implements 4.5.5 - Multi-Attribute Support - */ fun mergeEntity( - entities: List, - auxiliaryEntities: List = listOf() + localEntity: CompactedEntity?, + pairsOfEntitiyWithISAuxiliary: List ): CompactedEntity? { - val initialEntity = entities.getOrNull(0) ?: auxiliaryEntities.getOrNull(0) - if ((entities.size + auxiliaryEntities.size) <= 1) { - return initialEntity - } - val mergedEntity = initialEntity!!.toMutableMap() - entities.forEach { + if (localEntity == null && pairsOfEntitiyWithISAuxiliary.isEmpty()) return null + + val mergedEntity = localEntity?.toMutableMap() ?: mutableMapOf() + + pairsOfEntitiyWithISAuxiliary.forEach { entity -> - entity.entries.forEach { + entity.first.entries.forEach { (key, value) -> when { - JSONLD_COMPACTED_ENTITY_CORE_MEMBERS.contains(key) -> {} !mergedEntity.containsKey(key) -> mergedEntity[key] = value - else -> mergedEntity[key] = mergeAttribute(mergedEntity[key]!!, value) - } - } - } - auxiliaryEntities.forEach { entity -> - entity.entries.forEach { (key, value) -> - when { JSONLD_COMPACTED_ENTITY_CORE_MEMBERS.contains(key) -> {} - !mergedEntity.containsKey(key) -> mergedEntity[key] = value - else -> mergedEntity[key] = mergeAttribute(mergedEntity[key]!!, value, true) + else -> mergedEntity[key] = mergeAttribute(mergedEntity[key]!!, value, entity.second) } } } - return mergedEntity } - fun mergeAttribute( + /** + * Implements 4.5.5 - Multi-Attribute Support + */ + private fun mergeAttribute( attribute1: Any, attribute2: Any, auxiliary: Boolean = false @@ -106,12 +97,12 @@ object ContextSourceUtils { private fun attributeToDatasetIdMap(attribute: Any): Map> = when (attribute) { is Map<*, *> -> { attribute as SingleAttribute - mapOf(attribute[NGSILD_DATASET_ID_TERM] as String to attribute) + mapOf(attribute[NGSILD_DATASET_ID_TERM] as? String to attribute) } is List<*> -> { attribute as CompactedAttribute attribute.associate { - it[NGSILD_DATASET_ID_TERM] as String to it + it[NGSILD_DATASET_ID_TERM] as? String to it } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index f4be1b8b8..901f696b2 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -5,7 +5,10 @@ import arrow.core.left import arrow.core.raise.either import arrow.core.right import com.egm.stellio.search.csr.model.CSRFilters +import com.egm.stellio.search.csr.model.Mode +import com.egm.stellio.search.csr.service.CompactedEntityWithIsAuxiliary import com.egm.stellio.search.csr.service.ContextSourceRegistrationService +import com.egm.stellio.search.csr.service.ContextSourceUtils import com.egm.stellio.search.entity.service.EntityQueryService import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.entity.util.composeEntitiesQuery @@ -22,6 +25,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdTerm import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.web.BaseHandler import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod import org.springframework.http.HttpStatus import org.springframework.http.MediaType.APPLICATION_JSON_VALUE import org.springframework.http.ResponseEntity @@ -197,29 +201,45 @@ class EntityHandler( contexts ).bind() - // todo first CSR working val csrFilters = CSRFilters(setOf(entityId)) - contextSourceRegistrationService.getContextSourceRegistrations( + val matchingCSR = contextSourceRegistrationService.getContextSourceRegistrations( limit = Int.MAX_VALUE, offset = 0, sub, csrFilters ) - val expandedEntity = entityQueryService.queryEntity(entityId, sub.getOrNull()).bind() - - expandedEntity.checkContainsAnyOf(queryParams.attrs).bind() - - val filteredExpandedEntity = ExpandedEntity( - expandedEntity.filterAttributes(queryParams.attrs, queryParams.datasetId) - ) - val compactedEntity = compactEntity(filteredExpandedEntity, contexts) + // todo local parameter (6.3.18) + // todo parrallelize calls + val localEntity: CompactedEntity? = either { + val expandedEntity = entityQueryService.queryEntity(entityId, sub.getOrNull()).bind() + expandedEntity.checkContainsAnyOf(queryParams.attrs).bind() + + val filteredExpandedEntity = ExpandedEntity( + expandedEntity.filterAttributes(queryParams.attrs, queryParams.datasetId) + ) + compactEntity(filteredExpandedEntity, contexts) + }.fold({ null }, { it }) + + val compactedEntitiesWithIsAuxiliary: List = matchingCSR.mapNotNull { csr -> + ContextSourceUtils.call( + httpHeaders, + csr, + HttpMethod.GET, + "/ngsi-ld/v1/entities/$entityId", + null + )?.let { entity -> entity to (csr.mode == Mode.AUXILIARY) } + } + + val mergeEntity = ContextSourceUtils.mergeEntity(localEntity, compactedEntitiesWithIsAuxiliary) + ?: throw ResourceNotFoundException("No entity with id: $entityId found") val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) - prepareGetSuccessResponseHeaders(mediaType, contexts) - .body(serializeObject(compactedEntity.toFinalRepresentation(ngsiLdDataRepresentation))) + .body( + serializeObject(mergeEntity.toFinalRepresentation(ngsiLdDataRepresentation)) + ) }.fold( { it.toErrorResponse() }, { it } From e89f92f921bdd427a03066bd76fe83c1cdef071b Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Thu, 10 Oct 2024 16:29:00 +0200 Subject: [PATCH 04/19] feat: first error handling and fix pr comments --- .../ContextSourceRegistrationHandler.kt | 6 ++-- .../stellio/search/csr/model/CSRFilters.kt | 15 +++++----- .../ContextSourceRegistrationService.kt | 7 ++--- .../search/csr/service/ContextSourceUtils.kt | 30 +++++++++++++------ .../entity/service/EntityQueryService.kt | 1 - .../search/entity/web/EntityHandler.kt | 25 ++++++++-------- .../web/AnonymousUserHandlerTests.kt | 4 +++ .../search/entity/web/EntityHandlerTests.kt | 15 ++++++++++ .../egm/stellio/shared/model/ApiExceptions.kt | 2 ++ .../egm/stellio/shared/model/ErrorResponse.kt | 7 +++++ .../egm/stellio/shared/util/ApiResponses.kt | 3 ++ 11 files changed, 78 insertions(+), 37 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt index 067c894d8..110ac5818 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt @@ -72,9 +72,9 @@ class ContextSourceRegistrationHandler( applicationProperties.pagination.limitMax ).bind() val contextSourceRegistrations = contextSourceRegistrationService.getContextSourceRegistrations( - paginationQuery.limit, - paginationQuery.offset, - sub + limit = paginationQuery.limit, + offset = paginationQuery.offset, + sub = sub, ).serialize(contexts, mediaType, includeSysAttrs) val contextSourceRegistrationsCount = contextSourceRegistrationService.getContextSourceRegistrationsCount( sub diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt index cfd567c01..c2272565c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt @@ -7,13 +7,14 @@ data class CSRFilters( // todo use a combination of EntitiesQuery TemporalQuery ) { fun buildWHEREStatement(): String { - val idsMatcher = if (ids.isNotEmpty()) "(" + - "entity_info.id is null" + - " OR entity_info.id in ('${ids.joinToString("', '")}')" + - ") AND (" + - "entity_info.\"idPattern\" is null OR " + - ids.map { "'$it' ~ entity_info.\"idPattern\"" }.joinToString(" OR ") + - ")" + val idsMatcher = if (ids.isNotEmpty()) """"( + entity_info.id is null + OR entity_info.id in ('${ids.joinToString("', '")}') + ) AND ( + entity_info.\"idPattern\" is null OR + ${ids.joinToString(" OR ") { "'$it' ~ entity_info.\"idPattern\"" }} + ) + """.trimIndent() else "true" return idsMatcher } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index 4e81ad4a1..fea845d19 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -155,10 +155,10 @@ class ContextSourceRegistrationService( } suspend fun getContextSourceRegistrations( - limit: Int, - offset: Int, sub: Option, - filters: CSRFilters = CSRFilters() + filters: CSRFilters = CSRFilters(), + limit: Int = Int.MAX_VALUE, + offset: Int = 0, ): List { val filterQuery = filters.buildWHEREStatement() @@ -186,7 +186,6 @@ class ContextSourceRegistrationService( LIMIT :limit OFFSET :offset """.trimIndent() - println(selectStatement) return databaseClient.sql(selectStatement) .bind("limit", limit) .bind("offset", offset) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 4b0fe7ed9..05cec9925 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -1,6 +1,11 @@ package com.egm.stellio.search.csr.service +import arrow.core.Either +import arrow.core.left +import arrow.core.raise.either +import arrow.core.right import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.shared.model.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_COMPACTED_ENTITY_CORE_MEMBERS import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM @@ -10,6 +15,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.logger import com.egm.stellio.shared.util.isDate import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod +import org.springframework.http.HttpStatus import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.awaitBody import org.springframework.web.reactive.function.client.awaitExchange @@ -18,7 +24,7 @@ import kotlin.random.Random.Default.nextBoolean typealias SingleAttribute = Map // todo maybe use the actual attribute type typealias CompactedAttribute = List -typealias CompactedEntityWithIsAuxiliary = Pair +typealias CompactedEntityWithMode = Pair object ContextSourceUtils { @@ -28,7 +34,7 @@ object ContextSourceUtils { method: HttpMethod, path: String, body: String? = null - ): CompactedEntity? { + ): Either = either { val uri = "${csr.endpoint}$path" val request = WebClient.create(uri) .method(method) @@ -38,31 +44,37 @@ object ContextSourceUtils { .awaitExchange { response -> response.statusCode() to response.awaitBody() } return if (statusCode.is2xxSuccessful) { logger.info("Successfully received Informations from CSR at : $uri") - - response + response.right() } else { logger.info("Error contacting CSR at : $uri") logger.info("Error contacting CSR at : $response") - null + ContextSourceRequestException( + response.toString(), + HttpStatus.valueOf(statusCode.value()) + ).left() } } fun mergeEntity( localEntity: CompactedEntity?, - pairsOfEntitiyWithISAuxiliary: List + pairsOfEntitiyWithMode: List ): CompactedEntity? { - if (localEntity == null && pairsOfEntitiyWithISAuxiliary.isEmpty()) return null + if (localEntity == null && pairsOfEntitiyWithMode.isEmpty()) return null val mergedEntity = localEntity?.toMutableMap() ?: mutableMapOf() - pairsOfEntitiyWithISAuxiliary.forEach { + pairsOfEntitiyWithMode.forEach { entity -> entity.first.entries.forEach { (key, value) -> when { !mergedEntity.containsKey(key) -> mergedEntity[key] = value JSONLD_COMPACTED_ENTITY_CORE_MEMBERS.contains(key) -> {} - else -> mergedEntity[key] = mergeAttribute(mergedEntity[key]!!, value, entity.second) + else -> mergedEntity[key] = mergeAttribute( + mergedEntity[key]!!, + value, + entity.second == Mode.AUXILIARY + ) } } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt index 87a2f33e0..e7961bc90 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt @@ -137,7 +137,6 @@ class EntityQueryService( else null val formattedType = entitiesQuery.typeSelection?.let { "(" + buildTypeQuery(it) + ")" } val formattedAttrs = - if (entitiesQuery.attrs.isNotEmpty()) entitiesQuery.attrs.joinToString( separator = ",", diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 901f696b2..d292d9543 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -5,8 +5,6 @@ import arrow.core.left import arrow.core.raise.either import arrow.core.right import com.egm.stellio.search.csr.model.CSRFilters -import com.egm.stellio.search.csr.model.Mode -import com.egm.stellio.search.csr.service.CompactedEntityWithIsAuxiliary import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.csr.service.ContextSourceUtils import com.egm.stellio.search.entity.service.EntityQueryService @@ -204,15 +202,13 @@ class EntityHandler( val csrFilters = CSRFilters(setOf(entityId)) val matchingCSR = contextSourceRegistrationService.getContextSourceRegistrations( - limit = Int.MAX_VALUE, - offset = 0, sub, csrFilters ) // todo local parameter (6.3.18) // todo parrallelize calls - val localEntity: CompactedEntity? = either { + val localEntity = either { val expandedEntity = entityQueryService.queryEntity(entityId, sub.getOrNull()).bind() expandedEntity.checkContainsAnyOf(queryParams.attrs).bind() @@ -220,19 +216,22 @@ class EntityHandler( expandedEntity.filterAttributes(queryParams.attrs, queryParams.datasetId) ) compactEntity(filteredExpandedEntity, contexts) - }.fold({ null }, { it }) + } - val compactedEntitiesWithIsAuxiliary: List = matchingCSR.mapNotNull { csr -> + val entitiesWithMode = matchingCSR.map { csr -> ContextSourceUtils.call( httpHeaders, csr, HttpMethod.GET, - "/ngsi-ld/v1/entities/$entityId", - null - )?.let { entity -> entity to (csr.mode == Mode.AUXILIARY) } - } - - val mergeEntity = ContextSourceUtils.mergeEntity(localEntity, compactedEntitiesWithIsAuxiliary) + "/ngsi-ld/v1/entities/$entityId" + ) to csr.mode + }.filter { it.first.isRight() } // ignore all errors + .map { (response, mode) -> + response.getOrNull()!! to mode + } + + if (localEntity.isLeft() && entitiesWithMode.isEmpty()) localEntity.bind() + val mergeEntity = ContextSourceUtils.mergeEntity(localEntity.getOrNull(), entitiesWithMode) ?: throw ResourceNotFoundException("No entity with id: $entityId found") val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/AnonymousUserHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/AnonymousUserHandlerTests.kt index 5cbf00059..4afdf0daf 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/AnonymousUserHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/AnonymousUserHandlerTests.kt @@ -2,6 +2,7 @@ package com.egm.stellio.search.authorization.web import com.egm.stellio.search.authorization.service.AuthorizationService import com.egm.stellio.search.common.config.SearchProperties +import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.entity.service.EntityEventService import com.egm.stellio.search.entity.service.EntityQueryService import com.egm.stellio.search.entity.service.EntityService @@ -39,6 +40,9 @@ class AnonymousUserHandlerTests { @MockkBean private lateinit var entityQueryService: EntityQueryService + @MockkBean + private lateinit var contextSourceRegistrationService: ContextSourceRegistrationService + @Test @WithAnonymousUser fun `it should not authorize an anonymous to call the API`() { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt index 2eddfb4bc..10fff787f 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt @@ -3,6 +3,7 @@ package com.egm.stellio.search.entity.web import arrow.core.left import arrow.core.right import com.egm.stellio.search.common.config.SearchProperties +import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.entity.model.* import com.egm.stellio.search.entity.service.EntityQueryService import com.egm.stellio.search.entity.service.EntityService @@ -27,7 +28,9 @@ import io.mockk.* import kotlinx.coroutines.test.runTest import org.hamcrest.core.Is import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest @@ -46,6 +49,7 @@ import java.time.* @ActiveProfiles("test") @WebFluxTest(EntityHandler::class) @EnableConfigurationProperties(ApplicationProperties::class, SearchProperties::class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class EntityHandlerTests { @Autowired @@ -60,6 +64,9 @@ class EntityHandlerTests { @MockkBean private lateinit var entityQueryService: EntityQueryService + @MockkBean + private lateinit var contextSourceRegistrationService: ContextSourceRegistrationService + @BeforeAll fun configureWebClientDefaults() { webClient = webClient.mutate() @@ -72,6 +79,14 @@ class EntityHandlerTests { .build() } + @BeforeEach + fun mockCSR() { + coEvery { + contextSourceRegistrationService + .getContextSourceRegistrations(any(), any(), any(), any()) + } returns listOf() + } + private val beehiveId = "urn:ngsi-ld:BeeHive:TESTC".toUri() private val fishNumberAttribute = "https://ontology.eglobalmark.com/aquac#fishNumber" private val fishSizeAttribute = "https://ontology.eglobalmark.com/aquac#fishSize" diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt index 6a9269335..b48ccaa08 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt @@ -2,6 +2,7 @@ package com.egm.stellio.shared.model import com.apicatalog.jsonld.JsonLdError import com.apicatalog.jsonld.JsonLdErrorCode +import org.springframework.http.HttpStatus sealed class APIException( override val message: String @@ -20,6 +21,7 @@ data class NotImplementedException(override val message: String) : APIException( data class LdContextNotAvailableException(override val message: String) : APIException(message) data class NonexistentTenantException(override val message: String) : APIException(message) data class NotAcceptableException(override val message: String) : APIException(message) +data class ContextSourceRequestException(override val message: String, val status: HttpStatus) : APIException(message) fun Throwable.toAPIException(specificMessage: String? = null): APIException = when (this) { diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt index 3e7820cb7..ac26d9e80 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt @@ -101,6 +101,13 @@ data class NonexistentTenantResponse(override val detail: String) : detail ) +data class ContextSourceRequestResponse(override val detail: String) : + ErrorResponse( + ErrorType.NONEXISTENT_TENANT.type, + "The context source call failed", + detail + ) + enum class ErrorType(val type: URI) { INVALID_REQUEST(URI("https://uri.etsi.org/ngsi-ld/errors/InvalidRequest")), BAD_REQUEST_DATA(URI("https://uri.etsi.org/ngsi-ld/errors/BadRequestData")), diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index 6ee3a7477..f026bc367 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -67,6 +67,7 @@ private val logger = LoggerFactory.getLogger("com.egm.stellio.shared.util.ApiRes * this is globally duplicating what is in ExceptionHandler#transformErrorResponse() * but main code there should move here when we no longer raise business exceptions */ +// todo put in ApiException File? fun APIException.toErrorResponse(): ResponseEntity<*> = when (this) { is AlreadyExistsException -> @@ -87,6 +88,8 @@ fun APIException.toErrorResponse(): ResponseEntity<*> = generateErrorResponse(HttpStatus.SERVICE_UNAVAILABLE, LdContextNotAvailableResponse(this.message)) is TooManyResultsException -> generateErrorResponse(HttpStatus.FORBIDDEN, TooManyResultsResponse(this.message)) + is ContextSourceRequestException -> + generateErrorResponse(this.status, ContextSourceRequestResponse(this.message)) else -> generateErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, InternalErrorResponse("$cause")) } From 3de35ba5008e5da1d1303784bf729958686ba3ea Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Fri, 11 Oct 2024 18:10:58 +0200 Subject: [PATCH 05/19] feat: test for mergeEntity --- .../search/csr/service/ContextSourceUtils.kt | 67 +++++--- .../csr/service/ContextSourceUtilsTests.kt | 149 ++++++++++++++++++ 2 files changed, 191 insertions(+), 25 deletions(-) create mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 05cec9925..80e41834a 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -7,19 +7,23 @@ import arrow.core.right import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.shared.model.* -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_COMPACTED_ENTITY_CORE_MEMBERS +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID_TERM +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_TERM import com.egm.stellio.shared.util.JsonLdUtils.logger -import com.egm.stellio.shared.util.isDate +import com.egm.stellio.shared.util.isDateTime import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod import org.springframework.http.HttpStatus import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.awaitBody import org.springframework.web.reactive.function.client.awaitExchange -import java.time.LocalDateTime +import java.time.ZonedDateTime +import java.util.stream.Collectors.toSet import kotlin.random.Random.Default.nextBoolean typealias SingleAttribute = Map // todo maybe use the actual attribute type @@ -57,44 +61,57 @@ object ContextSourceUtils { fun mergeEntity( localEntity: CompactedEntity?, - pairsOfEntitiyWithMode: List + entitiesWithMode: List ): CompactedEntity? { - if (localEntity == null && pairsOfEntitiyWithMode.isEmpty()) return null + if (localEntity == null && entitiesWithMode.isEmpty()) return null val mergedEntity = localEntity?.toMutableMap() ?: mutableMapOf() - pairsOfEntitiyWithMode.forEach { - entity -> - entity.first.entries.forEach { - (key, value) -> - when { - !mergedEntity.containsKey(key) -> mergedEntity[key] = value - JSONLD_COMPACTED_ENTITY_CORE_MEMBERS.contains(key) -> {} - else -> mergedEntity[key] = mergeAttribute( - mergedEntity[key]!!, - value, - entity.second == Mode.AUXILIARY - ) + entitiesWithMode.sortedBy { (_, mode) -> mode == Mode.AUXILIARY } + .forEach { (entity, mode) -> + entity.entries.forEach { + (key, value) -> + when { + !mergedEntity.containsKey(key) -> mergedEntity[key] = value + key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> {} + key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> + mergedEntity[key] = mergeTypeOrScope(mergedEntity[key]!!, value) + else -> mergedEntity[key] = mergeAttribute( + mergedEntity[key]!!, + value, + mode == Mode.AUXILIARY + ) + } } } - } return mergedEntity } + fun mergeTypeOrScope( + type1: Any, // String || List || Set + type2: Any + ) = when { + type1 is List<*> && type2 is List<*> -> type1.toSet() + type2.toSet() + type1 is List<*> -> type1.toSet() + type2 + type2 is List<*> -> type2.toSet() + type1 + type1 == type2 -> setOf(type1) + else -> setOf(type1, type2) + }.toList() + /** * Implements 4.5.5 - Multi-Attribute Support */ - private fun mergeAttribute( + fun mergeAttribute( attribute1: Any, attribute2: Any, - auxiliary: Boolean = false + isAuxiliary: Boolean = false ): CompactedAttribute { val mergeMap = attributeToDatasetIdMap(attribute1).toMutableMap() val attribute2Map = attributeToDatasetIdMap(attribute2) attribute2Map.entries.forEach { (datasetId, value) -> when { mergeMap[datasetId] == null -> mergeMap[datasetId] = value - auxiliary -> {} + isAuxiliary -> {} mergeMap[datasetId]!!.isBefore(value, NGSILD_OBSERVED_AT_TERM) -> mergeMap[datasetId] = value value.isBefore(mergeMap[datasetId]!!, NGSILD_OBSERVED_AT_TERM) -> {} mergeMap[datasetId]!!.isBefore(value, NGSILD_MODIFIED_AT_TERM) -> mergeMap[datasetId] = value @@ -117,17 +134,17 @@ object ContextSourceUtils { it[NGSILD_DATASET_ID_TERM] as? String to it } } - else -> throw InternalErrorException( "the attribute is nor a list nor a map, check that you have excluded the CORE Members" ) } + private fun SingleAttribute.isBefore( attr: SingleAttribute, property: String ): Boolean = ( - (this[property] as? String)?.isDate() == true && - (attr[property] as? String)?.isDate() == true && - LocalDateTime.parse(this[property] as String) < LocalDateTime.parse(attr[property] as String) + (this[property] as? String)?.isDateTime() == true && + (attr[property] as? String)?.isDateTime() == true && + ZonedDateTime.parse(this[property] as String) < ZonedDateTime.parse(attr[property] as String) ) } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt new file mode 100644 index 000000000..925159e72 --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -0,0 +1,149 @@ +package com.egm.stellio.search.csr.service + +import com.egm.stellio.search.common.config.SearchProperties +import com.egm.stellio.search.csr.model.Mode +import com.egm.stellio.search.temporal.service.TemporalPaginationService +import com.egm.stellio.shared.model.CompactedEntity +import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM +import com.fasterxml.jackson.module.kotlin.readValue +import io.mockk.every +import io.mockk.mockkObject +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles + +@ActiveProfiles("test") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [TemporalPaginationService::class]) +@EnableConfigurationProperties(SearchProperties::class) +class ContextSourceUtilsTests { + private val name = "name" + private val minimalEntity: CompactedEntity = mapper.readValue(loadSampleData("beehive_minimal.jsonld")) + private val baseEntity: CompactedEntity = mapper.readValue(loadSampleData("beehive.jsonld")) + private val multipleTypeEntity: CompactedEntity = mapper.readValue(loadSampleData("entity_with_multi_types.jsonld")) + + private val nameAttribute: SingleAttribute = mapOf( + JSONLD_TYPE_TERM to "Property", + JSONLD_VALUE_TERM to "name", + NGSILD_DATASET_ID_TERM to "1", + NGSILD_OBSERVED_AT_TERM to "2010-01-01T01:01:01.01Z" + ) + + val moreRecentAttribute: SingleAttribute = mapOf( + JSONLD_TYPE_TERM to "Property", + JSONLD_VALUE_TERM to "moreRecentName", + NGSILD_DATASET_ID_TERM to "1", + NGSILD_OBSERVED_AT_TERM to "2020-01-01T01:01:01.01Z" + ) + + val evenMoreRecentAttribute: SingleAttribute = mapOf( + JSONLD_TYPE_TERM to "Property", + JSONLD_VALUE_TERM to "evenMoreRecentName", + NGSILD_DATASET_ID_TERM to "1", + NGSILD_OBSERVED_AT_TERM to "2030-01-01T01:01:01.01Z" + ) + + val moreRecentEntity = minimalEntity.toMutableMap() + (name to moreRecentAttribute) + val evenMoreRecentEntity = minimalEntity.toMutableMap() + (name to evenMoreRecentAttribute) + + private val entityWithName = minimalEntity.toMutableMap().plus(name to nameAttribute) + private val entityWithLastName = minimalEntity.toMutableMap().plus("lastName" to nameAttribute) + private val entityWithSurName = minimalEntity.toMutableMap().plus("surName" to nameAttribute) + + @Test + fun `merge entity should return localEntity when no other entities is provided`() = runTest { + val mergedEntity = ContextSourceUtils.mergeEntity(baseEntity, emptyList()) + assertEquals(baseEntity, mergedEntity) + } + + @Test + fun `merge entity should merge the localEntity with the list of entities `() = runTest { + val mergedEntity = ContextSourceUtils.mergeEntity(minimalEntity, listOf(baseEntity to Mode.AUXILIARY)) + assertEquals(baseEntity, mergedEntity) + } + + @Test + fun `merge entity should merge all the entities`() = runTest { + val mergedEntity = ContextSourceUtils.mergeEntity( + entityWithName, + listOf(entityWithLastName to Mode.AUXILIARY, entityWithSurName to Mode.INCLUSIVE) + ) + assertEquals(entityWithName + entityWithLastName + entityWithSurName, mergedEntity) + } + + @Test + fun `merge entity should call mergeAttribute or mergeTypeOrScope when keys are equals`() = runTest { + mockkObject(ContextSourceUtils) { + every { ContextSourceUtils.mergeAttribute(any(), any(), any()) } returns listOf( + nameAttribute + ) + every { ContextSourceUtils.mergeTypeOrScope(any(), any()) } returns listOf("Beehive") + ContextSourceUtils.mergeEntity( + entityWithName, + listOf(entityWithName to Mode.AUXILIARY, entityWithName to Mode.INCLUSIVE) + ) + verify(exactly = 2) { ContextSourceUtils.mergeAttribute(any(), any(), any()) } + verify(exactly = 2) { ContextSourceUtils.mergeTypeOrScope(any(), any()) } + } + } + + @Test + fun `merge entity should merge the types correctly `() = runTest { + val mergedEntity = ContextSourceUtils.mergeEntity( + minimalEntity, + listOf(multipleTypeEntity to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) + )!! + assertThat(mergedEntity[JSONLD_TYPE_TERM] as List<*>) + .hasSize(3) + .contains("Sensor", "BeeHive", "Beekeeper") + } + + @Test + fun `merge entity should keep both attribute if they have different datasetId `() = runTest { + val nameAttribute2: SingleAttribute = mapOf( + JSONLD_TYPE_TERM to "Property", + JSONLD_VALUE_TERM to "name2", + NGSILD_DATASET_ID_TERM to "2" + ) + val entityWithDifferentName = minimalEntity.toMutableMap() + (name to nameAttribute2) + val mergedEntity = ContextSourceUtils.mergeEntity( + entityWithName, + listOf(entityWithDifferentName to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) + )!! + assertThat(mergedEntity[name] as List<*>).hasSize(3).contains(nameAttribute, nameAttribute2, baseEntity[name]) + } + + @Test + fun `merge entity should merge attribute same datasetId keeping the most recentOne `() = runTest { + val mergedEntity = ContextSourceUtils.mergeEntity( + entityWithName, + listOf(evenMoreRecentEntity to Mode.EXCLUSIVE, moreRecentEntity to Mode.INCLUSIVE) + )!! + assertEquals( + "2030-01-01T01:01:01.01Z", + (mergedEntity[name] as List)[0][NGSILD_OBSERVED_AT_TERM] + ) + assertEquals("evenMoreRecentName", (mergedEntity[name] as List)[0][JSONLD_VALUE_TERM]) + } + + @Test + fun `merge entity should not merge Auxiliary entity `() = runTest { + val mergedEntity = ContextSourceUtils.mergeEntity( + entityWithName, + listOf(evenMoreRecentEntity to Mode.AUXILIARY, moreRecentEntity to Mode.AUXILIARY) + )!! + assertEquals( + "2010-01-01T01:01:01.01Z", + (mergedEntity[name] as List)[0][NGSILD_OBSERVED_AT_TERM] + ) + assertEquals("name", (mergedEntity[name] as List)[0][JSONLD_VALUE_TERM]) + } +} From 53f0b870e593edb234d8f30608a2f876b42b6fcd Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 13 Oct 2024 16:19:26 +0200 Subject: [PATCH 06/19] fix: working Where statement for CSR / add some tests on Query CSR / some typos and naming --- .../stellio/search/csr/model/CSRFilters.kt | 24 +-- .../ContextSourceRegistrationService.kt | 19 +- .../search/csr/service/ContextSourceUtils.kt | 5 +- .../ContextSourceRegistrationServiceTests.kt | 194 +++++++++++++++++- .../csr/contextSourceRegistration.jsonld | 8 +- 5 files changed, 215 insertions(+), 35 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt index c2272565c..6aee36625 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt @@ -3,19 +3,19 @@ package com.egm.stellio.search.csr.model import java.net.URI data class CSRFilters( // todo use a combination of EntitiesQuery TemporalQuery (when we implement all operations) - val ids: Set = emptySet(), - + val ids: Set = emptySet() ) { - fun buildWHEREStatement(): String { - val idsMatcher = if (ids.isNotEmpty()) """"( - entity_info.id is null - OR entity_info.id in ('${ids.joinToString("', '")}') - ) AND ( - entity_info.\"idPattern\" is null OR - ${ids.joinToString(" OR ") { "'$it' ~ entity_info.\"idPattern\"" }} + fun buildWHEREStatement(): String = + if (ids.isNotEmpty()) + """ + ( + entity_info.id is null OR + entity_info.id in ('${ids.joinToString("', '")}') + ) AND + ( + entity_info.idPattern is null OR + ${ids.joinToString(" OR ") { "'$it' ~ entity_info.idPattern" }} ) - """.trimIndent() + """.trimIndent() else "true" - return idsMatcher - } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index fea845d19..32d908e86 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -9,8 +9,12 @@ import com.egm.stellio.search.csr.model.ContextSourceRegistration.RegistrationIn import com.egm.stellio.search.csr.model.ContextSourceRegistration.TimeInterval import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.search.csr.model.Operation -import com.egm.stellio.shared.model.* -import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.model.APIException +import com.egm.stellio.shared.model.AlreadyExistsException +import com.egm.stellio.shared.model.ResourceNotFoundException +import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.mapper +import com.egm.stellio.shared.util.toStringValue import io.r2dbc.postgresql.codec.Json import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.data.relational.core.query.Criteria.where @@ -176,11 +180,12 @@ class ContextSourceRegistrationService( created_at, modified_at FROM context_source_registration as csr - LEFT JOIN jsonb_to_recordset(information) - as information(entities jsonb,propertyNames text[],relationshipNames text[] ) on true - LEFT JOIN jsonb_to_recordset(entities) - as entity_info(id text,"idPattern" text,"type" text) on true - WHERE sub = :sub AND $filterQuery + LEFT JOIN jsonb_to_recordset(information) + as information(entities jsonb, propertyNames text[], relationshipNames text[]) on true + LEFT JOIN jsonb_to_recordset(entities) + as entity_info(id text, idPattern text, type text) on true + WHERE sub = :sub + AND $filterQuery GROUP BY csr.id ORDER BY csr.id LIMIT :limit diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 80e41834a..4e45706c5 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -47,11 +47,10 @@ object ContextSourceUtils { val (statusCode, response) = request .awaitExchange { response -> response.statusCode() to response.awaitBody() } return if (statusCode.is2xxSuccessful) { - logger.info("Successfully received Informations from CSR at : $uri") + logger.debug("Successfully received response from CSR at $uri") response.right() } else { - logger.info("Error contacting CSR at : $uri") - logger.info("Error contacting CSR at : $response") + logger.warn("Error contacting CSR at $uri: $response") ContextSourceRequestException( response.toString(), HttpStatus.valueOf(statusCode.value()) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt index af37d548e..bb838aa70 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt @@ -1,6 +1,7 @@ package com.egm.stellio.search.csr.service import arrow.core.Some +import com.egm.stellio.search.csr.model.CSRFilters import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.ContextSourceRegistration.Companion.notFoundMessage import com.egm.stellio.search.support.WithTimescaleContainer @@ -11,6 +12,7 @@ import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest @@ -44,12 +46,18 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contexts: List = APIC_COMPOUND_CONTEXTS ): ContextSourceRegistration { val csrPayload = loadSampleData(filename) - return ContextSourceRegistration.deserialize(csrPayload.deserializeAsMap(), contexts) - .shouldSucceedAndResult() + return deserializeContextSourceRegistration(csrPayload, contexts) } + fun deserializeContextSourceRegistration( + csrPayload: String, + contexts: List = APIC_COMPOUND_CONTEXTS + ): ContextSourceRegistration = + ContextSourceRegistration.deserialize(csrPayload.deserializeAsMap(), contexts) + .shouldSucceedAndResult() + @Test - fun `creating a second CSR with the same id should fail with AlreadyExistError`() = runTest { + fun `create a second CSR with the same id should return an AlreadyExist error`() = runTest { val contextSourceRegistration = loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_minimal_entities.json") contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() @@ -59,7 +67,7 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { } @Test - fun `getting a simple CSR should return the created contextSourceRegistration`() = runTest { + fun `get a minimal CSR should return the created CSR`() = runTest { val contextSourceRegistration = loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_minimal_entities.json") contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() @@ -67,12 +75,12 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.getById( contextSourceRegistration.id ).shouldSucceedWith { - assertEquals(it, contextSourceRegistration) + assertEquals(contextSourceRegistration, it) } } @Test - fun `getting a full CSR should return the created CSR`() = runTest { + fun `get a full CSR should return the created CSR`() = runTest { val contextSourceRegistration = loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_full.json") contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() @@ -80,12 +88,180 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.getById( contextSourceRegistration.id ).shouldSucceedWith { - assertEquals(it, contextSourceRegistration) + assertEquals(contextSourceRegistration, it) } } @Test - fun `deleting an existing CSR should succeed`() = runTest { + fun `query CSR on id should return a CSR matching this id uniquely`() = runTest { + val contextSourceRegistration = + loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_minimal_entities.json") + contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() + + val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( + mockUserSub, + CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) + ) + + assertEquals(1, matchingCsrs.size) + } + + @Test + fun `query CSR on id should return a CSR matching this id in one of the entities`() = runTest { + val contextSourceRegistration = + deserializeContextSourceRegistration( + """ + { + "id": "urn:ngsi-ld:ContextSourceRegistration:1", + "type": "ContextSourceRegistration", + "information": [ + { + "entities": [ + { + "id": "urn:ngsi-ld:Vehicle:A456", + "type": "Vehicle" + }, + { + "id": "urn:ngsi-ld:Vehicle:A457", + "type": "Vehicle" + } + ] + } + ], + "endpoint": "http://my.csr.endpoint/" + } + """.trimIndent() + ) + contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() + + val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( + mockUserSub, + CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) + ) + + assertEquals(1, matchingCsrs.size) + } + + @Test + fun `query CSR on id should return a CSR matching this id on idPattern`() = runTest { + val contextSourceRegistration = + deserializeContextSourceRegistration( + """ + { + "id": "urn:ngsi-ld:ContextSourceRegistration:1", + "type": "ContextSourceRegistration", + "information": [ + { + "entities": [ + { + "idPattern": "urn:ngsi-ld:Vehicle:A4*", + "type": "Vehicle" + } + ] + } + ], + "endpoint": "http://my.csr.endpoint/" + } + """.trimIndent() + ) + contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() + + val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( + mockUserSub, + CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) + ) + + assertEquals(1, matchingCsrs.size) + } + + @Test + fun `query CSR on id should return a CSR matching this id on idPattern but not on id`() = runTest { + val contextSourceRegistration = + deserializeContextSourceRegistration( + """ + { + "id": "urn:ngsi-ld:ContextSourceRegistration:1", + "type": "ContextSourceRegistration", + "information": [ + { + "entities": [ + { + "idPattern": "urn:ngsi-ld:Vehicle:A4*", + "type": "Vehicle" + }, + { + "id": "urn:ngsi-ld:Vehicle:B123", + "type": "Vehicle" + } + ] + } + ], + "endpoint": "http://my.csr.endpoint/" + } + """.trimIndent() + ) + contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() + + val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( + mockUserSub, + CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) + ) + + assertEquals(1, matchingCsrs.size) + } + + @Test + fun `query CSR on id should return a single CSR when entities matches twice`() = runTest { + val contextSourceRegistration = + deserializeContextSourceRegistration( + """ + { + "id": "urn:ngsi-ld:ContextSourceRegistration:1", + "type": "ContextSourceRegistration", + "information": [ + { + "entities": [ + { + "idPattern": "urn:ngsi-ld:Vehicle:A4*", + "type": "Vehicle" + }, + { + "id": "urn:ngsi-ld:Vehicle:A456", + "type": "Vehicle" + } + ] + } + ], + "endpoint": "http://my.csr.endpoint/" + } + """.trimIndent() + ) + contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() + + val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( + mockUserSub, + CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) + ) + + assertEquals(1, matchingCsrs.size) + } + + @Test + fun `query CSR on id should return an empty list if no CSR matches`() = runTest { + val contextSourceRegistration = + loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_minimal_entities.json") + contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() + + val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( + mockUserSub, + CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A457".toUri())) + ) + + assertTrue(matchingCsrs.isEmpty()) + } + + @Test + fun `delete an existing CSR should succeed`() = runTest { val contextSourceRegistration = loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_minimal_entities.json") contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() @@ -99,7 +275,7 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { } @Test - fun `deletin an non existing CSR should return a RessourceNotFound Error`() = runTest { + fun `delete a non existing CSR should return a RessourceNotFound error`() = runTest { val id = "urn:ngsi-ld:ContextSourceRegistration:UnknownContextSourceRegistration".toUri() contextSourceRegistrationService.delete( id diff --git a/search-service/src/test/resources/ngsild/csr/contextSourceRegistration.jsonld b/search-service/src/test/resources/ngsild/csr/contextSourceRegistration.jsonld index a9f0ed4cb..b7a68ebeb 100644 --- a/search-service/src/test/resources/ngsild/csr/contextSourceRegistration.jsonld +++ b/search-service/src/test/resources/ngsild/csr/contextSourceRegistration.jsonld @@ -11,8 +11,8 @@ ] } ], - "@context":[ - "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" - ], + "@context": [ + "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" + ], "endpoint": "http://my.csr.endpoint/" -} \ No newline at end of file +} From c8bf170290080acb6a658b29aeb814bec45e1f30 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 14 Oct 2024 15:46:27 +0200 Subject: [PATCH 07/19] feat: fix merging returning lists and PR comments --- .../stellio/search/csr/model/CSRFilters.kt | 2 +- .../search/csr/service/ContextSourceUtils.kt | 74 ++++++++++--------- .../search/entity/web/EntityHandler.kt | 16 ++-- .../csr/service/ContextSourceUtilsTests.kt | 23 +++--- .../search/entity/web/EntityHandlerTests.kt | 2 - .../stellio/shared/model/CompactedEntity.kt | 2 + .../egm/stellio/shared/model/ErrorResponse.kt | 5 +- .../egm/stellio/shared/util/ApiResponses.kt | 1 - 8 files changed, 64 insertions(+), 61 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt index 6aee36625..bd68e4e2e 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt @@ -2,7 +2,7 @@ package com.egm.stellio.search.csr.model import java.net.URI -data class CSRFilters( // todo use a combination of EntitiesQuery TemporalQuery (when we implement all operations) +data class CSRFilters( // we should use a combination of EntitiesQuery TemporalQuery (when we implement all operations) val ids: Set = emptySet() ) { fun buildWHEREStatement(): String = diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 4e45706c5..3a822e2de 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -19,15 +19,14 @@ import com.egm.stellio.shared.util.isDateTime import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod import org.springframework.http.HttpStatus +import org.springframework.util.MultiValueMap import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.awaitBody import org.springframework.web.reactive.function.client.awaitExchange +import java.net.URI import java.time.ZonedDateTime -import java.util.stream.Collectors.toSet import kotlin.random.Random.Default.nextBoolean -typealias SingleAttribute = Map // todo maybe use the actual attribute type -typealias CompactedAttribute = List typealias CompactedEntityWithMode = Pair object ContextSourceUtils { @@ -37,13 +36,19 @@ object ContextSourceUtils { csr: ContextSourceRegistration, method: HttpMethod, path: String, - body: String? = null + params: MultiValueMap ): Either = either { - val uri = "${csr.endpoint}$path" - val request = WebClient.create(uri) + val uri = URI("${csr.endpoint}$path") + val request = WebClient.create() .method(method) - .headers { newHeader -> "Link" to httpHeaders["Link"] } - body?.let { request.bodyValue(it) } + .uri { uriBuilder -> + uriBuilder.scheme(uri.scheme) + .host(uri.host) + .path(uri.path) + .queryParams(params) + .build() + } + .header(HttpHeaders.LINK, httpHeaders.getFirst(HttpHeaders.LINK)) val (statusCode, response) = request .awaitExchange { response -> response.statusCode() to response.awaitBody() } return if (statusCode.is2xxSuccessful) { @@ -70,13 +75,14 @@ object ContextSourceUtils { .forEach { (entity, mode) -> entity.entries.forEach { (key, value) -> + val mergedValue = mergedEntity[key] when { - !mergedEntity.containsKey(key) -> mergedEntity[key] = value + mergedValue == null -> mergedEntity[key] = value key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> {} key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> - mergedEntity[key] = mergeTypeOrScope(mergedEntity[key]!!, value) + mergedEntity[key] = mergeTypeOrScope(mergedValue, value) else -> mergedEntity[key] = mergeAttribute( - mergedEntity[key]!!, + mergedValue, value, mode == Mode.AUXILIARY ) @@ -90,12 +96,12 @@ object ContextSourceUtils { type1: Any, // String || List || Set type2: Any ) = when { - type1 is List<*> && type2 is List<*> -> type1.toSet() + type2.toSet() - type1 is List<*> -> type1.toSet() + type2 - type2 is List<*> -> type2.toSet() + type1 - type1 == type2 -> setOf(type1) - else -> setOf(type1, type2) - }.toList() + type1 is List<*> && type2 is List<*> -> (type1.toSet() + type2.toSet()).toList() + type1 is List<*> -> (type1.toSet() + type2).toList() + type2 is List<*> -> (type2.toSet() + type1).toList() + type1 == type2 -> type1 + else -> listOf(type1, type2) + } /** * Implements 4.5.5 - Multi-Attribute Support @@ -104,42 +110,42 @@ object ContextSourceUtils { attribute1: Any, attribute2: Any, isAuxiliary: Boolean = false - ): CompactedAttribute { + ): Any { val mergeMap = attributeToDatasetIdMap(attribute1).toMutableMap() val attribute2Map = attributeToDatasetIdMap(attribute2) - attribute2Map.entries.forEach { (datasetId, value) -> + attribute2Map.entries.forEach { (datasetId, value2) -> + val value1 = mergeMap[datasetId] when { - mergeMap[datasetId] == null -> mergeMap[datasetId] = value + value1 == null -> mergeMap[datasetId] = value2 isAuxiliary -> {} - mergeMap[datasetId]!!.isBefore(value, NGSILD_OBSERVED_AT_TERM) -> mergeMap[datasetId] = value - value.isBefore(mergeMap[datasetId]!!, NGSILD_OBSERVED_AT_TERM) -> {} - mergeMap[datasetId]!!.isBefore(value, NGSILD_MODIFIED_AT_TERM) -> mergeMap[datasetId] = value - value.isBefore(mergeMap[datasetId]!!, NGSILD_MODIFIED_AT_TERM) -> {} - nextBoolean() -> mergeMap[datasetId] = value + value1.isBefore(value2, NGSILD_OBSERVED_AT_TERM) -> mergeMap[datasetId] = value2 + value2.isBefore(value1, NGSILD_OBSERVED_AT_TERM) -> {} + value1.isBefore(value2, NGSILD_MODIFIED_AT_TERM) -> mergeMap[datasetId] = value2 + value2.isBefore(value1, NGSILD_MODIFIED_AT_TERM) -> {} + nextBoolean() -> mergeMap[datasetId] = value2 else -> {} } } - return mergeMap.values.toList() + val values = mergeMap.values.toList() + return if (values.size == 1) values[0] else values } - private fun attributeToDatasetIdMap(attribute: Any): Map> = when (attribute) { + private fun attributeToDatasetIdMap(attribute: Any): Map = when (attribute) { is Map<*, *> -> { - attribute as SingleAttribute + attribute as CompactedAttributeInstance mapOf(attribute[NGSILD_DATASET_ID_TERM] as? String to attribute) } is List<*> -> { - attribute as CompactedAttribute - attribute.associate { - it[NGSILD_DATASET_ID_TERM] as? String to it - } + attribute as CompactedAttributeInstances + attribute.associateBy { it[NGSILD_DATASET_ID_TERM] as? String } } else -> throw InternalErrorException( "the attribute is nor a list nor a map, check that you have excluded the CORE Members" ) } - private fun SingleAttribute.isBefore( - attr: SingleAttribute, + private fun CompactedAttributeInstance.isBefore( + attr: CompactedAttributeInstance, property: String ): Boolean = ( (this[property] as? String)?.isDateTime() == true && diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index d292d9543..f2bcb9168 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -4,7 +4,9 @@ import arrow.core.getOrElse import arrow.core.left import arrow.core.raise.either import arrow.core.right +import arrow.fx.coroutines.parMap import com.egm.stellio.search.csr.model.CSRFilters +import com.egm.stellio.search.csr.model.Operation import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.csr.service.ContextSourceUtils import com.egm.stellio.search.entity.service.EntityQueryService @@ -204,10 +206,8 @@ class EntityHandler( val matchingCSR = contextSourceRegistrationService.getContextSourceRegistrations( sub, csrFilters - ) + ).filter { csr -> csr.operations.any { it == Operation.FEDERATION_OPS || it == Operation.RETRIEVE_ENTITY } } - // todo local parameter (6.3.18) - // todo parrallelize calls val localEntity = either { val expandedEntity = entityQueryService.queryEntity(entityId, sub.getOrNull()).bind() expandedEntity.checkContainsAnyOf(queryParams.attrs).bind() @@ -218,12 +218,14 @@ class EntityHandler( compactEntity(filteredExpandedEntity, contexts) } - val entitiesWithMode = matchingCSR.map { csr -> + // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time + val entitiesWithMode = matchingCSR.parMap { csr -> ContextSourceUtils.call( httpHeaders, csr, HttpMethod.GET, - "/ngsi-ld/v1/entities/$entityId" + "/ngsi-ld/v1/entities/$entityId", + params ) to csr.mode }.filter { it.first.isRight() } // ignore all errors .map { (response, mode) -> @@ -231,13 +233,13 @@ class EntityHandler( } if (localEntity.isLeft() && entitiesWithMode.isEmpty()) localEntity.bind() - val mergeEntity = ContextSourceUtils.mergeEntity(localEntity.getOrNull(), entitiesWithMode) + val mergedEntity = ContextSourceUtils.mergeEntity(localEntity.getOrNull(), entitiesWithMode) ?: throw ResourceNotFoundException("No entity with id: $entityId found") val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) prepareGetSuccessResponseHeaders(mediaType, contexts) .body( - serializeObject(mergeEntity.toFinalRepresentation(ngsiLdDataRepresentation)) + serializeObject(mergedEntity.toFinalRepresentation(ngsiLdDataRepresentation)) ) }.fold( { it.toErrorResponse() }, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt index 925159e72..46c7ebf19 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -1,8 +1,7 @@ package com.egm.stellio.search.csr.service -import com.egm.stellio.search.common.config.SearchProperties import com.egm.stellio.search.csr.model.Mode -import com.egm.stellio.search.temporal.service.TemporalPaginationService +import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedEntity import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM @@ -17,34 +16,30 @@ import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.springframework.boot.context.properties.EnableConfigurationProperties -import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @ActiveProfiles("test") -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [TemporalPaginationService::class]) -@EnableConfigurationProperties(SearchProperties::class) class ContextSourceUtilsTests { private val name = "name" private val minimalEntity: CompactedEntity = mapper.readValue(loadSampleData("beehive_minimal.jsonld")) private val baseEntity: CompactedEntity = mapper.readValue(loadSampleData("beehive.jsonld")) private val multipleTypeEntity: CompactedEntity = mapper.readValue(loadSampleData("entity_with_multi_types.jsonld")) - private val nameAttribute: SingleAttribute = mapOf( + private val nameAttribute: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "name", NGSILD_DATASET_ID_TERM to "1", NGSILD_OBSERVED_AT_TERM to "2010-01-01T01:01:01.01Z" ) - val moreRecentAttribute: SingleAttribute = mapOf( + val moreRecentAttribute: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "moreRecentName", NGSILD_DATASET_ID_TERM to "1", NGSILD_OBSERVED_AT_TERM to "2020-01-01T01:01:01.01Z" ) - val evenMoreRecentAttribute: SingleAttribute = mapOf( + val evenMoreRecentAttribute: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "evenMoreRecentName", NGSILD_DATASET_ID_TERM to "1", @@ -108,7 +103,7 @@ class ContextSourceUtilsTests { @Test fun `merge entity should keep both attribute if they have different datasetId `() = runTest { - val nameAttribute2: SingleAttribute = mapOf( + val nameAttribute2: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "name2", NGSILD_DATASET_ID_TERM to "2" @@ -129,9 +124,9 @@ class ContextSourceUtilsTests { )!! assertEquals( "2030-01-01T01:01:01.01Z", - (mergedEntity[name] as List)[0][NGSILD_OBSERVED_AT_TERM] + (mergedEntity[name] as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] ) - assertEquals("evenMoreRecentName", (mergedEntity[name] as List)[0][JSONLD_VALUE_TERM]) + assertEquals("evenMoreRecentName", (mergedEntity[name] as CompactedAttributeInstance)[JSONLD_VALUE_TERM]) } @Test @@ -142,8 +137,8 @@ class ContextSourceUtilsTests { )!! assertEquals( "2010-01-01T01:01:01.01Z", - (mergedEntity[name] as List)[0][NGSILD_OBSERVED_AT_TERM] + (mergedEntity[name] as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] ) - assertEquals("name", (mergedEntity[name] as List)[0][JSONLD_VALUE_TERM]) + assertEquals("name", (mergedEntity[name] as CompactedAttributeInstance)[JSONLD_VALUE_TERM]) } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt index 10fff787f..1502e15f0 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt @@ -30,7 +30,6 @@ import org.hamcrest.core.Is import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInstance import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest @@ -49,7 +48,6 @@ import java.time.* @ActiveProfiles("test") @WebFluxTest(EntityHandler::class) @EnableConfigurationProperties(ApplicationProperties::class, SearchProperties::class) -@TestInstance(TestInstance.Lifecycle.PER_CLASS) class EntityHandlerTests { @Autowired diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt index d9dd98216..608113751 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt @@ -24,6 +24,8 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_TERM import java.util.Locale typealias CompactedEntity = Map +typealias CompactedAttributeInstance = Map +typealias CompactedAttributeInstances = List fun CompactedEntity.toSimplifiedAttributes(): Map = this.mapValues { (_, value) -> diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt index ac26d9e80..0baea12cf 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt @@ -103,7 +103,7 @@ data class NonexistentTenantResponse(override val detail: String) : data class ContextSourceRequestResponse(override val detail: String) : ErrorResponse( - ErrorType.NONEXISTENT_TENANT.type, + ErrorType.CONTEXT_SOURCE_REQUEST.type, "The context source call failed", detail ) @@ -122,5 +122,6 @@ enum class ErrorType(val type: URI) { NOT_IMPLEMENTED(URI("https://uri.etsi.org/ngsi-ld/errors/NotImplemented")), UNSUPPORTED_MEDIA_TYPE(URI("https://uri.etsi.org/ngsi-ld/errors/UnsupportedMediaType")), NOT_ACCEPTABLE(URI("https://uri.etsi.org/ngsi-ld/errors/NotAcceptable")), - NONEXISTENT_TENANT(URI("https://uri.etsi.org/ngsi-ld/errors/NonexistentTenant")) + NONEXISTENT_TENANT(URI("https://uri.etsi.org/ngsi-ld/errors/NonexistentTenant")), + CONTEXT_SOURCE_REQUEST(URI("https://uri.etsi.org/ngsi-ld/errors/ContextSourceRequest")) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index f026bc367..0122c7014 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -67,7 +67,6 @@ private val logger = LoggerFactory.getLogger("com.egm.stellio.shared.util.ApiRes * this is globally duplicating what is in ExceptionHandler#transformErrorResponse() * but main code there should move here when we no longer raise business exceptions */ -// todo put in ApiException File? fun APIException.toErrorResponse(): ResponseEntity<*> = when (this) { is AlreadyExistsException -> From e8ccf0c0dd4ee49270f52a378d696e0101cf49c4 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Tue, 15 Oct 2024 19:25:58 +0200 Subject: [PATCH 08/19] feat: fix merging returning lists and PR comments --- .../search/csr/service/ContextSourceUtils.kt | 58 +++++++++++++------ .../search/entity/web/EntityHandler.kt | 25 ++++---- shared/config/detekt/baseline.xml | 1 + .../egm/stellio/shared/model/ApiExceptions.kt | 6 +- .../egm/stellio/shared/model/ErrorResponse.kt | 11 +--- .../egm/stellio/shared/util/ApiResponses.kt | 50 +++++++++++----- .../egm/stellio/shared/util/NGSILDWarning.kt | 21 +++++++ 7 files changed, 115 insertions(+), 57 deletions(-) create mode 100644 shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 3a822e2de..e8a578367 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -4,9 +4,12 @@ import arrow.core.Either import arrow.core.left import arrow.core.raise.either import arrow.core.right -import com.egm.stellio.search.csr.model.ContextSourceRegistration -import com.egm.stellio.search.csr.model.Mode -import com.egm.stellio.shared.model.* +import com.egm.stellio.search.csr.model.* +import com.egm.stellio.shared.model.CompactedAttributeInstance +import com.egm.stellio.shared.model.CompactedAttributeInstances +import com.egm.stellio.shared.model.CompactedEntity +import com.egm.stellio.shared.model.InternalErrorException +import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM @@ -15,12 +18,13 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_TERM import com.egm.stellio.shared.util.JsonLdUtils.logger -import com.egm.stellio.shared.util.isDateTime +import com.fasterxml.jackson.core.JacksonException import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod import org.springframework.http.HttpStatus import org.springframework.util.MultiValueMap import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.WebClientException import org.springframework.web.reactive.function.client.awaitBody import org.springframework.web.reactive.function.client.awaitExchange import java.net.URI @@ -31,16 +35,15 @@ typealias CompactedEntityWithMode = Pair object ContextSourceUtils { - suspend fun call( + suspend fun getDistributedInformation( httpHeaders: HttpHeaders, csr: ContextSourceRegistration, - method: HttpMethod, path: String, params: MultiValueMap - ): Either = either { + ): Either = either { val uri = URI("${csr.endpoint}$path") val request = WebClient.create() - .method(method) + .method(HttpMethod.GET) .uri { uriBuilder -> uriBuilder.scheme(uri.scheme) .host(uri.host) @@ -49,16 +52,35 @@ object ContextSourceUtils { .build() } .header(HttpHeaders.LINK, httpHeaders.getFirst(HttpHeaders.LINK)) - val (statusCode, response) = request - .awaitExchange { response -> response.statusCode() to response.awaitBody() } - return if (statusCode.is2xxSuccessful) { - logger.debug("Successfully received response from CSR at $uri") - response.right() - } else { - logger.warn("Error contacting CSR at $uri: $response") - ContextSourceRequestException( - response.toString(), - HttpStatus.valueOf(statusCode.value()) + return try { + val (statusCode, response) = request + .awaitExchange { response -> + response.statusCode() to response.awaitBody() + } + when { + statusCode.is2xxSuccessful -> { + logger.info("Successfully received response from CSR at $uri") + response.right() + } + statusCode.isSameCodeAs(HttpStatus.NOT_FOUND) -> { + logger.info("CSR returned 404 at $uri: $response") + null.left() + } + else -> { + logger.warn("Error contacting CSR at $uri: $response") + + MiscellaneousPersistentWarning( + "the CSR ${csr.id} returned an error $statusCode at : $uri response: \"$response\"" + ).left() + } + } + } catch (e: WebClientException) { + MiscellaneousWarning( + "Error connecting to csr ${csr.id} at : $uri message : \"${e.message}\"" + ).left() + } catch (e: JacksonException) { // todo get the good exception for invalid payload + RevalidationFailedWarning( + "the CSR ${csr.id} as : $uri returned badly formed data message: \"${e.message}\"" ).left() } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index f2bcb9168..2cd19b77b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -25,7 +25,6 @@ import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdTerm import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.web.BaseHandler import org.springframework.http.HttpHeaders -import org.springframework.http.HttpMethod import org.springframework.http.HttpStatus import org.springframework.http.MediaType.APPLICATION_JSON_VALUE import org.springframework.http.ResponseEntity @@ -219,25 +218,29 @@ class EntityHandler( } // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time - val entitiesWithMode = matchingCSR.parMap { csr -> - ContextSourceUtils.call( + val (entitiesWithMode, warnings) = matchingCSR.parMap { csr -> + ContextSourceUtils.getDistributedInformation( httpHeaders, csr, - HttpMethod.GET, "/ngsi-ld/v1/entities/$entityId", params ) to csr.mode - }.filter { it.first.isRight() } // ignore all errors - .map { (response, mode) -> - response.getOrNull()!! to mode + }.partition { it.first.isRight() } + .let { (responses, warnings) -> + responses.map { (response, mode) -> response.getOrNull()!! to mode } to + warnings.mapNotNull { (warning, _) -> warning.swap().getOrNull() } } - if (localEntity.isLeft() && entitiesWithMode.isEmpty()) localEntity.bind() - val mergedEntity = ContextSourceUtils.mergeEntity(localEntity.getOrNull(), entitiesWithMode) - ?: throw ResourceNotFoundException("No entity with id: $entityId found") + val localError = localEntity.swap().getOrNull() + if (localError != null && entitiesWithMode.isEmpty()) { + localError.warnings = warnings + localError.left().bind() + } + + val mergedEntity = ContextSourceUtils.mergeEntity(localEntity.getOrNull(), entitiesWithMode)!! val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) - prepareGetSuccessResponseHeaders(mediaType, contexts) + prepareGetSuccessResponseHeaders(mediaType, contexts, warnings) .body( serializeObject(mergedEntity.toFinalRepresentation(ngsiLdDataRepresentation)) ) diff --git a/shared/config/detekt/baseline.xml b/shared/config/detekt/baseline.xml index c3deb4468..4a79679ab 100644 --- a/shared/config/detekt/baseline.xml +++ b/shared/config/detekt/baseline.xml @@ -8,6 +8,7 @@ LongParameterList:ApiResponses.kt$( body: String, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) LongParameterList:ApiResponses.kt$( entities: Any, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) SpreadOperator:EntityEvent.kt$EntityEvent$( *[ JsonSubTypes.Type(value = EntityCreateEvent::class), JsonSubTypes.Type(value = EntityReplaceEvent::class), JsonSubTypes.Type(value = EntityDeleteEvent::class), JsonSubTypes.Type(value = AttributeAppendEvent::class), JsonSubTypes.Type(value = AttributeReplaceEvent::class), JsonSubTypes.Type(value = AttributeUpdateEvent::class), JsonSubTypes.Type(value = AttributeDeleteEvent::class), JsonSubTypes.Type(value = AttributeDeleteAllInstancesEvent::class) ] ) + SpreadOperator:NGSILDWarning.kt$(NGSILDWarning.HEADER_NAME, *this.map { it.message }.toTypedArray()) SwallowedException:JsonLdUtils.kt$JsonLdUtils$e: JsonLdError TooManyFunctions:JsonLdUtils.kt$JsonLdUtils diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt index b48ccaa08..5a79b7e2c 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt @@ -2,10 +2,11 @@ package com.egm.stellio.shared.model import com.apicatalog.jsonld.JsonLdError import com.apicatalog.jsonld.JsonLdErrorCode -import org.springframework.http.HttpStatus +import com.egm.stellio.shared.util.NGSILDWarning sealed class APIException( - override val message: String + override val message: String, + var warnings: List? = null ) : Exception(message) data class InvalidRequestException(override val message: String) : APIException(message) @@ -21,7 +22,6 @@ data class NotImplementedException(override val message: String) : APIException( data class LdContextNotAvailableException(override val message: String) : APIException(message) data class NonexistentTenantException(override val message: String) : APIException(message) data class NotAcceptableException(override val message: String) : APIException(message) -data class ContextSourceRequestException(override val message: String, val status: HttpStatus) : APIException(message) fun Throwable.toAPIException(specificMessage: String? = null): APIException = when (this) { diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt index 0baea12cf..e47458798 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt @@ -5,7 +5,7 @@ import java.net.URI sealed class ErrorResponse( val type: URI, open val title: String, - open val detail: String + open val detail: String, ) data class InvalidRequestResponse(override val detail: String) : ErrorResponse( @@ -98,14 +98,7 @@ data class NonexistentTenantResponse(override val detail: String) : ErrorResponse( ErrorType.NONEXISTENT_TENANT.type, "The addressed tenant does not exist", - detail - ) - -data class ContextSourceRequestResponse(override val detail: String) : - ErrorResponse( - ErrorType.CONTEXT_SOURCE_REQUEST.type, - "The context source call failed", - detail + detail, ) enum class ErrorType(val type: URI) { diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index 0122c7014..b60223633 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -70,33 +70,44 @@ private val logger = LoggerFactory.getLogger("com.egm.stellio.shared.util.ApiRes fun APIException.toErrorResponse(): ResponseEntity<*> = when (this) { is AlreadyExistsException -> - generateErrorResponse(HttpStatus.CONFLICT, AlreadyExistsResponse(this.message)) + generateErrorResponse(HttpStatus.CONFLICT, AlreadyExistsResponse(this.message), this.warnings) is ResourceNotFoundException -> - generateErrorResponse(HttpStatus.NOT_FOUND, ResourceNotFoundResponse(this.message)) + generateErrorResponse(HttpStatus.NOT_FOUND, ResourceNotFoundResponse(this.message), this.warnings) is InvalidRequestException -> - generateErrorResponse(HttpStatus.BAD_REQUEST, InvalidRequestResponse(this.message)) + generateErrorResponse(HttpStatus.BAD_REQUEST, InvalidRequestResponse(this.message), this.warnings) is BadRequestDataException -> - generateErrorResponse(HttpStatus.BAD_REQUEST, BadRequestDataResponse(this.message)) + generateErrorResponse(HttpStatus.BAD_REQUEST, BadRequestDataResponse(this.message), this.warnings) is OperationNotSupportedException -> - generateErrorResponse(HttpStatus.BAD_REQUEST, OperationNotSupportedResponse(this.message)) + generateErrorResponse(HttpStatus.BAD_REQUEST, OperationNotSupportedResponse(this.message), this.warnings) is AccessDeniedException -> - generateErrorResponse(HttpStatus.FORBIDDEN, AccessDeniedResponse(this.message)) + generateErrorResponse(HttpStatus.FORBIDDEN, AccessDeniedResponse(this.message), this.warnings) is NotImplementedException -> - generateErrorResponse(HttpStatus.NOT_IMPLEMENTED, NotImplementedResponse(this.message)) + generateErrorResponse(HttpStatus.NOT_IMPLEMENTED, NotImplementedResponse(this.message), this.warnings) is LdContextNotAvailableException -> - generateErrorResponse(HttpStatus.SERVICE_UNAVAILABLE, LdContextNotAvailableResponse(this.message)) + generateErrorResponse( + HttpStatus.SERVICE_UNAVAILABLE, + LdContextNotAvailableResponse(this.message), + this.warnings + ) is TooManyResultsException -> - generateErrorResponse(HttpStatus.FORBIDDEN, TooManyResultsResponse(this.message)) - is ContextSourceRequestException -> - generateErrorResponse(this.status, ContextSourceRequestResponse(this.message)) - else -> generateErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, InternalErrorResponse("$cause")) + generateErrorResponse(HttpStatus.FORBIDDEN, TooManyResultsResponse(this.message), this.warnings) + else -> generateErrorResponse( + HttpStatus.INTERNAL_SERVER_ERROR, + InternalErrorResponse("$cause"), + this.warnings + ) } -private fun generateErrorResponse(status: HttpStatus, exception: ErrorResponse): ResponseEntity<*> { +private fun generateErrorResponse( + status: HttpStatus, + exception: ErrorResponse, + warnings: List? +): ResponseEntity<*> { logger.info("Returning error ${exception.type} (${exception.detail})") - return ResponseEntity.status(status) + val response = ResponseEntity.status(status) .contentType(MediaType.APPLICATION_JSON) - .body(serializeObject(exception)) + warnings?.addToResponse(response) + return response.body(serializeObject(exception)) } fun missingPathErrorResponse(errorMessage: String): ResponseEntity<*> { @@ -158,7 +169,12 @@ fun buildQueryResponse( else responseHeaders.body(body) } -fun prepareGetSuccessResponseHeaders(mediaType: MediaType, contexts: List): ResponseEntity.BodyBuilder = +fun prepareGetSuccessResponseHeaders( + mediaType: MediaType, + contexts: + List, + warnings: List? = null +): ResponseEntity.BodyBuilder = ResponseEntity.status(HttpStatus.OK) .apply { if (mediaType == JSON_LD_MEDIA_TYPE) { @@ -167,4 +183,6 @@ fun prepareGetSuccessResponseHeaders(mediaType: MediaType, contexts: List.addToResponse(response: BodyBuilder) { + if (this.isNotEmpty()) response.header(NGSILDWarning.HEADER_NAME, *this.map { it.message }.toTypedArray()) +} From 83871303c663a9cf56ead2beae983a04b6f24745 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 16 Oct 2024 18:04:54 +0200 Subject: [PATCH 09/19] feat: updating --- .../csr/model/ContextSourceRegistration.kt | 17 ++++++++++++- .../ContextSourceRegistrationService.kt | 24 +++++++++++++++++++ .../search/entity/web/EntityHandler.kt | 6 +++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt index 0cc55b154..e482c61a1 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt @@ -13,6 +13,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdTerm import com.egm.stellio.shared.util.JsonUtils.deserializeAs import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.fasterxml.jackson.annotation.JsonFormat +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.module.kotlin.convertValue import org.springframework.http.MediaType import java.net.URI @@ -30,7 +31,13 @@ data class ContextSourceRegistration( val createdAt: ZonedDateTime = ngsiLdDateTime(), val modifiedAt: ZonedDateTime? = null, val observationInterval: TimeInterval? = null, - val managementInterval: TimeInterval? = null + val managementInterval: TimeInterval? = null, + + var status: StatusType? = null, + val timesSent: Int = 0, + val timesFailed: Int = 0, + val lastFailure: ZonedDateTime? = null, + val lastSuccess: ZonedDateTime? = null, ) { data class TimeInterval( @@ -156,6 +163,14 @@ data class ContextSourceRegistration( fun alreadyExistsMessage(id: URI) = "A CSourceRegistration with id $id already exists" fun unauthorizedMessage(id: URI) = "User is not authorized to access CSourceRegistration $id" } + + enum class StatusType(val status: String) { + @JsonProperty("ok") + OK("ok"), + + @JsonProperty("failed") + FAILED("failed") + } } fun List.serialize( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index 32d908e86..865181ebb 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -16,14 +16,18 @@ import com.egm.stellio.shared.util.Sub import com.egm.stellio.shared.util.mapper import com.egm.stellio.shared.util.toStringValue import io.r2dbc.postgresql.codec.Json +import kotlinx.coroutines.reactive.awaitFirst import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.data.relational.core.query.Criteria.where import org.springframework.data.relational.core.query.Query.query +import org.springframework.data.relational.core.query.Update import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URI +import java.time.Instant +import java.time.ZoneOffset @Component class ContextSourceRegistrationService( @@ -234,4 +238,24 @@ class ContextSourceRegistrationService( }, ) } + + suspend fun updateContextSourceStatus( + csr: ContextSourceRegistration, + success: Boolean + ): Long { + val updateStatement = if (success) + Update.update("status", ContextSourceRegistration.StatusType.OK.name) + .set("times_sent", csr.timesSent + 1) + .set("last_success", Instant.now().atZone(ZoneOffset.UTC)) + else Update.update("status", ContextSourceRegistration.StatusType.FAILED.name) + .set("times_sent", csr.timesSent + 1) + .set("times_failed", csr.timesFailed + 1) + .set("last_failure", Instant.now().atZone(ZoneOffset.UTC)) + + return r2dbcEntityTemplate.update( + query(where("id").`is`(csr.id)), + updateStatement, + ContextSourceRegistration::class.java + ).awaitFirst() + } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 2cd19b77b..807310540 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -219,12 +219,14 @@ class EntityHandler( // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time val (entitiesWithMode, warnings) = matchingCSR.parMap { csr -> - ContextSourceUtils.getDistributedInformation( + val response = ContextSourceUtils.getDistributedInformation( httpHeaders, csr, "/ngsi-ld/v1/entities/$entityId", params - ) to csr.mode + ) + contextSourceRegistrationService.updateContextSourceStatus(csr, response.isRight()) + response to csr.mode }.partition { it.first.isRight() } .let { (responses, warnings) -> responses.map { (response, mode) -> response.getOrNull()!! to mode } to From dd315763bac125d571dac28842c2bc9b0a196818 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 21 Oct 2024 17:52:56 +0200 Subject: [PATCH 10/19] feat: MR comment - clearer name - no warning impact in ApiException - merging return warnings --- docker-compose.yml | 1 + search-service/build.gradle.kts | 1 + .../ContextSourceRegistrationHandler.kt | 1 - .../ContextSourceRegistrationService.kt | 12 +-- .../search/csr/service/ContextSourceUtils.kt | 99 ++++++++++--------- .../search/entity/web/EntityHandler.kt | 26 +++-- .../csr/service/ContextSourceCallerTests.kt | 80 +++++++++++++++ .../ContextSourceRegistrationServiceTests.kt | 6 -- .../csr/service/ContextSourceUtilsTests.kt | 27 ++--- .../search/entity/web/EntityHandlerTests.kt | 2 +- shared/config/detekt/baseline.xml | 2 +- .../egm/stellio/shared/model/ApiExceptions.kt | 2 - .../egm/stellio/shared/model/ErrorResponse.kt | 5 +- .../egm/stellio/shared/util/ApiResponses.kt | 31 +++--- .../egm/stellio/shared/util/NGSILDWarning.kt | 16 ++- 15 files changed, 200 insertions(+), 111 deletions(-) create mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt diff --git a/docker-compose.yml b/docker-compose.yml index 0022856cf..086c33efe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,7 @@ services: environment: - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} ports: +# todo define configurable port for easy CSR test - "8080:8080" search-service: container_name: stellio-search-service diff --git a/search-service/build.gradle.kts b/search-service/build.gradle.kts index e951031ce..6199180d2 100644 --- a/search-service/build.gradle.kts +++ b/search-service/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { runtimeOnly("org.postgresql:postgresql") runtimeOnly("io.r2dbc:r2dbc-pool") + testImplementation("org.wiremock:wiremock-standalone:3.3.1") testImplementation("org.testcontainers:postgresql") testImplementation("org.testcontainers:kafka") testImplementation("org.testcontainers:r2dbc") diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt index 110ac5818..bc87b75fd 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/handler/ContextSourceRegistrationHandler.kt @@ -74,7 +74,6 @@ class ContextSourceRegistrationHandler( val contextSourceRegistrations = contextSourceRegistrationService.getContextSourceRegistrations( limit = paginationQuery.limit, offset = paginationQuery.offset, - sub = sub, ).serialize(contexts, mediaType, includeSysAttrs) val contextSourceRegistrationsCount = contextSourceRegistrationService.getContextSourceRegistrationsCount( sub diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index 865181ebb..adf521738 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -14,6 +14,7 @@ import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.Sub import com.egm.stellio.shared.util.mapper +import com.egm.stellio.shared.util.ngsiLdDateTime import com.egm.stellio.shared.util.toStringValue import io.r2dbc.postgresql.codec.Json import kotlinx.coroutines.reactive.awaitFirst @@ -26,8 +27,6 @@ import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URI -import java.time.Instant -import java.time.ZoneOffset @Component class ContextSourceRegistrationService( @@ -163,7 +162,6 @@ class ContextSourceRegistrationService( } suspend fun getContextSourceRegistrations( - sub: Option, filters: CSRFilters = CSRFilters(), limit: Int = Int.MAX_VALUE, offset: Int = 0, @@ -188,8 +186,7 @@ class ContextSourceRegistrationService( as information(entities jsonb, propertyNames text[], relationshipNames text[]) on true LEFT JOIN jsonb_to_recordset(entities) as entity_info(id text, idPattern text, type text) on true - WHERE sub = :sub - AND $filterQuery + WHERE $filterQuery GROUP BY csr.id ORDER BY csr.id LIMIT :limit @@ -198,7 +195,6 @@ class ContextSourceRegistrationService( return databaseClient.sql(selectStatement) .bind("limit", limit) .bind("offset", offset) - .bind("sub", sub.toStringValue()) .allToMappedList { rowToContextSourceRegistration(it) } } @@ -246,11 +242,11 @@ class ContextSourceRegistrationService( val updateStatement = if (success) Update.update("status", ContextSourceRegistration.StatusType.OK.name) .set("times_sent", csr.timesSent + 1) - .set("last_success", Instant.now().atZone(ZoneOffset.UTC)) + .set("last_success", ngsiLdDateTime()) else Update.update("status", ContextSourceRegistration.StatusType.FAILED.name) .set("times_sent", csr.timesSent + 1) .set("times_failed", csr.timesFailed + 1) - .set("last_failure", Instant.now().atZone(ZoneOffset.UTC)) + .set("last_failure", ngsiLdDateTime()) return r2dbcEntityTemplate.update( query(where("id").`is`(csr.id)), diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index e8a578367..4f36c43ab 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -8,7 +8,6 @@ import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedAttributeInstances import com.egm.stellio.shared.model.CompactedEntity -import com.egm.stellio.shared.model.InternalErrorException import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID_TERM @@ -32,7 +31,8 @@ import java.time.ZonedDateTime import kotlin.random.Random.Default.nextBoolean typealias CompactedEntityWithMode = Pair - +typealias DataSetId = String? +typealias AttributeByDataSetId = Map object ContextSourceUtils { suspend fun getDistributedInformation( @@ -40,13 +40,15 @@ object ContextSourceUtils { csr: ContextSourceRegistration, path: String, params: MultiValueMap - ): Either = either { + ): Either = either { val uri = URI("${csr.endpoint}$path") + val request = WebClient.create() .method(HttpMethod.GET) .uri { uriBuilder -> uriBuilder.scheme(uri.scheme) .host(uri.host) + .port(uri.port) .path(uri.path) .queryParams(params) .build() @@ -64,7 +66,7 @@ object ContextSourceUtils { } statusCode.isSameCodeAs(HttpStatus.NOT_FOUND) -> { logger.info("CSR returned 404 at $uri: $response") - null.left() + null.right() } else -> { logger.warn("Error contacting CSR at $uri: $response") @@ -87,18 +89,18 @@ object ContextSourceUtils { fun mergeEntity( localEntity: CompactedEntity?, - entitiesWithMode: List - ): CompactedEntity? { - if (localEntity == null && entitiesWithMode.isEmpty()) return null + remoteEntitiesWithMode: List + ): Either = either { + if (localEntity == null && remoteEntitiesWithMode.isEmpty()) return@either null val mergedEntity = localEntity?.toMutableMap() ?: mutableMapOf() - entitiesWithMode.sortedBy { (_, mode) -> mode == Mode.AUXILIARY } + remoteEntitiesWithMode.sortedBy { (_, mode) -> mode == Mode.AUXILIARY } .forEach { (entity, mode) -> entity.entries.forEach { (key, value) -> val mergedValue = mergedEntity[key] - when { + when { // todo sysAttrs mergedValue == null -> mergedEntity[key] = value key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> {} key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> @@ -107,64 +109,71 @@ object ContextSourceUtils { mergedValue, value, mode == Mode.AUXILIARY - ) + ).bind() } } } - return mergedEntity + mergedEntity } fun mergeTypeOrScope( - type1: Any, // String || List || Set - type2: Any + currentValue: Any, // String || List || Set + remoteValue: Any ) = when { - type1 is List<*> && type2 is List<*> -> (type1.toSet() + type2.toSet()).toList() - type1 is List<*> -> (type1.toSet() + type2).toList() - type2 is List<*> -> (type2.toSet() + type1).toList() - type1 == type2 -> type1 - else -> listOf(type1, type2) + currentValue == remoteValue -> currentValue + currentValue is List<*> && remoteValue is List<*> -> (currentValue.toSet() + remoteValue.toSet()).toList() + currentValue is List<*> -> (currentValue.toSet() + remoteValue).toList() + remoteValue is List<*> -> (remoteValue.toSet() + currentValue).toList() + else -> listOf(currentValue, remoteValue) } /** * Implements 4.5.5 - Multi-Attribute Support */ fun mergeAttribute( - attribute1: Any, - attribute2: Any, + currentAttribute: Any, + remoteAttribute: Any, isAuxiliary: Boolean = false - ): Any { - val mergeMap = attributeToDatasetIdMap(attribute1).toMutableMap() - val attribute2Map = attributeToDatasetIdMap(attribute2) - attribute2Map.entries.forEach { (datasetId, value2) -> - val value1 = mergeMap[datasetId] + ): Either = either { + val currentInstances = groupInstancesByDataSetId(currentAttribute).bind().toMutableMap() + val remoteInstances = groupInstancesByDataSetId(remoteAttribute).bind() + remoteInstances.entries.forEach { (datasetId, remoteInstance) -> + val currentInstance = currentInstances[datasetId] when { - value1 == null -> mergeMap[datasetId] = value2 + currentInstance == null -> currentInstances[datasetId] = remoteInstance isAuxiliary -> {} - value1.isBefore(value2, NGSILD_OBSERVED_AT_TERM) -> mergeMap[datasetId] = value2 - value2.isBefore(value1, NGSILD_OBSERVED_AT_TERM) -> {} - value1.isBefore(value2, NGSILD_MODIFIED_AT_TERM) -> mergeMap[datasetId] = value2 - value2.isBefore(value1, NGSILD_MODIFIED_AT_TERM) -> {} - nextBoolean() -> mergeMap[datasetId] = value2 + currentInstance.isBefore(remoteInstance, NGSILD_OBSERVED_AT_TERM) -> + currentInstances[datasetId] = remoteInstance + remoteInstance.isBefore(currentInstance, NGSILD_OBSERVED_AT_TERM) -> {} + currentInstance.isBefore(remoteInstance, NGSILD_MODIFIED_AT_TERM) -> + currentInstances[datasetId] = remoteInstance + remoteInstance.isBefore(currentInstance, NGSILD_MODIFIED_AT_TERM) -> {} + // if there is no discriminating factor choose one at random + nextBoolean() -> currentInstances[datasetId] = remoteInstance else -> {} } } - val values = mergeMap.values.toList() - return if (values.size == 1) values[0] else values + val values = currentInstances.values.toList() + if (values.size == 1) values[0] else values } - private fun attributeToDatasetIdMap(attribute: Any): Map = when (attribute) { - is Map<*, *> -> { - attribute as CompactedAttributeInstance - mapOf(attribute[NGSILD_DATASET_ID_TERM] as? String to attribute) - } - is List<*> -> { - attribute as CompactedAttributeInstances - attribute.associateBy { it[NGSILD_DATASET_ID_TERM] as? String } + // do not work with CORE MEMBER since they are nor list nor map + private fun groupInstancesByDataSetId(attribute: Any): Either = + when (attribute) { + is Map<*, *> -> { + attribute as CompactedAttributeInstance + mapOf(attribute[NGSILD_DATASET_ID_TERM] as? String to attribute).right() + } + is List<*> -> { + attribute as CompactedAttributeInstances + attribute.associateBy { it[NGSILD_DATASET_ID_TERM] as? String }.right() + } + else -> { + RevalidationFailedWarning( + "The received payload is invalid. Attribute is nor List nor a Map : $attribute" + ).left() + } } - else -> throw InternalErrorException( - "the attribute is nor a list nor a map, check that you have excluded the CORE Members" - ) - } private fun CompactedAttributeInstance.isBefore( attr: CompactedAttributeInstance, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 807310540..1e2998704 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -203,7 +203,6 @@ class EntityHandler( val csrFilters = CSRFilters(setOf(entityId)) val matchingCSR = contextSourceRegistrationService.getContextSourceRegistrations( - sub, csrFilters ).filter { csr -> csr.operations.any { it == Operation.FEDERATION_OPS || it == Operation.RETRIEVE_ENTITY } } @@ -218,7 +217,7 @@ class EntityHandler( } // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time - val (entitiesWithMode, warnings) = matchingCSR.parMap { csr -> + val (remoteEntitiesWithMode, warnings) = matchingCSR.parMap { csr -> val response = ContextSourceUtils.getDistributedInformation( httpHeaders, csr, @@ -227,24 +226,31 @@ class EntityHandler( ) contextSourceRegistrationService.updateContextSourceStatus(csr, response.isRight()) response to csr.mode - }.partition { it.first.isRight() } + }.partition { it.first.getOrNull() != null } .let { (responses, warnings) -> responses.map { (response, mode) -> response.getOrNull()!! to mode } to - warnings.mapNotNull { (warning, _) -> warning.swap().getOrNull() } + warnings.mapNotNull { (warning, _) -> warning.leftOrNull() }.toMutableList() } - val localError = localEntity.swap().getOrNull() - if (localError != null && entitiesWithMode.isEmpty()) { - localError.warnings = warnings - localError.left().bind() + val localError = localEntity.leftOrNull() + if (localError != null && remoteEntitiesWithMode.isEmpty()) { + val error = localError.toErrorResponse() + if (warnings.isNotEmpty()) { + error.headers.addAll(NGSILDWarning.HEADER_NAME, warnings.getHeaderMessages()) + } + + return error } - val mergedEntity = ContextSourceUtils.mergeEntity(localEntity.getOrNull(), entitiesWithMode)!! + val mergedEntity = ContextSourceUtils.mergeEntity( + localEntity.getOrNull(), + remoteEntitiesWithMode + ).onLeft { warnings.add(it) }.getOrNull() // todo treat warning case val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) prepareGetSuccessResponseHeaders(mediaType, contexts, warnings) .body( - serializeObject(mergedEntity.toFinalRepresentation(ngsiLdDataRepresentation)) + serializeObject(mergedEntity!!.toFinalRepresentation(ngsiLdDataRepresentation)) ) }.fold( { it.toErrorResponse() }, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt new file mode 100644 index 000000000..5d941473e --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt @@ -0,0 +1,80 @@ +package com.egm.stellio.search.csr.service + +import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.util.JsonUtils.serializeObject +import com.github.tomakehurst.wiremock.client.WireMock.* +import com.github.tomakehurst.wiremock.junit5.WireMockTest +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.springframework.http.HttpHeaders +import org.springframework.http.MediaType.APPLICATION_JSON_VALUE +import org.springframework.test.context.ActiveProfiles +import org.springframework.util.LinkedMultiValueMap +import wiremock.com.google.common.net.HttpHeaders.CONTENT_TYPE + +@WireMockTest(httpPort = 8089) +@ActiveProfiles("test") +class ContextSourceCallerTests { + + private val apiaryId = "urn:ngsi-ld:Apiary:TEST" + + fun gimmeRawCSR() = ContextSourceRegistration( + id = "urn:ngsi-ld:ContextSourceRegistration:test".toUri(), + endpoint = "http://localhost:8089".toUri(), + information = emptyList(), + operations = listOf(Operation.FEDERATION_OPS), + createdAt = ngsiLdDateTime(), + + ) + val emptyParams = LinkedMultiValueMap() + private val entityWithSysAttrs = + """ + { + "id":"$apiaryId", + "type":"Apiary", + "createdAt": "2024-02-13T18:15:00Z", + "modifiedAt": "2024-02-13T18:16:00Z", + "name": { + "type":"Property", + "value":"ApiarySophia", + "createdAt": "2024-02-13T18:15:00Z", + "modifiedAt": "2024-02-13T18:16:00Z" + }, + "@context":[ "$APIC_COMPOUND_CONTEXT" ] + } + """.trimIndent() + + @Test + fun `getDistributedInformation should return the entity when the request succeed`() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + stubFor( + get(urlMatching(path)) + .willReturn( + ok() + .withHeader(CONTENT_TYPE, APPLICATION_JSON_VALUE).withBody(entityWithSysAttrs) + ) + ) + + val response = ContextSourceUtils.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + assertJsonPayloadsAreEqual(entityWithSysAttrs, serializeObject(response.getOrNull()!!)) + } + + @Test + fun `getDistributedInformation should fail with a `() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + stubFor( + get(urlMatching(path)) + .willReturn( + ok() + .withHeader(CONTENT_TYPE, APPLICATION_JSON_VALUE).withBody(entityWithSysAttrs) + ) + ) + + val response = ContextSourceUtils.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + assertJsonPayloadsAreEqual(entityWithSysAttrs, serializeObject(response.getOrNull()!!)) + } +} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt index bb838aa70..c82c0e44d 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt @@ -99,7 +99,6 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( - mockUserSub, CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) ) @@ -135,7 +134,6 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( - mockUserSub, CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) ) @@ -167,7 +165,6 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( - mockUserSub, CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) ) @@ -203,7 +200,6 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( - mockUserSub, CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) ) @@ -239,7 +235,6 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( - mockUserSub, CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A456".toUri())) ) @@ -253,7 +248,6 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer { contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed() val matchingCsrs = contextSourceRegistrationService.getContextSourceRegistrations( - mockUserSub, CSRFilters(ids = setOf("urn:ngsi-ld:Vehicle:A457".toUri())) ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt index 46c7ebf19..bb565a737 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -1,5 +1,6 @@ package com.egm.stellio.search.csr.service +import arrow.core.right import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedEntity @@ -56,13 +57,13 @@ class ContextSourceUtilsTests { @Test fun `merge entity should return localEntity when no other entities is provided`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntity(baseEntity, emptyList()) - assertEquals(baseEntity, mergedEntity) + assertEquals(baseEntity, mergedEntity.getOrNull()) } @Test fun `merge entity should merge the localEntity with the list of entities `() = runTest { val mergedEntity = ContextSourceUtils.mergeEntity(minimalEntity, listOf(baseEntity to Mode.AUXILIARY)) - assertEquals(baseEntity, mergedEntity) + assertEquals(baseEntity, mergedEntity.getOrNull()) } @Test @@ -71,7 +72,7 @@ class ContextSourceUtilsTests { entityWithName, listOf(entityWithLastName to Mode.AUXILIARY, entityWithSurName to Mode.INCLUSIVE) ) - assertEquals(entityWithName + entityWithLastName + entityWithSurName, mergedEntity) + assertEquals(entityWithName + entityWithLastName + entityWithSurName, mergedEntity.getOrNull()) } @Test @@ -79,7 +80,7 @@ class ContextSourceUtilsTests { mockkObject(ContextSourceUtils) { every { ContextSourceUtils.mergeAttribute(any(), any(), any()) } returns listOf( nameAttribute - ) + ).right() every { ContextSourceUtils.mergeTypeOrScope(any(), any()) } returns listOf("Beehive") ContextSourceUtils.mergeEntity( entityWithName, @@ -95,8 +96,8 @@ class ContextSourceUtilsTests { val mergedEntity = ContextSourceUtils.mergeEntity( minimalEntity, listOf(multipleTypeEntity to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) - )!! - assertThat(mergedEntity[JSONLD_TYPE_TERM] as List<*>) + ).getOrNull() + assertThat(mergedEntity?.get(JSONLD_TYPE_TERM) as List<*>) .hasSize(3) .contains("Sensor", "BeeHive", "Beekeeper") } @@ -112,8 +113,10 @@ class ContextSourceUtilsTests { val mergedEntity = ContextSourceUtils.mergeEntity( entityWithName, listOf(entityWithDifferentName to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) - )!! - assertThat(mergedEntity[name] as List<*>).hasSize(3).contains(nameAttribute, nameAttribute2, baseEntity[name]) + ).getOrNull() + assertThat( + mergedEntity?.get(name) as List<*> + ).hasSize(3).contains(nameAttribute, nameAttribute2, baseEntity[name]) } @Test @@ -121,10 +124,10 @@ class ContextSourceUtilsTests { val mergedEntity = ContextSourceUtils.mergeEntity( entityWithName, listOf(evenMoreRecentEntity to Mode.EXCLUSIVE, moreRecentEntity to Mode.INCLUSIVE) - )!! + ).getOrNull() assertEquals( "2030-01-01T01:01:01.01Z", - (mergedEntity[name] as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] + (mergedEntity?.get(name) as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] ) assertEquals("evenMoreRecentName", (mergedEntity[name] as CompactedAttributeInstance)[JSONLD_VALUE_TERM]) } @@ -134,10 +137,10 @@ class ContextSourceUtilsTests { val mergedEntity = ContextSourceUtils.mergeEntity( entityWithName, listOf(evenMoreRecentEntity to Mode.AUXILIARY, moreRecentEntity to Mode.AUXILIARY) - )!! + ).getOrNull() assertEquals( "2010-01-01T01:01:01.01Z", - (mergedEntity[name] as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] + (mergedEntity?.get(name) as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] ) assertEquals("name", (mergedEntity[name] as CompactedAttributeInstance)[JSONLD_VALUE_TERM]) } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt index 1502e15f0..38d46b112 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt @@ -81,7 +81,7 @@ class EntityHandlerTests { fun mockCSR() { coEvery { contextSourceRegistrationService - .getContextSourceRegistrations(any(), any(), any(), any()) + .getContextSourceRegistrations(any(), any(), any()) } returns listOf() } diff --git a/shared/config/detekt/baseline.xml b/shared/config/detekt/baseline.xml index 4a79679ab..6081a74e0 100644 --- a/shared/config/detekt/baseline.xml +++ b/shared/config/detekt/baseline.xml @@ -8,7 +8,7 @@ LongParameterList:ApiResponses.kt$( body: String, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) LongParameterList:ApiResponses.kt$( entities: Any, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) SpreadOperator:EntityEvent.kt$EntityEvent$( *[ JsonSubTypes.Type(value = EntityCreateEvent::class), JsonSubTypes.Type(value = EntityReplaceEvent::class), JsonSubTypes.Type(value = EntityDeleteEvent::class), JsonSubTypes.Type(value = AttributeAppendEvent::class), JsonSubTypes.Type(value = AttributeReplaceEvent::class), JsonSubTypes.Type(value = AttributeUpdateEvent::class), JsonSubTypes.Type(value = AttributeDeleteEvent::class), JsonSubTypes.Type(value = AttributeDeleteAllInstancesEvent::class) ] ) - SpreadOperator:NGSILDWarning.kt$(NGSILDWarning.HEADER_NAME, *this.map { it.message }.toTypedArray()) + SpreadOperator:NGSILDWarning.kt$( NGSILDWarning.HEADER_NAME, *this.getHeaderMessages().toTypedArray() ) SwallowedException:JsonLdUtils.kt$JsonLdUtils$e: JsonLdError TooManyFunctions:JsonLdUtils.kt$JsonLdUtils diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt index 5a79b7e2c..79e59056a 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt @@ -2,11 +2,9 @@ package com.egm.stellio.shared.model import com.apicatalog.jsonld.JsonLdError import com.apicatalog.jsonld.JsonLdErrorCode -import com.egm.stellio.shared.util.NGSILDWarning sealed class APIException( override val message: String, - var warnings: List? = null ) : Exception(message) data class InvalidRequestException(override val message: String) : APIException(message) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt index e47458798..9eb79ba97 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt @@ -5,7 +5,7 @@ import java.net.URI sealed class ErrorResponse( val type: URI, open val title: String, - open val detail: String, + open val detail: String ) data class InvalidRequestResponse(override val detail: String) : ErrorResponse( @@ -98,7 +98,7 @@ data class NonexistentTenantResponse(override val detail: String) : ErrorResponse( ErrorType.NONEXISTENT_TENANT.type, "The addressed tenant does not exist", - detail, + detail ) enum class ErrorType(val type: URI) { @@ -116,5 +116,4 @@ enum class ErrorType(val type: URI) { UNSUPPORTED_MEDIA_TYPE(URI("https://uri.etsi.org/ngsi-ld/errors/UnsupportedMediaType")), NOT_ACCEPTABLE(URI("https://uri.etsi.org/ngsi-ld/errors/NotAcceptable")), NONEXISTENT_TENANT(URI("https://uri.etsi.org/ngsi-ld/errors/NonexistentTenant")), - CONTEXT_SOURCE_REQUEST(URI("https://uri.etsi.org/ngsi-ld/errors/ContextSourceRequest")) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index b60223633..de5c55fe4 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -70,44 +70,39 @@ private val logger = LoggerFactory.getLogger("com.egm.stellio.shared.util.ApiRes fun APIException.toErrorResponse(): ResponseEntity<*> = when (this) { is AlreadyExistsException -> - generateErrorResponse(HttpStatus.CONFLICT, AlreadyExistsResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.CONFLICT, AlreadyExistsResponse(this.message)) is ResourceNotFoundException -> - generateErrorResponse(HttpStatus.NOT_FOUND, ResourceNotFoundResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.NOT_FOUND, ResourceNotFoundResponse(this.message)) is InvalidRequestException -> - generateErrorResponse(HttpStatus.BAD_REQUEST, InvalidRequestResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.BAD_REQUEST, InvalidRequestResponse(this.message)) is BadRequestDataException -> - generateErrorResponse(HttpStatus.BAD_REQUEST, BadRequestDataResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.BAD_REQUEST, BadRequestDataResponse(this.message)) is OperationNotSupportedException -> - generateErrorResponse(HttpStatus.BAD_REQUEST, OperationNotSupportedResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.BAD_REQUEST, OperationNotSupportedResponse(this.message)) is AccessDeniedException -> - generateErrorResponse(HttpStatus.FORBIDDEN, AccessDeniedResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.FORBIDDEN, AccessDeniedResponse(this.message)) is NotImplementedException -> - generateErrorResponse(HttpStatus.NOT_IMPLEMENTED, NotImplementedResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.NOT_IMPLEMENTED, NotImplementedResponse(this.message)) is LdContextNotAvailableException -> generateErrorResponse( HttpStatus.SERVICE_UNAVAILABLE, - LdContextNotAvailableResponse(this.message), - this.warnings + LdContextNotAvailableResponse(this.message) ) is TooManyResultsException -> - generateErrorResponse(HttpStatus.FORBIDDEN, TooManyResultsResponse(this.message), this.warnings) + generateErrorResponse(HttpStatus.FORBIDDEN, TooManyResultsResponse(this.message)) else -> generateErrorResponse( HttpStatus.INTERNAL_SERVER_ERROR, - InternalErrorResponse("$cause"), - this.warnings + InternalErrorResponse("$cause") ) } -private fun generateErrorResponse( +fun generateErrorResponse( status: HttpStatus, exception: ErrorResponse, - warnings: List? ): ResponseEntity<*> { logger.info("Returning error ${exception.type} (${exception.detail})") - val response = ResponseEntity.status(status) - .contentType(MediaType.APPLICATION_JSON) - warnings?.addToResponse(response) - return response.body(serializeObject(exception)) + return ResponseEntity.status(status) + .contentType(MediaType.APPLICATION_JSON).body(serializeObject(exception)) } fun missingPathErrorResponse(errorMessage: String): ResponseEntity<*> { diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt index 533ba488c..ea2c9a58c 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt @@ -1,11 +1,14 @@ package com.egm.stellio.shared.util import org.springframework.http.ResponseEntity.BodyBuilder +import java.util.* -// todo i took the name from the spec could also be name APIWarning like ApiException open class NGSILDWarning( - override val message: String -) : Exception(message) { + open val message: String +) { + // new line are forbidden in headers + fun getHeaderMessage(): String = Base64.getEncoder().encodeToString(message.toByteArray()) + companion object { const val HEADER_NAME = "NGSILD-Warning" } @@ -16,6 +19,11 @@ data class RevalidationFailedWarning(override val message: String) : NGSILDWarni data class MiscellaneousWarning(override val message: String) : NGSILDWarning(message) data class MiscellaneousPersistentWarning(override val message: String) : NGSILDWarning(message) +fun List.getHeaderMessages() = this.map { it.getHeaderMessage() } + fun List.addToResponse(response: BodyBuilder) { - if (this.isNotEmpty()) response.header(NGSILDWarning.HEADER_NAME, *this.map { it.message }.toTypedArray()) + if (this.isNotEmpty()) response.header( + NGSILDWarning.HEADER_NAME, + *this.getHeaderMessages().toTypedArray() + ) } From 3bcf7b3b9708f03a4035e93e1c1fe554950ddd20 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Tue, 22 Oct 2024 23:08:15 +0200 Subject: [PATCH 11/19] feat: get warnings from merge --- .../search/csr/service/ContextSourceUtils.kt | 56 +++++++++++-------- .../search/entity/web/EntityHandler.kt | 20 +++---- 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 4f36c43ab..d33946c51 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -1,9 +1,8 @@ package com.egm.stellio.search.csr.service -import arrow.core.Either -import arrow.core.left +import arrow.core.* import arrow.core.raise.either -import arrow.core.right +import arrow.core.raise.iorNel import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedAttributeInstances @@ -87,33 +86,44 @@ object ContextSourceUtils { } } - fun mergeEntity( + fun mergeEntities( localEntity: CompactedEntity?, remoteEntitiesWithMode: List - ): Either = either { - if (localEntity == null && remoteEntitiesWithMode.isEmpty()) return@either null + ): IorNel = iorNel { + if (localEntity == null && remoteEntitiesWithMode.isEmpty()) return@iorNel null - val mergedEntity = localEntity?.toMutableMap() ?: mutableMapOf() + val mergedEntity: MutableMap = localEntity?.toMutableMap() ?: mutableMapOf() remoteEntitiesWithMode.sortedBy { (_, mode) -> mode == Mode.AUXILIARY } .forEach { (entity, mode) -> - entity.entries.forEach { - (key, value) -> - val mergedValue = mergedEntity[key] - when { // todo sysAttrs - mergedValue == null -> mergedEntity[key] = value - key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> {} - key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> - mergedEntity[key] = mergeTypeOrScope(mergedValue, value) - else -> mergedEntity[key] = mergeAttribute( - mergedValue, - value, - mode == Mode.AUXILIARY - ).bind() - } - } + mergedEntity.putAll( + getMergeNewValues(mergedEntity, entity, mode).toIor().toIorNel().bind() + ) + } + + return@iorNel mergedEntity.toMap() + } + + private fun getMergeNewValues( + localEntity: CompactedEntity, + remoteEntity: CompactedEntity, + mode: Mode + ): Either = either { + remoteEntity.mapValues { (key, value) -> + val localValue = localEntity[key] + when { // todo sysAttrs + localValue == null -> value + key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> localValue + key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> + mergeTypeOrScope(localValue, value) + + else -> mergeAttribute( + localValue, + value, + mode == Mode.AUXILIARY + ).bind() } - mergedEntity + } } fun mergeTypeOrScope( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 1e2998704..7be7c5036 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -232,25 +232,25 @@ class EntityHandler( warnings.mapNotNull { (warning, _) -> warning.leftOrNull() }.toMutableList() } - val localError = localEntity.leftOrNull() - if (localError != null && remoteEntitiesWithMode.isEmpty()) { - val error = localError.toErrorResponse() + val (mergeWarnings, mergedEntity) = ContextSourceUtils.mergeEntities( + localEntity.getOrNull(), + remoteEntitiesWithMode + ).toPair() + + if (mergedEntity.isNullOrEmpty()) { + val localError = localEntity.leftOrNull() + val error = localError!!.toErrorResponse() + mergeWarnings?.let { warnings.addAll(it) } if (warnings.isNotEmpty()) { error.headers.addAll(NGSILDWarning.HEADER_NAME, warnings.getHeaderMessages()) } - return error } - val mergedEntity = ContextSourceUtils.mergeEntity( - localEntity.getOrNull(), - remoteEntitiesWithMode - ).onLeft { warnings.add(it) }.getOrNull() // todo treat warning case - val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) prepareGetSuccessResponseHeaders(mediaType, contexts, warnings) .body( - serializeObject(mergedEntity!!.toFinalRepresentation(ngsiLdDataRepresentation)) + serializeObject(mergedEntity.toFinalRepresentation(ngsiLdDataRepresentation)) ) }.fold( { it.toErrorResponse() }, From 690fd5911aee57bfec643b2fcf5f7c1a2f29fb12 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 23 Oct 2024 15:05:36 +0200 Subject: [PATCH 12/19] feat: separate context source utils + work with sysAttrs --- search-service/config/detekt/baseline.xml | 1 + .../search/csr/service/ContextSourceCaller.kt | 75 +++++++++++++++++ .../search/csr/service/ContextSourceUtils.kt | 84 ++++--------------- .../search/entity/web/EntityHandler.kt | 4 +- .../csr/service/ContextSourceCallerTests.kt | 71 ++++++++++++++-- .../csr/service/ContextSourceUtilsTests.kt | 78 +++++++++++++---- 6 files changed, 217 insertions(+), 96 deletions(-) create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml index c392b1a73..0fc5038e9 100644 --- a/search-service/config/detekt/baseline.xml +++ b/search-service/config/detekt/baseline.xml @@ -27,5 +27,6 @@ LongParameterList:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$( entityId: URI, attributeName: ExpandedTerm, datasetId: URI?, attributePayload: ExpandedAttributeInstance, ngsiLdAttributeInstance: NgsiLdAttributeInstance, defaultCreatedAt: ZonedDateTime ) NestedBlockDepth:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$override fun migrate(context: Context) SwallowedException:TemporalQueryUtils.kt$e: IllegalArgumentException + TooGenericExceptionCaught:ContextSourceCaller.kt$ContextSourceCaller$e: Exception diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt new file mode 100644 index 000000000..cbf385ed7 --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt @@ -0,0 +1,75 @@ +package com.egm.stellio.search.csr.service + +import arrow.core.* +import arrow.core.raise.either +import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.shared.model.CompactedEntity +import com.egm.stellio.shared.util.* +import org.springframework.core.codec.DecodingException +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.http.HttpStatus +import org.springframework.util.MultiValueMap +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.awaitBodyOrNull +import org.springframework.web.reactive.function.client.awaitExchange +import java.net.URI + +object ContextSourceCaller { + + suspend fun getDistributedInformation( + httpHeaders: HttpHeaders, + csr: ContextSourceRegistration, + path: String, + params: MultiValueMap + ): Either = either { + val uri = URI("${csr.endpoint}$path") + + val request = WebClient.create() + .method(HttpMethod.GET) + .uri { uriBuilder -> + uriBuilder.scheme(uri.scheme) + .host(uri.host) + .port(uri.port) + .path(uri.path) + .queryParams(params) + .build() + } + .header(HttpHeaders.LINK, httpHeaders.getFirst(HttpHeaders.LINK)) + return try { + val (statusCode, response) = request + .awaitExchange { response -> + response.statusCode() to response.awaitBodyOrNull() + } + when { + statusCode.is2xxSuccessful -> { + JsonLdUtils.logger.info("Successfully received response from CSR at $uri") + response.right() + } + + statusCode.isSameCodeAs(HttpStatus.NOT_FOUND) -> { + JsonLdUtils.logger.info("CSR returned 404 at $uri: $response") + null.right() + } + + else -> { + JsonLdUtils.logger.warn("Error contacting CSR at $uri: $response") + + MiscellaneousPersistentWarning( + "the CSR ${csr.id} returned an error $statusCode at : $uri response: \"$response\"" + ).left() + } + } + } catch (e: Exception) { + when (e) { + is DecodingException -> RevalidationFailedWarning( + "the CSR ${csr.id} as : $uri returned badly formed data message: \"${e.cause}:${e.message}\"" + ) + + else -> MiscellaneousWarning( + "Error connecting to csr ${csr.id} at : $uri message : \"${e.cause}:${e.message}\"" + ) + }.left() + } + } +} diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index d33946c51..9124f9337 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -11,21 +11,11 @@ import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_TERM -import com.egm.stellio.shared.util.JsonLdUtils.logger -import com.fasterxml.jackson.core.JacksonException -import org.springframework.http.HttpHeaders -import org.springframework.http.HttpMethod -import org.springframework.http.HttpStatus -import org.springframework.util.MultiValueMap -import org.springframework.web.reactive.function.client.WebClient -import org.springframework.web.reactive.function.client.WebClientException -import org.springframework.web.reactive.function.client.awaitBody -import org.springframework.web.reactive.function.client.awaitExchange -import java.net.URI import java.time.ZonedDateTime import kotlin.random.Random.Default.nextBoolean @@ -34,58 +24,6 @@ typealias DataSetId = String? typealias AttributeByDataSetId = Map object ContextSourceUtils { - suspend fun getDistributedInformation( - httpHeaders: HttpHeaders, - csr: ContextSourceRegistration, - path: String, - params: MultiValueMap - ): Either = either { - val uri = URI("${csr.endpoint}$path") - - val request = WebClient.create() - .method(HttpMethod.GET) - .uri { uriBuilder -> - uriBuilder.scheme(uri.scheme) - .host(uri.host) - .port(uri.port) - .path(uri.path) - .queryParams(params) - .build() - } - .header(HttpHeaders.LINK, httpHeaders.getFirst(HttpHeaders.LINK)) - return try { - val (statusCode, response) = request - .awaitExchange { response -> - response.statusCode() to response.awaitBody() - } - when { - statusCode.is2xxSuccessful -> { - logger.info("Successfully received response from CSR at $uri") - response.right() - } - statusCode.isSameCodeAs(HttpStatus.NOT_FOUND) -> { - logger.info("CSR returned 404 at $uri: $response") - null.right() - } - else -> { - logger.warn("Error contacting CSR at $uri: $response") - - MiscellaneousPersistentWarning( - "the CSR ${csr.id} returned an error $statusCode at : $uri response: \"$response\"" - ).left() - } - } - } catch (e: WebClientException) { - MiscellaneousWarning( - "Error connecting to csr ${csr.id} at : $uri message : \"${e.message}\"" - ).left() - } catch (e: JacksonException) { // todo get the good exception for invalid payload - RevalidationFailedWarning( - "the CSR ${csr.id} as : $uri returned badly formed data message: \"${e.message}\"" - ).left() - } - } - fun mergeEntities( localEntity: CompactedEntity?, remoteEntitiesWithMode: List @@ -111,12 +49,17 @@ object ContextSourceUtils { ): Either = either { remoteEntity.mapValues { (key, value) -> val localValue = localEntity[key] - when { // todo sysAttrs + when { localValue == null -> value key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> localValue key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> mergeTypeOrScope(localValue, value) - + key == NGSILD_CREATED_AT_TERM -> + if ((value as String?).isBefore(localValue as String?)) value + else localValue + key == NGSILD_MODIFIED_AT_TERM -> + if ((localValue as String?).isBefore(value as String?)) value + else localValue else -> mergeAttribute( localValue, value, @@ -188,9 +131,10 @@ object ContextSourceUtils { private fun CompactedAttributeInstance.isBefore( attr: CompactedAttributeInstance, property: String - ): Boolean = ( - (this[property] as? String)?.isDateTime() == true && - (attr[property] as? String)?.isDateTime() == true && - ZonedDateTime.parse(this[property] as String) < ZonedDateTime.parse(attr[property] as String) - ) + ): Boolean = (this[property] as String?)?.isBefore(attr[property] as String?) == true + + private fun String?.isBefore(date: String?) = + this?.isDateTime() == true && + date?.isDateTime() == true && + ZonedDateTime.parse(this) < ZonedDateTime.parse(date) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 7be7c5036..b3b593758 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -7,6 +7,7 @@ import arrow.core.right import arrow.fx.coroutines.parMap import com.egm.stellio.search.csr.model.CSRFilters import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.search.csr.service.ContextSourceCaller import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.csr.service.ContextSourceUtils import com.egm.stellio.search.entity.service.EntityQueryService @@ -218,7 +219,7 @@ class EntityHandler( // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time val (remoteEntitiesWithMode, warnings) = matchingCSR.parMap { csr -> - val response = ContextSourceUtils.getDistributedInformation( + val response = ContextSourceCaller.getDistributedInformation( httpHeaders, csr, "/ngsi-ld/v1/entities/$entityId", @@ -232,6 +233,7 @@ class EntityHandler( warnings.mapNotNull { (warning, _) -> warning.leftOrNull() }.toMutableList() } + // we could simplify the code if we check the JsonPayload beforehand val (mergeWarnings, mergedEntity) = ContextSourceUtils.mergeEntities( localEntity.getOrNull(), remoteEntitiesWithMode diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt index 5d941473e..52203f0cc 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt @@ -7,6 +7,7 @@ import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.junit5.WireMockTest import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test import org.springframework.http.HttpHeaders import org.springframework.http.MediaType.APPLICATION_JSON_VALUE @@ -20,7 +21,7 @@ class ContextSourceCallerTests { private val apiaryId = "urn:ngsi-ld:Apiary:TEST" - fun gimmeRawCSR() = ContextSourceRegistration( + private fun gimmeRawCSR() = ContextSourceRegistration( id = "urn:ngsi-ld:ContextSourceRegistration:test".toUri(), endpoint = "http://localhost:8089".toUri(), information = emptyList(), @@ -28,7 +29,7 @@ class ContextSourceCallerTests { createdAt = ngsiLdDateTime(), ) - val emptyParams = LinkedMultiValueMap() + private val emptyParams = LinkedMultiValueMap() private val entityWithSysAttrs = """ { @@ -46,6 +47,19 @@ class ContextSourceCallerTests { } """.trimIndent() + private val entityWithBadPayload = + """ + { + "id":"$apiaryId", + "type":"Apiary", + "name": { + "type":"Property", + "value":"ApiarySophia", + , + "@context":[ "$APIC_COMPOUND_CONTEXT" ] + } + """.trimIndent() + @Test fun `getDistributedInformation should return the entity when the request succeed`() = runTest { val csr = gimmeRawCSR() @@ -58,23 +72,66 @@ class ContextSourceCallerTests { ) ) - val response = ContextSourceUtils.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + val response = ContextSourceCaller.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) assertJsonPayloadsAreEqual(entityWithSysAttrs, serializeObject(response.getOrNull()!!)) } @Test - fun `getDistributedInformation should fail with a `() = runTest { + fun `getDistributedInformation should return a MiscellaneousWarning if it receive no answer`() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + + val response = ContextSourceCaller.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + + assertTrue(response.isLeft()) + assertInstanceOf(MiscellaneousWarning::class.java, response.leftOrNull()) + } + + @Test + fun `getDistributedInformation should return a RevalidationFailedWarning when receiving a bad payload`() = runTest { val csr = gimmeRawCSR() val path = "/ngsi-ld/v1/entities/$apiaryId" stubFor( get(urlMatching(path)) .willReturn( ok() - .withHeader(CONTENT_TYPE, APPLICATION_JSON_VALUE).withBody(entityWithSysAttrs) + .withHeader(CONTENT_TYPE, APPLICATION_JSON_VALUE).withBody(entityWithBadPayload) ) ) - val response = ContextSourceUtils.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) - assertJsonPayloadsAreEqual(entityWithSysAttrs, serializeObject(response.getOrNull()!!)) + val response = ContextSourceCaller.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + + assertTrue(response.isLeft()) + assertInstanceOf(RevalidationFailedWarning::class.java, response.leftOrNull()) + } + + @Test + fun `getDistributedInformation should return MiscellaneousPersistentWarning when receiving error 500`() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + stubFor( + get(urlMatching(path)) + .willReturn(unauthorized()) + ) + + val response = ContextSourceCaller.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + + assertTrue(response.isLeft()) + assertInstanceOf(MiscellaneousPersistentWarning::class.java, response.leftOrNull()) + } + + @Test + fun `getDistributedInformation should return null when receiving an error 404`() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + stubFor( + get(urlMatching(path)) + .willReturn(notFound()) + ) + + val response = ContextSourceCaller.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) + + assertTrue(response.isRight()) + assertNull(response.leftOrNull()) } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt index bb565a737..7ce7333e2 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -7,7 +7,9 @@ import com.egm.stellio.shared.model.CompactedEntity import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM import com.fasterxml.jackson.module.kotlin.readValue import io.mockk.every @@ -16,6 +18,7 @@ import io.mockk.verify import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.test.context.ActiveProfiles @@ -25,30 +28,32 @@ class ContextSourceUtilsTests { private val minimalEntity: CompactedEntity = mapper.readValue(loadSampleData("beehive_minimal.jsonld")) private val baseEntity: CompactedEntity = mapper.readValue(loadSampleData("beehive.jsonld")) private val multipleTypeEntity: CompactedEntity = mapper.readValue(loadSampleData("entity_with_multi_types.jsonld")) - + private val time = "2010-01-01T01:01:01.01Z" + private val moreRecentTime = "2020-01-01T01:01:01.01Z" + private val evenMoreRecentTime = "2030-01-01T01:01:01.01Z" private val nameAttribute: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "name", NGSILD_DATASET_ID_TERM to "1", - NGSILD_OBSERVED_AT_TERM to "2010-01-01T01:01:01.01Z" + NGSILD_OBSERVED_AT_TERM to time ) - val moreRecentAttribute: CompactedAttributeInstance = mapOf( + private val moreRecentAttribute: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "moreRecentName", NGSILD_DATASET_ID_TERM to "1", - NGSILD_OBSERVED_AT_TERM to "2020-01-01T01:01:01.01Z" + NGSILD_OBSERVED_AT_TERM to moreRecentTime ) - val evenMoreRecentAttribute: CompactedAttributeInstance = mapOf( + private val evenMoreRecentAttribute: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "evenMoreRecentName", NGSILD_DATASET_ID_TERM to "1", - NGSILD_OBSERVED_AT_TERM to "2030-01-01T01:01:01.01Z" + NGSILD_OBSERVED_AT_TERM to evenMoreRecentTime ) - val moreRecentEntity = minimalEntity.toMutableMap() + (name to moreRecentAttribute) - val evenMoreRecentEntity = minimalEntity.toMutableMap() + (name to evenMoreRecentAttribute) + private val moreRecentEntity = minimalEntity.toMutableMap() + (name to moreRecentAttribute) + private val evenMoreRecentEntity = minimalEntity.toMutableMap() + (name to evenMoreRecentAttribute) private val entityWithName = minimalEntity.toMutableMap().plus(name to nameAttribute) private val entityWithLastName = minimalEntity.toMutableMap().plus("lastName" to nameAttribute) @@ -56,19 +61,19 @@ class ContextSourceUtilsTests { @Test fun `merge entity should return localEntity when no other entities is provided`() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntity(baseEntity, emptyList()) + val mergedEntity = ContextSourceUtils.mergeEntities(baseEntity, emptyList()) assertEquals(baseEntity, mergedEntity.getOrNull()) } @Test fun `merge entity should merge the localEntity with the list of entities `() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntity(minimalEntity, listOf(baseEntity to Mode.AUXILIARY)) + val mergedEntity = ContextSourceUtils.mergeEntities(minimalEntity, listOf(baseEntity to Mode.AUXILIARY)) assertEquals(baseEntity, mergedEntity.getOrNull()) } @Test fun `merge entity should merge all the entities`() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntity( + val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, listOf(entityWithLastName to Mode.AUXILIARY, entityWithSurName to Mode.INCLUSIVE) ) @@ -82,7 +87,7 @@ class ContextSourceUtilsTests { nameAttribute ).right() every { ContextSourceUtils.mergeTypeOrScope(any(), any()) } returns listOf("Beehive") - ContextSourceUtils.mergeEntity( + ContextSourceUtils.mergeEntities( entityWithName, listOf(entityWithName to Mode.AUXILIARY, entityWithName to Mode.INCLUSIVE) ) @@ -93,7 +98,7 @@ class ContextSourceUtilsTests { @Test fun `merge entity should merge the types correctly `() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntity( + val mergedEntity = ContextSourceUtils.mergeEntities( minimalEntity, listOf(multipleTypeEntity to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) ).getOrNull() @@ -110,7 +115,7 @@ class ContextSourceUtilsTests { NGSILD_DATASET_ID_TERM to "2" ) val entityWithDifferentName = minimalEntity.toMutableMap() + (name to nameAttribute2) - val mergedEntity = ContextSourceUtils.mergeEntity( + val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, listOf(entityWithDifferentName to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) ).getOrNull() @@ -121,12 +126,12 @@ class ContextSourceUtilsTests { @Test fun `merge entity should merge attribute same datasetId keeping the most recentOne `() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntity( + val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, listOf(evenMoreRecentEntity to Mode.EXCLUSIVE, moreRecentEntity to Mode.INCLUSIVE) ).getOrNull() assertEquals( - "2030-01-01T01:01:01.01Z", + evenMoreRecentTime, (mergedEntity?.get(name) as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] ) assertEquals("evenMoreRecentName", (mergedEntity[name] as CompactedAttributeInstance)[JSONLD_VALUE_TERM]) @@ -134,14 +139,51 @@ class ContextSourceUtilsTests { @Test fun `merge entity should not merge Auxiliary entity `() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntity( + val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, listOf(evenMoreRecentEntity to Mode.AUXILIARY, moreRecentEntity to Mode.AUXILIARY) ).getOrNull() assertEquals( - "2010-01-01T01:01:01.01Z", + time, (mergedEntity?.get(name) as CompactedAttributeInstance)[NGSILD_OBSERVED_AT_TERM] ) assertEquals("name", (mergedEntity[name] as CompactedAttributeInstance)[JSONLD_VALUE_TERM]) } + + @Test + fun `merge entity should keep more recent modifiedAt`() = runTest { + val entity = minimalEntity.toMutableMap() + (NGSILD_MODIFIED_AT_TERM to time) + val recentlyModifiedEntity = minimalEntity.toMutableMap() + (NGSILD_MODIFIED_AT_TERM to moreRecentTime) + val evenMoreRecentlyModifiedEntity = + minimalEntity.toMutableMap() + (NGSILD_MODIFIED_AT_TERM to evenMoreRecentTime) + + val mergedEntity = ContextSourceUtils.mergeEntities( + entity, + listOf(evenMoreRecentlyModifiedEntity to Mode.AUXILIARY, recentlyModifiedEntity to Mode.AUXILIARY) + ) + assertTrue(mergedEntity.isRight()) + assertEquals( + evenMoreRecentTime, + (mergedEntity.getOrNull()?.get(NGSILD_MODIFIED_AT_TERM)) + ) + } + + @Test + fun `merge entity should keep least recent createdAt`() = runTest { + val entity = minimalEntity.toMutableMap() + (NGSILD_CREATED_AT_TERM to time) + val recentlyModifiedEntity = minimalEntity.toMutableMap() + (NGSILD_CREATED_AT_TERM to moreRecentTime) + val evenMoreRecentlyModifiedEntity = + minimalEntity.toMutableMap() + (NGSILD_CREATED_AT_TERM to evenMoreRecentTime) + + val mergedEntity = ContextSourceUtils.mergeEntities( + entity, + listOf(evenMoreRecentlyModifiedEntity to Mode.AUXILIARY, recentlyModifiedEntity to Mode.AUXILIARY) + ) + assertTrue(mergedEntity.isRight()) + + assertEquals( + time, + (mergedEntity.getOrNull()?.get(NGSILD_CREATED_AT_TERM)) + ) + } } From f38c04bba673cec6d009e334cf03ef0c98de01e6 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Thu, 24 Oct 2024 15:35:27 +0200 Subject: [PATCH 13/19] feat: warning following rfc7234 + docker-compose for csr + fix tests --- context-source-docker-compose.yml | 54 ------------------- csr.env | 13 +++++ docker-compose.yml | 23 ++++---- .../stellio/search/csr/model/CSRFilters.kt | 9 ++-- .../csr/model/ContextSourceRegistration.kt | 3 ++ .../stellio/search/csr/model/NGSILDWarning.kt | 54 +++++++++++++++++++ .../search/csr/service/ContextSourceCaller.kt | 22 +++++--- .../search/csr/service/ContextSourceUtils.kt | 30 ++++++----- .../search/entity/web/EntityHandler.kt | 26 ++++----- .../csr/service/ContextSourceCallerTests.kt | 4 +- .../csr/service/ContextSourceUtilsTests.kt | 22 ++++---- shared/config/detekt/baseline.xml | 1 - .../egm/stellio/shared/util/ApiResponses.kt | 3 -- .../egm/stellio/shared/util/NGSILDWarning.kt | 29 ---------- 14 files changed, 145 insertions(+), 148 deletions(-) delete mode 100644 context-source-docker-compose.yml create mode 100644 csr.env create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt delete mode 100644 shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt diff --git a/context-source-docker-compose.yml b/context-source-docker-compose.yml deleted file mode 100644 index 9ca09790c..000000000 --- a/context-source-docker-compose.yml +++ /dev/null @@ -1,54 +0,0 @@ -services: - context-source-postgres: - image: stellio/stellio-timescale-postgis:16-2.16.0-3.3 - container_name: context-source-stellio-postgres - environment: - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASS=${POSTGRES_PASS} - - POSTGRES_DBNAME=${POSTGRES_DBNAME} - - POSTGRES_MULTIPLE_EXTENSIONS=postgis,timescaledb,pgcrypto - - ACCEPT_TIMESCALE_TUNING=TRUE - ports: - - "65432:5432" - volumes: - - context-source-stellio-postgres-storage:/var/lib/postgresql - healthcheck: - test: ["CMD-SHELL", "pg_isready -h localhost -U stellio"] - interval: 10s - timeout: 5s - retries: 20 - start_period: 10s - context-source-api-gateway: - container_name: context-source-stellio-api-gateway - image: stellio/stellio-api-gateway:${STELLIO_DOCKER_TAG} - environment: - - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} - ports: - - "18080:8080" - context-source-search-service: - container_name: context-source-stellio-search-service - image: stellio/stellio-search-service:${STELLIO_DOCKER_TAG} - environment: - - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} - - SPRING_R2DBC_URL=r2dbc:pool:postgresql://context-source-postgres:5432/${STELLIO_SEARCH_DB_DATABASE} - - SPRING_FLYWAY_URL=jdbc:postgresql://context-source-postgres:5432/${STELLIO_SEARCH_DB_DATABASE} - - SPRING_R2DBC_USERNAME=${POSTGRES_USER} - - SPRING_R2DBC_PASSWORD=${POSTGRES_PASS} - - APPLICATION_AUTHENTICATION_ENABLED=${STELLIO_AUTHENTICATION_ENABLED} - - APPLICATION_TENANTS_0_ISSUER=${APPLICATION_TENANTS_0_ISSUER} - - APPLICATION_TENANTS_0_NAME=${APPLICATION_TENANTS_0_NAME} - - APPLICATION_TENANTS_0_DBSCHEMA=${APPLICATION_TENANTS_0_DBSCHEMA} - - APPLICATION_PAGINATION_LIMIT-DEFAULT=${APPLICATION_PAGINATION_LIMIT_DEFAULT} - - APPLICATION_PAGINATION_LIMIT-MAX=${APPLICATION_PAGINATION_LIMIT_MAX} - - APPLICATION_PAGINATION_TEMPORAL-LIMIT=${APPLICATION_PAGINATION_TEMPORAL_LIMIT} - - ports: - - "18083:8083" - depends_on: - context-source-postgres: - condition: service_healthy - - - -volumes: - context-source-stellio-postgres-storage: diff --git a/csr.env b/csr.env new file mode 100644 index 000000000..9ef0134f1 --- /dev/null +++ b/csr.env @@ -0,0 +1,13 @@ +API_GATEWAY_PORT=8090 +KAFKA_PORT=29093 +DB_PORT=5433 +SEARCH_SERVICE_PORT=8093 +SUBSCRIPTION_SERVICE_PORT=8094 +CONTAINER_NAME_PREFIX= csr- + +# Used by subscription service when searching entities for recurring subscriptions +# (those defined with a timeInterval parameter) +SUBSCRIPTION_ENTITY_SERVICE_URL=http://search-service:8093 + +# Used as a base URL by subscription service when serving contexts for notifications +SUBSCRIPTION_STELLIO_URL=http://localhost:8090 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 086c33efe..85f62a6ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ services: kafka: image: confluentinc/cp-kafka:7.6.0 - container_name: stellio-kafka + container_name: "${CONTAINER_NAME_PREFIX}stellio-kafka" hostname: stellio-kafka ports: - - "29092:29092" + - "${KAFKA_PORT:-29092}:29092" environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT @@ -20,7 +20,7 @@ services: CLUSTER_ID: ZGE2MTQ4NDk4NGU3NDE2Mm postgres: image: stellio/stellio-timescale-postgis:16-2.16.0-3.3 - container_name: stellio-postgres + container_name: "${CONTAINER_NAME_PREFIX}stellio-postgres" environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASS=${POSTGRES_PASS} @@ -28,7 +28,7 @@ services: - POSTGRES_MULTIPLE_EXTENSIONS=postgis,timescaledb,pgcrypto - ACCEPT_TIMESCALE_TUNING=TRUE ports: - - "5432:5432" + - "${DB_PORT:-5432}:5432" volumes: - stellio-postgres-storage:/var/lib/postgresql healthcheck: @@ -38,15 +38,14 @@ services: retries: 20 start_period: 10s api-gateway: - container_name: stellio-api-gateway + container_name: "${CONTAINER_NAME_PREFIX}stellio-api-gateway" image: stellio/stellio-api-gateway:${STELLIO_DOCKER_TAG} environment: - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} ports: -# todo define configurable port for easy CSR test - - "8080:8080" + - "${API_GATEWAY_PORT:-8080}:8080" search-service: - container_name: stellio-search-service + container_name: "${CONTAINER_NAME_PREFIX}stellio-search-service" image: stellio/stellio-search-service:${STELLIO_DOCKER_TAG} environment: - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} @@ -63,14 +62,14 @@ services: - APPLICATION_PAGINATION_TEMPORAL-LIMIT=${APPLICATION_PAGINATION_TEMPORAL_LIMIT} ports: - - "8083:8083" + - "${SEARCH_SERVICE_PORT:-8083}:8083" depends_on: postgres: condition: service_healthy kafka: condition: service_started subscription-service: - container_name: stellio-subscription-service + container_name: "${CONTAINER_NAME_PREFIX}stellio-subscription-service" image: stellio/stellio-subscription-service:${STELLIO_DOCKER_TAG} environment: - SPRING_PROFILES_ACTIVE=${ENVIRONMENT} @@ -87,7 +86,7 @@ services: - APPLICATION_PAGINATION_LIMIT-DEFAULT=${APPLICATION_PAGINATION_LIMIT_DEFAULT} - APPLICATION_PAGINATION_LIMIT-MAX=${APPLICATION_PAGINATION_LIMIT_MAX} ports: - - "8084:8084" + - "${SUBSCRIPTION_SERVICE_PORT:-8084}:8084" depends_on: postgres: condition: service_healthy @@ -96,3 +95,5 @@ services: volumes: stellio-postgres-storage: + name: "${CONTAINER_NAME_PREFIX}stellio-postgres-storage" + diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt index bd68e4e2e..0f107e2d7 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt @@ -3,10 +3,11 @@ package com.egm.stellio.search.csr.model import java.net.URI data class CSRFilters( // we should use a combination of EntitiesQuery TemporalQuery (when we implement all operations) - val ids: Set = emptySet() + val ids: Set = emptySet(), + val csf: String? = null ) { - fun buildWHEREStatement(): String = - if (ids.isNotEmpty()) + fun buildWHEREStatement(): String { + val idFilter = if (ids.isNotEmpty()) """ ( entity_info.id is null OR @@ -18,4 +19,6 @@ data class CSRFilters( // we should use a combination of EntitiesQuery TemporalQ ) """.trimIndent() else "true" + return idFilter + } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt index e482c61a1..fecdc426b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt @@ -24,6 +24,7 @@ import java.util.regex.Pattern data class ContextSourceRegistration( val id: URI = "urn:ngsi-ld:ContextSourceRegistration:${UUID.randomUUID()}".toUri(), val endpoint: URI, + val registrationName: String? = null, val type: String = NGSILD_CSR_TERM, val mode: Mode = Mode.INCLUSIVE, val information: List = emptyList(), @@ -40,6 +41,8 @@ data class ContextSourceRegistration( val lastSuccess: ZonedDateTime? = null, ) { + fun isAuxiliary(): Boolean = mode == Mode.AUXILIARY + data class TimeInterval( val start: ZonedDateTime, val end: ZonedDateTime? = null diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt new file mode 100644 index 000000000..3f9f4d28c --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt @@ -0,0 +1,54 @@ +package com.egm.stellio.search.csr.model + +import org.springframework.http.ResponseEntity +import java.util.* + +/* implement 6.3.17 */ +open class NGSILDWarning( + val code: Int, + open val message: String, + open val csr: ContextSourceRegistration +) { + // follow rfc7234 https://www.rfc-editor.org/rfc/rfc7234.html#section-5.5 + fun getHeaderMessage(): String = "$code ${getWarnAgent()} \"${getWarnText()}\"" + + // new line are forbidden in headers + private fun getWarnText(): String = Base64.getEncoder().encodeToString(message.toByteArray()) + private fun getWarnAgent(): String = csr?.registrationName ?: csr?.id?.toString() ?: "-" + + companion object { + const val HEADER_NAME = "NGSILD-Warning" + const val RESPONSE_IS_STALE_WARNING_CODE = 110 + const val REVALIDATION_FAILED_WARNING_CODE = 111 + const val MISCELLANEOUS_WARNING_CODE = 199 + const val MISCELLANEOUS_PERSISTENT_WARNING_CODE = 299 + } +} + +data class ResponseIsStaleWarning( + override val message: String, + override val csr: ContextSourceRegistration +) : NGSILDWarning(RESPONSE_IS_STALE_WARNING_CODE, message, csr) + +data class RevalidationFailedWarning( + override val message: String, + override val csr: ContextSourceRegistration +) : NGSILDWarning(REVALIDATION_FAILED_WARNING_CODE, message, csr) + +data class MiscellaneousWarning( + override val message: String, + override val csr: ContextSourceRegistration +) : NGSILDWarning(MISCELLANEOUS_WARNING_CODE, message, csr) + +data class MiscellaneousPersistentWarning( + override val message: String, + override val csr: ContextSourceRegistration +) : NGSILDWarning(MISCELLANEOUS_PERSISTENT_WARNING_CODE, message, csr) + +fun ResponseEntity<*>.addWarnings(warnings: List?): ResponseEntity<*> { + if (!warnings.isNullOrEmpty()) this.headers.addAll( + NGSILDWarning.HEADER_NAME, + warnings.map { it.getHeaderMessage() } + ) + return this +} diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt index cbf385ed7..ec0da3e06 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt @@ -2,9 +2,11 @@ package com.egm.stellio.search.csr.service import arrow.core.* import arrow.core.raise.either -import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.model.CompactedEntity import com.egm.stellio.shared.util.* +import org.slf4j.Logger +import org.slf4j.LoggerFactory import org.springframework.core.codec.DecodingException import org.springframework.http.HttpHeaders import org.springframework.http.HttpMethod @@ -16,6 +18,7 @@ import org.springframework.web.reactive.function.client.awaitExchange import java.net.URI object ContextSourceCaller { + val logger: Logger = LoggerFactory.getLogger(javaClass) suspend fun getDistributedInformation( httpHeaders: HttpHeaders, @@ -43,31 +46,36 @@ object ContextSourceCaller { } when { statusCode.is2xxSuccessful -> { - JsonLdUtils.logger.info("Successfully received response from CSR at $uri") + logger.info("Successfully received data from CSR ${csr.id} at $uri") response.right() } statusCode.isSameCodeAs(HttpStatus.NOT_FOUND) -> { - JsonLdUtils.logger.info("CSR returned 404 at $uri: $response") + logger.info("CSR returned 404 at $uri: $response") null.right() } else -> { - JsonLdUtils.logger.warn("Error contacting CSR at $uri: $response") + logger.warn("Error contacting CSR at $uri: $response") MiscellaneousPersistentWarning( - "the CSR ${csr.id} returned an error $statusCode at : $uri response: \"$response\"" + "$uri returned an error $statusCode with response: \"$response\"", + csr ).left() } } } catch (e: Exception) { + logger.warn("Error contacting CSR at $uri: ${e.message}") + logger.warn(e.stackTraceToString()) when (e) { is DecodingException -> RevalidationFailedWarning( - "the CSR ${csr.id} as : $uri returned badly formed data message: \"${e.cause}:${e.message}\"" + "$uri returned badly formed data message: \"${e.cause}:${e.message}\"", + csr ) else -> MiscellaneousWarning( - "Error connecting to csr ${csr.id} at : $uri message : \"${e.cause}:${e.message}\"" + "Error connecting to $uri message : \"${e.cause}:${e.message}\"", + csr ) }.left() } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 9124f9337..4c36a9368 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -19,23 +19,23 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_TERM import java.time.ZonedDateTime import kotlin.random.Random.Default.nextBoolean -typealias CompactedEntityWithMode = Pair +typealias CompactedEntityWithCSR = Pair typealias DataSetId = String? typealias AttributeByDataSetId = Map object ContextSourceUtils { fun mergeEntities( localEntity: CompactedEntity?, - remoteEntitiesWithMode: List + remoteEntitiesWithMode: List ): IorNel = iorNel { if (localEntity == null && remoteEntitiesWithMode.isEmpty()) return@iorNel null val mergedEntity: MutableMap = localEntity?.toMutableMap() ?: mutableMapOf() - remoteEntitiesWithMode.sortedBy { (_, mode) -> mode == Mode.AUXILIARY } - .forEach { (entity, mode) -> + remoteEntitiesWithMode.sortedBy { (_, csr) -> csr.isAuxiliary() } + .forEach { (entity, csr) -> mergedEntity.putAll( - getMergeNewValues(mergedEntity, entity, mode).toIor().toIorNel().bind() + getMergeNewValues(mergedEntity, entity, csr).toIor().toIorNel().bind() ) } @@ -45,7 +45,7 @@ object ContextSourceUtils { private fun getMergeNewValues( localEntity: CompactedEntity, remoteEntity: CompactedEntity, - mode: Mode + csr: ContextSourceRegistration ): Either = either { remoteEntity.mapValues { (key, value) -> val localValue = localEntity[key] @@ -63,7 +63,7 @@ object ContextSourceUtils { else -> mergeAttribute( localValue, value, - mode == Mode.AUXILIARY + csr ).bind() } } @@ -86,15 +86,15 @@ object ContextSourceUtils { fun mergeAttribute( currentAttribute: Any, remoteAttribute: Any, - isAuxiliary: Boolean = false + csr: ContextSourceRegistration ): Either = either { - val currentInstances = groupInstancesByDataSetId(currentAttribute).bind().toMutableMap() - val remoteInstances = groupInstancesByDataSetId(remoteAttribute).bind() + val currentInstances = groupInstancesByDataSetId(currentAttribute, csr).bind().toMutableMap() + val remoteInstances = groupInstancesByDataSetId(remoteAttribute, csr).bind() remoteInstances.entries.forEach { (datasetId, remoteInstance) -> val currentInstance = currentInstances[datasetId] when { currentInstance == null -> currentInstances[datasetId] = remoteInstance - isAuxiliary -> {} + csr.isAuxiliary() -> {} currentInstance.isBefore(remoteInstance, NGSILD_OBSERVED_AT_TERM) -> currentInstances[datasetId] = remoteInstance remoteInstance.isBefore(currentInstance, NGSILD_OBSERVED_AT_TERM) -> {} @@ -111,7 +111,10 @@ object ContextSourceUtils { } // do not work with CORE MEMBER since they are nor list nor map - private fun groupInstancesByDataSetId(attribute: Any): Either = + private fun groupInstancesByDataSetId( + attribute: Any, + csr: ContextSourceRegistration + ): Either = when (attribute) { is Map<*, *> -> { attribute as CompactedAttributeInstance @@ -123,7 +126,8 @@ object ContextSourceUtils { } else -> { RevalidationFailedWarning( - "The received payload is invalid. Attribute is nor List nor a Map : $attribute" + "The received payload is invalid. Attribute is nor List nor a Map : $attribute", + csr ).left() } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index b3b593758..791c1ae9f 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -5,8 +5,7 @@ import arrow.core.left import arrow.core.raise.either import arrow.core.right import arrow.fx.coroutines.parMap -import com.egm.stellio.search.csr.model.CSRFilters -import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.search.csr.model.* import com.egm.stellio.search.csr.service.ContextSourceCaller import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.csr.service.ContextSourceUtils @@ -218,7 +217,7 @@ class EntityHandler( } // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time - val (remoteEntitiesWithMode, warnings) = matchingCSR.parMap { csr -> + val (remoteEntitiesWithCSR, warnings) = matchingCSR.parMap { csr -> val response = ContextSourceCaller.getDistributedInformation( httpHeaders, csr, @@ -226,34 +225,31 @@ class EntityHandler( params ) contextSourceRegistrationService.updateContextSourceStatus(csr, response.isRight()) - response to csr.mode + response to csr }.partition { it.first.getOrNull() != null } .let { (responses, warnings) -> - responses.map { (response, mode) -> response.getOrNull()!! to mode } to + responses.map { (response, csr) -> response.getOrNull()!! to csr } to warnings.mapNotNull { (warning, _) -> warning.leftOrNull() }.toMutableList() } // we could simplify the code if we check the JsonPayload beforehand val (mergeWarnings, mergedEntity) = ContextSourceUtils.mergeEntities( localEntity.getOrNull(), - remoteEntitiesWithMode + remoteEntitiesWithCSR ).toPair() - if (mergedEntity.isNullOrEmpty()) { + mergeWarnings?.let { warnings.addAll(it) } + + if (mergedEntity == null) { val localError = localEntity.leftOrNull() - val error = localError!!.toErrorResponse() - mergeWarnings?.let { warnings.addAll(it) } - if (warnings.isNotEmpty()) { - error.headers.addAll(NGSILDWarning.HEADER_NAME, warnings.getHeaderMessages()) - } - return error + return localError!!.toErrorResponse().addWarnings(warnings) } val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) - prepareGetSuccessResponseHeaders(mediaType, contexts, warnings) + prepareGetSuccessResponseHeaders(mediaType, contexts) .body( serializeObject(mergedEntity.toFinalRepresentation(ngsiLdDataRepresentation)) - ) + ).addWarnings(warnings) }.fold( { it.toErrorResponse() }, { it } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt index 52203f0cc..2b886c2e8 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt @@ -1,7 +1,6 @@ package com.egm.stellio.search.csr.service -import com.egm.stellio.search.csr.model.ContextSourceRegistration -import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.github.tomakehurst.wiremock.client.WireMock.* @@ -27,7 +26,6 @@ class ContextSourceCallerTests { information = emptyList(), operations = listOf(Operation.FEDERATION_OPS), createdAt = ngsiLdDateTime(), - ) private val emptyParams = LinkedMultiValueMap() private val entityWithSysAttrs = diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt index 7ce7333e2..75bf4ba04 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -1,6 +1,7 @@ package com.egm.stellio.search.csr.service import arrow.core.right +import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedEntity @@ -59,6 +60,9 @@ class ContextSourceUtilsTests { private val entityWithLastName = minimalEntity.toMutableMap().plus("lastName" to nameAttribute) private val entityWithSurName = minimalEntity.toMutableMap().plus("surName" to nameAttribute) + private val auxiliaryCSR = ContextSourceRegistration(endpoint = "http://mock-uri".toUri(), mode = Mode.AUXILIARY) + private val inclusiveCSR = ContextSourceRegistration(endpoint = "http://mock-uri".toUri()) + @Test fun `merge entity should return localEntity when no other entities is provided`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities(baseEntity, emptyList()) @@ -67,7 +71,7 @@ class ContextSourceUtilsTests { @Test fun `merge entity should merge the localEntity with the list of entities `() = runTest { - val mergedEntity = ContextSourceUtils.mergeEntities(minimalEntity, listOf(baseEntity to Mode.AUXILIARY)) + val mergedEntity = ContextSourceUtils.mergeEntities(minimalEntity, listOf(baseEntity to auxiliaryCSR)) assertEquals(baseEntity, mergedEntity.getOrNull()) } @@ -75,7 +79,7 @@ class ContextSourceUtilsTests { fun `merge entity should merge all the entities`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, - listOf(entityWithLastName to Mode.AUXILIARY, entityWithSurName to Mode.INCLUSIVE) + listOf(entityWithLastName to auxiliaryCSR, entityWithSurName to inclusiveCSR) ) assertEquals(entityWithName + entityWithLastName + entityWithSurName, mergedEntity.getOrNull()) } @@ -89,7 +93,7 @@ class ContextSourceUtilsTests { every { ContextSourceUtils.mergeTypeOrScope(any(), any()) } returns listOf("Beehive") ContextSourceUtils.mergeEntities( entityWithName, - listOf(entityWithName to Mode.AUXILIARY, entityWithName to Mode.INCLUSIVE) + listOf(entityWithName to auxiliaryCSR, entityWithName to inclusiveCSR) ) verify(exactly = 2) { ContextSourceUtils.mergeAttribute(any(), any(), any()) } verify(exactly = 2) { ContextSourceUtils.mergeTypeOrScope(any(), any()) } @@ -100,7 +104,7 @@ class ContextSourceUtilsTests { fun `merge entity should merge the types correctly `() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( minimalEntity, - listOf(multipleTypeEntity to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) + listOf(multipleTypeEntity to auxiliaryCSR, baseEntity to inclusiveCSR) ).getOrNull() assertThat(mergedEntity?.get(JSONLD_TYPE_TERM) as List<*>) .hasSize(3) @@ -117,7 +121,7 @@ class ContextSourceUtilsTests { val entityWithDifferentName = minimalEntity.toMutableMap() + (name to nameAttribute2) val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, - listOf(entityWithDifferentName to Mode.AUXILIARY, baseEntity to Mode.INCLUSIVE) + listOf(entityWithDifferentName to auxiliaryCSR, baseEntity to inclusiveCSR) ).getOrNull() assertThat( mergedEntity?.get(name) as List<*> @@ -128,7 +132,7 @@ class ContextSourceUtilsTests { fun `merge entity should merge attribute same datasetId keeping the most recentOne `() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, - listOf(evenMoreRecentEntity to Mode.EXCLUSIVE, moreRecentEntity to Mode.INCLUSIVE) + listOf(evenMoreRecentEntity to inclusiveCSR, moreRecentEntity to inclusiveCSR) ).getOrNull() assertEquals( evenMoreRecentTime, @@ -141,7 +145,7 @@ class ContextSourceUtilsTests { fun `merge entity should not merge Auxiliary entity `() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, - listOf(evenMoreRecentEntity to Mode.AUXILIARY, moreRecentEntity to Mode.AUXILIARY) + listOf(evenMoreRecentEntity to auxiliaryCSR, moreRecentEntity to auxiliaryCSR) ).getOrNull() assertEquals( time, @@ -159,7 +163,7 @@ class ContextSourceUtilsTests { val mergedEntity = ContextSourceUtils.mergeEntities( entity, - listOf(evenMoreRecentlyModifiedEntity to Mode.AUXILIARY, recentlyModifiedEntity to Mode.AUXILIARY) + listOf(evenMoreRecentlyModifiedEntity to auxiliaryCSR, recentlyModifiedEntity to auxiliaryCSR) ) assertTrue(mergedEntity.isRight()) assertEquals( @@ -177,7 +181,7 @@ class ContextSourceUtilsTests { val mergedEntity = ContextSourceUtils.mergeEntities( entity, - listOf(evenMoreRecentlyModifiedEntity to Mode.AUXILIARY, recentlyModifiedEntity to Mode.AUXILIARY) + listOf(evenMoreRecentlyModifiedEntity to auxiliaryCSR, recentlyModifiedEntity to auxiliaryCSR) ) assertTrue(mergedEntity.isRight()) diff --git a/shared/config/detekt/baseline.xml b/shared/config/detekt/baseline.xml index 6081a74e0..c3deb4468 100644 --- a/shared/config/detekt/baseline.xml +++ b/shared/config/detekt/baseline.xml @@ -8,7 +8,6 @@ LongParameterList:ApiResponses.kt$( body: String, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) LongParameterList:ApiResponses.kt$( entities: Any, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) SpreadOperator:EntityEvent.kt$EntityEvent$( *[ JsonSubTypes.Type(value = EntityCreateEvent::class), JsonSubTypes.Type(value = EntityReplaceEvent::class), JsonSubTypes.Type(value = EntityDeleteEvent::class), JsonSubTypes.Type(value = AttributeAppendEvent::class), JsonSubTypes.Type(value = AttributeReplaceEvent::class), JsonSubTypes.Type(value = AttributeUpdateEvent::class), JsonSubTypes.Type(value = AttributeDeleteEvent::class), JsonSubTypes.Type(value = AttributeDeleteAllInstancesEvent::class) ] ) - SpreadOperator:NGSILDWarning.kt$( NGSILDWarning.HEADER_NAME, *this.getHeaderMessages().toTypedArray() ) SwallowedException:JsonLdUtils.kt$JsonLdUtils$e: JsonLdError TooManyFunctions:JsonLdUtils.kt$JsonLdUtils diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index de5c55fe4..70e61be61 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -168,7 +168,6 @@ fun prepareGetSuccessResponseHeaders( mediaType: MediaType, contexts: List, - warnings: List? = null ): ResponseEntity.BodyBuilder = ResponseEntity.status(HttpStatus.OK) .apply { @@ -178,6 +177,4 @@ fun prepareGetSuccessResponseHeaders( this.header(HttpHeaders.LINK, buildContextLinkHeader(contexts.first())) this.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) } - - warnings?.let { warnings.addToResponse(this) } } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt deleted file mode 100644 index ea2c9a58c..000000000 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/NGSILDWarning.kt +++ /dev/null @@ -1,29 +0,0 @@ -package com.egm.stellio.shared.util - -import org.springframework.http.ResponseEntity.BodyBuilder -import java.util.* - -open class NGSILDWarning( - open val message: String -) { - // new line are forbidden in headers - fun getHeaderMessage(): String = Base64.getEncoder().encodeToString(message.toByteArray()) - - companion object { - const val HEADER_NAME = "NGSILD-Warning" - } -} - -data class ResponseIsStaleWarning(override val message: String) : NGSILDWarning(message) -data class RevalidationFailedWarning(override val message: String) : NGSILDWarning(message) -data class MiscellaneousWarning(override val message: String) : NGSILDWarning(message) -data class MiscellaneousPersistentWarning(override val message: String) : NGSILDWarning(message) - -fun List.getHeaderMessages() = this.map { it.getHeaderMessage() } - -fun List.addToResponse(response: BodyBuilder) { - if (this.isNotEmpty()) response.header( - NGSILDWarning.HEADER_NAME, - *this.getHeaderMessages().toTypedArray() - ) -} From 129228ae9761524577b13c7f81e5cfa498102d7b Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Fri, 25 Oct 2024 10:42:57 +0200 Subject: [PATCH 14/19] feat: doc csr launch --- csr.env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/csr.env b/csr.env index 9ef0134f1..6d02bd99b 100644 --- a/csr.env +++ b/csr.env @@ -1,3 +1,5 @@ +# you can launch a second instance of stellio with +# docker compose --env-file .env --env-file csr.env -p csr-stellio up API_GATEWAY_PORT=8090 KAFKA_PORT=29093 DB_PORT=5433 From b07e03207c338fdfedc91c7628c06cc9e19b9cec Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Fri, 25 Oct 2024 12:15:07 +0200 Subject: [PATCH 15/19] feat: doc csr launch --- csr.env => context-source.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename csr.env => context-source.env (78%) diff --git a/csr.env b/context-source.env similarity index 78% rename from csr.env rename to context-source.env index 6d02bd99b..5e4e16d7f 100644 --- a/csr.env +++ b/context-source.env @@ -1,11 +1,11 @@ # you can launch a second instance of stellio with -# docker compose --env-file .env --env-file csr.env -p csr-stellio up +# docker compose --env-file .env --env-file context-source.env -p stellio-context-source up API_GATEWAY_PORT=8090 KAFKA_PORT=29093 DB_PORT=5433 SEARCH_SERVICE_PORT=8093 SUBSCRIPTION_SERVICE_PORT=8094 -CONTAINER_NAME_PREFIX= csr- +CONTAINER_NAME_PREFIX= context-source- # Used by subscription service when searching entities for recurring subscriptions # (those defined with a timeInterval parameter) From 6d4831a3efb30a92aaab95d9d4c4d129673dd82f Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 26 Oct 2024 15:49:06 +0200 Subject: [PATCH 16/19] chore: slight refactoring of the docker compose context source instance --- context-source.env => .context-source.env | 9 ++++----- .env | 2 ++ docker-compose.yml | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) rename context-source.env => .context-source.env (60%) diff --git a/context-source.env b/.context-source.env similarity index 60% rename from context-source.env rename to .context-source.env index 5e4e16d7f..7ca35d43b 100644 --- a/context-source.env +++ b/.context-source.env @@ -1,15 +1,14 @@ -# you can launch a second instance of stellio with -# docker compose --env-file .env --env-file context-source.env -p stellio-context-source up API_GATEWAY_PORT=8090 KAFKA_PORT=29093 -DB_PORT=5433 +POSTGRES_PORT=5433 SEARCH_SERVICE_PORT=8093 SUBSCRIPTION_SERVICE_PORT=8094 -CONTAINER_NAME_PREFIX= context-source- # Used by subscription service when searching entities for recurring subscriptions # (those defined with a timeInterval parameter) SUBSCRIPTION_ENTITY_SERVICE_URL=http://search-service:8093 # Used as a base URL by subscription service when serving contexts for notifications -SUBSCRIPTION_STELLIO_URL=http://localhost:8090 \ No newline at end of file +SUBSCRIPTION_STELLIO_URL=http://localhost:8090 + +CONTAINER_NAME_PREFIX=context-source- diff --git a/.env b/.env index 2ba5383fe..42ea441ec 100644 --- a/.env +++ b/.env @@ -27,3 +27,5 @@ APPLICATION_TENANTS_0_DBSCHEMA=public APPLICATION_PAGINATION_LIMIT_DEFAULT=30 APPLICATION_PAGINATION_LIMIT_MAX=100 APPLICATION_PAGINATION_TEMPORAL_LIMIT=10000 + +CONTAINER_NAME_PREFIX= diff --git a/docker-compose.yml b/docker-compose.yml index 85f62a6ef..626d73e43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,5 @@ +# you can launch a second instance of Stellio with (for instance to use it as a context source) +# docker compose --env-file .env --env-file .context-source.env -p stellio-context-source up -d services: kafka: image: confluentinc/cp-kafka:7.6.0 @@ -28,7 +30,7 @@ services: - POSTGRES_MULTIPLE_EXTENSIONS=postgis,timescaledb,pgcrypto - ACCEPT_TIMESCALE_TUNING=TRUE ports: - - "${DB_PORT:-5432}:5432" + - "${POSTGRES_PORT:-5432}:5432" volumes: - stellio-postgres-storage:/var/lib/postgresql healthcheck: @@ -96,4 +98,3 @@ services: volumes: stellio-postgres-storage: name: "${CONTAINER_NAME_PREFIX}stellio-postgres-storage" - From 874023947c1fbdc326d7dc3f2f0ec2108425de3a Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 27 Oct 2024 18:47:39 +0100 Subject: [PATCH 17/19] chore: misc typo, wording and naming --- .../stellio/search/csr/model/CSRFilters.kt | 2 +- .../csr/model/ContextSourceRegistration.kt | 8 ++++--- .../stellio/search/csr/model/NGSILDWarning.kt | 19 ++++++++------- .../search/csr/service/ContextSourceCaller.kt | 8 +++---- .../ContextSourceRegistrationService.kt | 2 +- .../search/csr/service/ContextSourceUtils.kt | 20 ++++++++-------- .../search/entity/web/EntityHandler.kt | 6 +++-- .../csr/service/ContextSourceCallerTests.kt | 7 ++++-- .../csr/service/ContextSourceUtilsTests.kt | 24 ++++++++++--------- .../egm/stellio/shared/model/ApiExceptions.kt | 2 +- .../egm/stellio/shared/model/ErrorResponse.kt | 2 +- .../egm/stellio/shared/util/ApiResponses.kt | 18 ++++---------- 12 files changed, 61 insertions(+), 57 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt index 0f107e2d7..0ddb2e2f2 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt @@ -6,7 +6,7 @@ data class CSRFilters( // we should use a combination of EntitiesQuery TemporalQ val ids: Set = emptySet(), val csf: String? = null ) { - fun buildWHEREStatement(): String { + fun buildWhereStatement(): String { val idFilter = if (ids.isNotEmpty()) """ ( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt index fecdc426b..796464457 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/ContextSourceRegistration.kt @@ -4,7 +4,9 @@ import arrow.core.Either import arrow.core.left import arrow.core.raise.either import arrow.core.right -import com.egm.stellio.shared.model.* +import com.egm.stellio.shared.model.APIException +import com.egm.stellio.shared.model.BadRequestDataException +import com.egm.stellio.shared.model.toAPIException import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CSR_TERM @@ -18,7 +20,7 @@ import com.fasterxml.jackson.module.kotlin.convertValue import org.springframework.http.MediaType import java.net.URI import java.time.ZonedDateTime -import java.util.* +import java.util.UUID import java.util.regex.Pattern data class ContextSourceRegistration( @@ -34,7 +36,7 @@ data class ContextSourceRegistration( val observationInterval: TimeInterval? = null, val managementInterval: TimeInterval? = null, - var status: StatusType? = null, + val status: StatusType? = null, val timesSent: Int = 0, val timesFailed: Int = 0, val lastFailure: ZonedDateTime? = null, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt index 3f9f4d28c..bc0c57157 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt @@ -1,11 +1,13 @@ package com.egm.stellio.search.csr.model import org.springframework.http.ResponseEntity -import java.util.* +import java.util.Base64 -/* implement 6.3.17 */ +/** + * Implements NGSILD-Warning as defined in 6.3.17 + */ open class NGSILDWarning( - val code: Int, + private val code: Int, open val message: String, open val csr: ContextSourceRegistration ) { @@ -14,7 +16,7 @@ open class NGSILDWarning( // new line are forbidden in headers private fun getWarnText(): String = Base64.getEncoder().encodeToString(message.toByteArray()) - private fun getWarnAgent(): String = csr?.registrationName ?: csr?.id?.toString() ?: "-" + private fun getWarnAgent(): String = csr.registrationName ?: csr.id.toString() companion object { const val HEADER_NAME = "NGSILD-Warning" @@ -46,9 +48,10 @@ data class MiscellaneousPersistentWarning( ) : NGSILDWarning(MISCELLANEOUS_PERSISTENT_WARNING_CODE, message, csr) fun ResponseEntity<*>.addWarnings(warnings: List?): ResponseEntity<*> { - if (!warnings.isNullOrEmpty()) this.headers.addAll( - NGSILDWarning.HEADER_NAME, - warnings.map { it.getHeaderMessage() } - ) + if (!warnings.isNullOrEmpty()) + this.headers.addAll( + NGSILDWarning.HEADER_NAME, + warnings.map { it.getHeaderMessage() } + ) return this } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt index ec0da3e06..402cfd6f5 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt @@ -1,10 +1,11 @@ package com.egm.stellio.search.csr.service -import arrow.core.* +import arrow.core.Either +import arrow.core.left import arrow.core.raise.either +import arrow.core.right import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.model.CompactedEntity -import com.egm.stellio.shared.util.* import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.core.codec.DecodingException @@ -57,9 +58,8 @@ object ContextSourceCaller { else -> { logger.warn("Error contacting CSR at $uri: $response") - MiscellaneousPersistentWarning( - "$uri returned an error $statusCode with response: \"$response\"", + "$uri returned an error $statusCode with response: $response", csr ).left() } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index adf521738..0052ea4a5 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -166,7 +166,7 @@ class ContextSourceRegistrationService( limit: Int = Int.MAX_VALUE, offset: Int = 0, ): List { - val filterQuery = filters.buildWHEREStatement() + val filterQuery = filters.buildWhereStatement() val selectStatement = """ diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 4c36a9368..978853f09 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -3,11 +3,12 @@ package com.egm.stellio.search.csr.service import arrow.core.* import arrow.core.raise.either import arrow.core.raise.iorNel -import com.egm.stellio.search.csr.model.* +import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.search.csr.model.NGSILDWarning +import com.egm.stellio.search.csr.model.RevalidationFailedWarning import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedAttributeInstances import com.egm.stellio.shared.model.CompactedEntity -import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM @@ -16,23 +17,25 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_TERM +import com.egm.stellio.shared.util.isDateTime import java.time.ZonedDateTime import kotlin.random.Random.Default.nextBoolean typealias CompactedEntityWithCSR = Pair typealias DataSetId = String? typealias AttributeByDataSetId = Map + object ContextSourceUtils { fun mergeEntities( localEntity: CompactedEntity?, - remoteEntitiesWithMode: List + remoteEntitiesWithCSR: List ): IorNel = iorNel { - if (localEntity == null && remoteEntitiesWithMode.isEmpty()) return@iorNel null + if (localEntity == null && remoteEntitiesWithCSR.isEmpty()) return@iorNel null val mergedEntity: MutableMap = localEntity?.toMutableMap() ?: mutableMapOf() - remoteEntitiesWithMode.sortedBy { (_, csr) -> csr.isAuxiliary() } + remoteEntitiesWithCSR.sortedBy { (_, csr) -> csr.isAuxiliary() } .forEach { (entity, csr) -> mergedEntity.putAll( getMergeNewValues(mergedEntity, entity, csr).toIor().toIorNel().bind() @@ -60,11 +63,8 @@ object ContextSourceUtils { key == NGSILD_MODIFIED_AT_TERM -> if ((localValue as String?).isBefore(value as String?)) value else localValue - else -> mergeAttribute( - localValue, - value, - csr - ).bind() + else -> + mergeAttribute(localValue, value, csr).bind() } } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index 791c1ae9f..d2ee04d42 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -5,7 +5,9 @@ import arrow.core.left import arrow.core.raise.either import arrow.core.right import arrow.fx.coroutines.parMap -import com.egm.stellio.search.csr.model.* +import com.egm.stellio.search.csr.model.CSRFilters +import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.search.csr.model.addWarnings import com.egm.stellio.search.csr.service.ContextSourceCaller import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.csr.service.ContextSourceUtils @@ -188,7 +190,7 @@ class EntityHandler( suspend fun getByURI( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, - @RequestParam params: MultiValueMap, + @RequestParam params: MultiValueMap ): ResponseEntity<*> = either { val mediaType = getApplicableMediaType(httpHeaders).bind() val sub = getSubFromSecurityContext() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt index 2b886c2e8..a8182871f 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt @@ -1,8 +1,11 @@ package com.egm.stellio.search.csr.service import com.egm.stellio.search.csr.model.* -import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.JsonUtils.serializeObject +import com.egm.stellio.shared.util.assertJsonPayloadsAreEqual +import com.egm.stellio.shared.util.ngsiLdDateTime +import com.egm.stellio.shared.util.toUri import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.junit5.WireMockTest import kotlinx.coroutines.test.runTest @@ -75,7 +78,7 @@ class ContextSourceCallerTests { } @Test - fun `getDistributedInformation should return a MiscellaneousWarning if it receive no answer`() = runTest { + fun `getDistributedInformation should return a MiscellaneousWarning if it receives no answer`() = runTest { val csr = gimmeRawCSR() val path = "/ngsi-ld/v1/entities/$apiaryId" diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt index 75bf4ba04..ead974e27 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -5,13 +5,15 @@ import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedEntity -import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_TERM +import com.egm.stellio.shared.util.loadSampleData +import com.egm.stellio.shared.util.mapper +import com.egm.stellio.shared.util.toUri import com.fasterxml.jackson.module.kotlin.readValue import io.mockk.every import io.mockk.mockkObject @@ -64,13 +66,13 @@ class ContextSourceUtilsTests { private val inclusiveCSR = ContextSourceRegistration(endpoint = "http://mock-uri".toUri()) @Test - fun `merge entity should return localEntity when no other entities is provided`() = runTest { + fun `merge entity should return localEntity when no other entities are provided`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities(baseEntity, emptyList()) assertEquals(baseEntity, mergedEntity.getOrNull()) } @Test - fun `merge entity should merge the localEntity with the list of entities `() = runTest { + fun `merge entity should merge the localEntity with the list of entities`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities(minimalEntity, listOf(baseEntity to auxiliaryCSR)) assertEquals(baseEntity, mergedEntity.getOrNull()) } @@ -85,7 +87,7 @@ class ContextSourceUtilsTests { } @Test - fun `merge entity should call mergeAttribute or mergeTypeOrScope when keys are equals`() = runTest { + fun `merge entity should call mergeAttribute or mergeTypeOrScope when keys are equal`() = runTest { mockkObject(ContextSourceUtils) { every { ContextSourceUtils.mergeAttribute(any(), any(), any()) } returns listOf( nameAttribute @@ -112,7 +114,7 @@ class ContextSourceUtilsTests { } @Test - fun `merge entity should keep both attribute if they have different datasetId `() = runTest { + fun `merge entity should keep both attribute instances if they have different datasetId `() = runTest { val nameAttribute2: CompactedAttributeInstance = mapOf( JSONLD_TYPE_TERM to "Property", JSONLD_VALUE_TERM to "name2", @@ -123,13 +125,13 @@ class ContextSourceUtilsTests { entityWithName, listOf(entityWithDifferentName to auxiliaryCSR, baseEntity to inclusiveCSR) ).getOrNull() - assertThat( - mergedEntity?.get(name) as List<*> - ).hasSize(3).contains(nameAttribute, nameAttribute2, baseEntity[name]) + assertThat(mergedEntity?.get(name) as List<*>) + .hasSize(3) + .contains(nameAttribute, nameAttribute2, baseEntity[name]) } @Test - fun `merge entity should merge attribute same datasetId keeping the most recentOne `() = runTest { + fun `merge entity should merge attribute with same datasetId keeping the most recent one`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, listOf(evenMoreRecentEntity to inclusiveCSR, moreRecentEntity to inclusiveCSR) @@ -142,7 +144,7 @@ class ContextSourceUtilsTests { } @Test - fun `merge entity should not merge Auxiliary entity `() = runTest { + fun `merge entity should not merge info from auxiliary entity if already present`() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( entityWithName, listOf(evenMoreRecentEntity to auxiliaryCSR, moreRecentEntity to auxiliaryCSR) @@ -155,7 +157,7 @@ class ContextSourceUtilsTests { } @Test - fun `merge entity should keep more recent modifiedAt`() = runTest { + fun `merge entity should keep most recent modifiedAt`() = runTest { val entity = minimalEntity.toMutableMap() + (NGSILD_MODIFIED_AT_TERM to time) val recentlyModifiedEntity = minimalEntity.toMutableMap() + (NGSILD_MODIFIED_AT_TERM to moreRecentTime) val evenMoreRecentlyModifiedEntity = diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt index 79e59056a..6a9269335 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt @@ -4,7 +4,7 @@ import com.apicatalog.jsonld.JsonLdError import com.apicatalog.jsonld.JsonLdErrorCode sealed class APIException( - override val message: String, + override val message: String ) : Exception(message) data class InvalidRequestException(override val message: String) : APIException(message) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt index 9eb79ba97..3e7820cb7 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt @@ -115,5 +115,5 @@ enum class ErrorType(val type: URI) { NOT_IMPLEMENTED(URI("https://uri.etsi.org/ngsi-ld/errors/NotImplemented")), UNSUPPORTED_MEDIA_TYPE(URI("https://uri.etsi.org/ngsi-ld/errors/UnsupportedMediaType")), NOT_ACCEPTABLE(URI("https://uri.etsi.org/ngsi-ld/errors/NotAcceptable")), - NONEXISTENT_TENANT(URI("https://uri.etsi.org/ngsi-ld/errors/NonexistentTenant")), + NONEXISTENT_TENANT(URI("https://uri.etsi.org/ngsi-ld/errors/NonexistentTenant")) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index 70e61be61..d5cf1cd2f 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -84,25 +84,17 @@ fun APIException.toErrorResponse(): ResponseEntity<*> = is NotImplementedException -> generateErrorResponse(HttpStatus.NOT_IMPLEMENTED, NotImplementedResponse(this.message)) is LdContextNotAvailableException -> - generateErrorResponse( - HttpStatus.SERVICE_UNAVAILABLE, - LdContextNotAvailableResponse(this.message) - ) + generateErrorResponse(HttpStatus.SERVICE_UNAVAILABLE, LdContextNotAvailableResponse(this.message)) is TooManyResultsException -> generateErrorResponse(HttpStatus.FORBIDDEN, TooManyResultsResponse(this.message)) - else -> generateErrorResponse( - HttpStatus.INTERNAL_SERVER_ERROR, - InternalErrorResponse("$cause") - ) + else -> generateErrorResponse(HttpStatus.INTERNAL_SERVER_ERROR, InternalErrorResponse("$cause")) } -fun generateErrorResponse( - status: HttpStatus, - exception: ErrorResponse, -): ResponseEntity<*> { +fun generateErrorResponse(status: HttpStatus, exception: ErrorResponse): ResponseEntity<*> { logger.info("Returning error ${exception.type} (${exception.detail})") return ResponseEntity.status(status) - .contentType(MediaType.APPLICATION_JSON).body(serializeObject(exception)) + .contentType(MediaType.APPLICATION_JSON) + .body(serializeObject(exception)) } fun missingPathErrorResponse(errorMessage: String): ResponseEntity<*> { From f74ceec42eff992b0f1f771822dfc2dfff3880b0 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 30 Oct 2024 15:00:08 +0100 Subject: [PATCH 18/19] feat: always call context source for normalized representation + PR comments --- .../search/csr/service/ContextSourceCaller.kt | 40 ++++++++------ .../search/csr/service/ContextSourceUtils.kt | 33 ++++++------ .../search/entity/web/EntityHandler.kt | 12 ++--- .../csr/service/ContextSourceCallerTests.kt | 53 ++++++++++++++++--- 4 files changed, 94 insertions(+), 44 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt index 402cfd6f5..f1a076343 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceCaller.kt @@ -6,6 +6,9 @@ import arrow.core.raise.either import arrow.core.right import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.model.CompactedEntity +import com.egm.stellio.shared.util.QUERY_PARAM_GEOMETRY_PROPERTY +import com.egm.stellio.shared.util.QUERY_PARAM_LANG +import com.egm.stellio.shared.util.QUERY_PARAM_OPTIONS import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.core.codec.DecodingException @@ -28,7 +31,9 @@ object ContextSourceCaller { params: MultiValueMap ): Either = either { val uri = URI("${csr.endpoint}$path") - + params.remove(QUERY_PARAM_GEOMETRY_PROPERTY) + params.remove(QUERY_PARAM_OPTIONS) // only request normalized + params.remove(QUERY_PARAM_LANG) // todo not sure its needed val request = WebClient.create() .method(HttpMethod.GET) .uri { uriBuilder -> @@ -40,7 +45,7 @@ object ContextSourceCaller { .build() } .header(HttpHeaders.LINK, httpHeaders.getFirst(HttpHeaders.LINK)) - return try { + return runCatching { val (statusCode, response) = request .awaitExchange { response -> response.statusCode() to response.awaitBodyOrNull() @@ -64,20 +69,23 @@ object ContextSourceCaller { ).left() } } - } catch (e: Exception) { - logger.warn("Error contacting CSR at $uri: ${e.message}") - logger.warn(e.stackTraceToString()) - when (e) { - is DecodingException -> RevalidationFailedWarning( - "$uri returned badly formed data message: \"${e.cause}:${e.message}\"", - csr - ) + }.fold( + onSuccess = { it }, + onFailure = { e -> + logger.warn("Error contacting CSR at $uri: ${e.message}") + logger.warn(e.stackTraceToString()) + when (e) { + is DecodingException -> RevalidationFailedWarning( + "$uri returned badly formed data message: \"${e.cause}:${e.message}\"", + csr + ) - else -> MiscellaneousWarning( - "Error connecting to $uri message : \"${e.cause}:${e.message}\"", - csr - ) - }.left() - } + else -> MiscellaneousWarning( + "Error connecting to $uri message : \"${e.cause}:${e.message}\"", + csr + ) + }.left() + } + ) } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 978853f09..9abd40c64 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -22,9 +22,7 @@ import java.time.ZonedDateTime import kotlin.random.Random.Default.nextBoolean typealias CompactedEntityWithCSR = Pair -typealias DataSetId = String? -typealias AttributeByDataSetId = Map - +typealias AttributeByDataSetId = Map object ContextSourceUtils { fun mergeEntities( @@ -42,29 +40,32 @@ object ContextSourceUtils { ) } - return@iorNel mergedEntity.toMap() + mergedEntity.toMap() } private fun getMergeNewValues( - localEntity: CompactedEntity, + currentEntity: CompactedEntity, remoteEntity: CompactedEntity, csr: ContextSourceRegistration ): Either = either { remoteEntity.mapValues { (key, value) -> - val localValue = localEntity[key] + val currentValue = currentEntity[key] when { - localValue == null -> value - key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> localValue + currentValue == null -> value + key == JSONLD_ID_TERM || key == JSONLD_CONTEXT -> currentValue key == JSONLD_TYPE_TERM || key == NGSILD_SCOPE_TERM -> - mergeTypeOrScope(localValue, value) + mergeTypeOrScope(currentValue, value) key == NGSILD_CREATED_AT_TERM -> - if ((value as String?).isBefore(localValue as String?)) value - else localValue + if ((value as String?).isBefore(currentValue as String?)) value + else currentValue key == NGSILD_MODIFIED_AT_TERM -> - if ((localValue as String?).isBefore(value as String?)) value - else localValue - else -> - mergeAttribute(localValue, value, csr).bind() + if ((currentValue as String?).isBefore(value as String?)) value + else currentValue + else -> mergeAttribute( + currentValue, + value, + csr + ).bind() } } } @@ -110,7 +111,7 @@ object ContextSourceUtils { if (values.size == 1) values[0] else values } - // do not work with CORE MEMBER since they are nor list nor map + // do not work with CORE MEMBER since they can be a String private fun groupInstancesByDataSetId( attribute: Any, csr: ContextSourceRegistration diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt index d2ee04d42..af49f479d 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt @@ -4,6 +4,7 @@ import arrow.core.getOrElse import arrow.core.left import arrow.core.raise.either import arrow.core.right +import arrow.core.separateEither import arrow.fx.coroutines.parMap import com.egm.stellio.search.csr.model.CSRFilters import com.egm.stellio.search.csr.model.Operation @@ -219,7 +220,7 @@ class EntityHandler( } // we can add parMap(concurrency = X) if this trigger too much http connexion at the same time - val (remoteEntitiesWithCSR, warnings) = matchingCSR.parMap { csr -> + val (warnings, remoteEntitiesWithCSR) = matchingCSR.parMap { csr -> val response = ContextSourceCaller.getDistributedInformation( httpHeaders, csr, @@ -227,11 +228,10 @@ class EntityHandler( params ) contextSourceRegistrationService.updateContextSourceStatus(csr, response.isRight()) - response to csr - }.partition { it.first.getOrNull() != null } - .let { (responses, warnings) -> - responses.map { (response, csr) -> response.getOrNull()!! to csr } to - warnings.mapNotNull { (warning, _) -> warning.leftOrNull() }.toMutableList() + response.map { it?.let { it to csr } } + }.separateEither() + .let { (warnings, maybeResponses) -> + warnings.toMutableList() to maybeResponses.filterNotNull() } // we could simplify the code if we check the JsonPayload beforehand diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt index a8182871f..a224c9473 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt @@ -1,11 +1,8 @@ package com.egm.stellio.search.csr.service import com.egm.stellio.search.csr.model.* -import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT +import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonUtils.serializeObject -import com.egm.stellio.shared.util.assertJsonPayloadsAreEqual -import com.egm.stellio.shared.util.ngsiLdDateTime -import com.egm.stellio.shared.util.toUri import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.junit5.WireMockTest import kotlinx.coroutines.test.runTest @@ -15,6 +12,7 @@ import org.springframework.http.HttpHeaders import org.springframework.http.MediaType.APPLICATION_JSON_VALUE import org.springframework.test.context.ActiveProfiles import org.springframework.util.LinkedMultiValueMap +import wiremock.com.google.common.net.HttpHeaders.ACCEPT import wiremock.com.google.common.net.HttpHeaders.CONTENT_TYPE @WireMockTest(httpPort = 8089) @@ -107,7 +105,7 @@ class ContextSourceCallerTests { } @Test - fun `getDistributedInformation should return MiscellaneousPersistentWarning when receiving error 500`() = runTest { + fun `getDistributedInformation should return MiscellaneousPersistentWarning when receiving error 401`() = runTest { val csr = gimmeRawCSR() val path = "/ngsi-ld/v1/entities/$apiaryId" stubFor( @@ -133,6 +131,49 @@ class ContextSourceCallerTests { val response = ContextSourceCaller.getDistributedInformation(HttpHeaders.EMPTY, csr, path, emptyParams) assertTrue(response.isRight()) - assertNull(response.leftOrNull()) + assertNull(response.getOrNull()) + } + + @Test + fun `getDistributedInformation should not ask context source for a GEO_JSON`() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + stubFor( + get(urlMatching(path)) + .willReturn(ok()) + ) + val header = HttpHeaders() + header.accept = listOf(GEO_JSON_MEDIA_TYPE) + ContextSourceCaller.getDistributedInformation( + header, + csr, + path, + emptyParams + ) + verify( + getRequestedFor(urlPathEqualTo(path)) + .withHeader(ACCEPT, notContaining("GEO_JSON_MEDIA_TYPE")) + ) + } + + @Test + fun `getDistributedInformation should always ask for the normalized representation`() = runTest { + val csr = gimmeRawCSR() + val path = "/ngsi-ld/v1/entities/$apiaryId" + stubFor( + get(urlMatching(path)) + .willReturn(notFound()) + ) + val params = LinkedMultiValueMap(mapOf(QUERY_PARAM_OPTIONS to listOf("simplified"))) + ContextSourceCaller.getDistributedInformation( + HttpHeaders.EMPTY, + csr, + path, + params + ) + verify( + getRequestedFor(urlPathEqualTo(path)) + .withQueryParam(QUERY_PARAM_OPTIONS, notContaining("simplified")) + ) } } From deca20c626b574660c9b4fa22093e35bcbc79fc8 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Thu, 31 Oct 2024 14:37:57 +0100 Subject: [PATCH 19/19] feat: registrationName in CSR + Warning fixes --- .../stellio/search/csr/model/NGSILDWarning.kt | 15 +++--- .../ContextSourceRegistrationService.kt | 10 +++- .../search/csr/service/ContextSourceUtils.kt | 20 ++++--- .../resources/db/migration/V0_43__add_csr.sql | 1 + .../com/egm/stellio/search/csr/CsrUtils.kt | 16 ++++++ .../csr/service/ContextSourceCallerTests.kt | 8 +-- .../csr/service/ContextSourceUtilsTests.kt | 52 +++++++++++++------ .../search/entity/web/EntityHandlerTests.kt | 41 +++++++++++++++ 8 files changed, 120 insertions(+), 43 deletions(-) create mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/csr/CsrUtils.kt diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt index bc0c57157..2372b651e 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/model/NGSILDWarning.kt @@ -1,7 +1,7 @@ package com.egm.stellio.search.csr.model +import org.springframework.http.HttpHeaders import org.springframework.http.ResponseEntity -import java.util.Base64 /** * Implements NGSILD-Warning as defined in 6.3.17 @@ -15,7 +15,7 @@ open class NGSILDWarning( fun getHeaderMessage(): String = "$code ${getWarnAgent()} \"${getWarnText()}\"" // new line are forbidden in headers - private fun getWarnText(): String = Base64.getEncoder().encodeToString(message.toByteArray()) + private fun getWarnText(): String = message.replace("\n", " ") private fun getWarnAgent(): String = csr.registrationName ?: csr.id.toString() companion object { @@ -48,10 +48,11 @@ data class MiscellaneousPersistentWarning( ) : NGSILDWarning(MISCELLANEOUS_PERSISTENT_WARNING_CODE, message, csr) fun ResponseEntity<*>.addWarnings(warnings: List?): ResponseEntity<*> { + val headers = HttpHeaders.writableHttpHeaders(this.headers) if (!warnings.isNullOrEmpty()) - this.headers.addAll( - NGSILDWarning.HEADER_NAME, - warnings.map { it.getHeaderMessage() } - ) - return this + headers.addAll(NGSILDWarning.HEADER_NAME, warnings.map { it.getHeaderMessage() }) + + return ResponseEntity.status(this.statusCode) + .headers(headers) + .body(this.body) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt index 0052ea4a5..efb218771 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt @@ -50,6 +50,7 @@ class ContextSourceRegistrationService( mode, information, operations, + registration_name, observation_interval_start, observation_interval_end, management_interval_start, @@ -63,6 +64,7 @@ class ContextSourceRegistrationService( :mode, :information, :operations, + :registration_name, :observation_interval_start, :observation_interval_end, :management_interval_start, @@ -80,6 +82,7 @@ class ContextSourceRegistrationService( Json.of(mapper.writeValueAsString(contextSourceRegistration.information)) ) .bind("operations", contextSourceRegistration.operations.map { it.key }.toTypedArray()) + .bind("registration_name", contextSourceRegistration.registrationName) .bind("observation_interval_start", contextSourceRegistration.observationInterval?.start) .bind("observation_interval_end", contextSourceRegistration.observationInterval?.end) .bind("management_interval_start", contextSourceRegistration.managementInterval?.start) @@ -124,6 +127,7 @@ class ContextSourceRegistrationService( mode, information, operations, + registration_name, observation_interval_start, observation_interval_end, management_interval_start, @@ -175,6 +179,7 @@ class ContextSourceRegistrationService( mode, information, operations, + registration_name, observation_interval_start, observation_interval_end, management_interval_start, @@ -218,6 +223,7 @@ class ContextSourceRegistrationService( information = mapper.readerForListOf(RegistrationInfo::class.java) .readValue((row["information"] as Json).asString()), operations = (row["operations"] as Array).mapNotNull { Operation.fromString(it) }, + registrationName = row["registration_name"] as? String, createdAt = toZonedDateTime(row["created_at"]), modifiedAt = toOptionalZonedDateTime(row["modified_at"]), observationInterval = row["observation_interval_start"]?.let { @@ -238,7 +244,7 @@ class ContextSourceRegistrationService( suspend fun updateContextSourceStatus( csr: ContextSourceRegistration, success: Boolean - ): Long { + ) { val updateStatement = if (success) Update.update("status", ContextSourceRegistration.StatusType.OK.name) .set("times_sent", csr.timesSent + 1) @@ -248,7 +254,7 @@ class ContextSourceRegistrationService( .set("times_failed", csr.timesFailed + 1) .set("last_failure", ngsiLdDateTime()) - return r2dbcEntityTemplate.update( + r2dbcEntityTemplate.update( query(where("id").`is`(csr.id)), updateStatement, ContextSourceRegistration::class.java diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt index 9abd40c64..0e1da9e95 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtils.kt @@ -2,7 +2,6 @@ package com.egm.stellio.search.csr.service import arrow.core.* import arrow.core.raise.either -import arrow.core.raise.iorNel import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.NGSILDWarning import com.egm.stellio.search.csr.model.RevalidationFailedWarning @@ -28,22 +27,21 @@ object ContextSourceUtils { fun mergeEntities( localEntity: CompactedEntity?, remoteEntitiesWithCSR: List - ): IorNel = iorNel { - if (localEntity == null && remoteEntitiesWithCSR.isEmpty()) return@iorNel null + ): IorNel { + if (localEntity == null && remoteEntitiesWithCSR.isEmpty()) return Ior.Right(null) val mergedEntity: MutableMap = localEntity?.toMutableMap() ?: mutableMapOf() - remoteEntitiesWithCSR.sortedBy { (_, csr) -> csr.isAuxiliary() } - .forEach { (entity, csr) -> - mergedEntity.putAll( - getMergeNewValues(mergedEntity, entity, csr).toIor().toIorNel().bind() - ) - } + val warnings = remoteEntitiesWithCSR.sortedBy { (_, csr) -> csr.isAuxiliary() } + .mapNotNull { (entity, csr) -> + getMergeNewValues(mergedEntity, entity, csr) + .onRight { mergedEntity.putAll(it) }.leftOrNull() + }.toNonEmptyListOrNull() - mergedEntity.toMap() + return if (warnings == null) Ior.Right(mergedEntity) else Ior.Both(warnings, mergedEntity) } - private fun getMergeNewValues( + fun getMergeNewValues( currentEntity: CompactedEntity, remoteEntity: CompactedEntity, csr: ContextSourceRegistration diff --git a/search-service/src/main/resources/db/migration/V0_43__add_csr.sql b/search-service/src/main/resources/db/migration/V0_43__add_csr.sql index 9d4c19d89..c7edfd830 100644 --- a/search-service/src/main/resources/db/migration/V0_43__add_csr.sql +++ b/search-service/src/main/resources/db/migration/V0_43__add_csr.sql @@ -5,6 +5,7 @@ CREATE TABLE context_source_registration mode text NOT NULL, information jsonb NOT NULL, operations text[] NOT NULL, + registration_name text, observation_interval_start timestamp with time zone, observation_interval_end timestamp with time zone, management_interval_start timestamp with time zone, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/CsrUtils.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/CsrUtils.kt new file mode 100644 index 000000000..b06c1d9f5 --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/CsrUtils.kt @@ -0,0 +1,16 @@ +package com.egm.stellio.search.csr + +import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.shared.util.ngsiLdDateTime +import com.egm.stellio.shared.util.toUri + +object CsrUtils { + fun gimmeRawCSR() = ContextSourceRegistration( + id = "urn:ngsi-ld:ContextSourceRegistration:test".toUri(), + endpoint = "http://localhost:8089".toUri(), + information = emptyList(), + operations = listOf(Operation.FEDERATION_OPS), + createdAt = ngsiLdDateTime(), + ) +} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt index a224c9473..61a8eae52 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceCallerTests.kt @@ -1,5 +1,6 @@ package com.egm.stellio.search.csr.service +import com.egm.stellio.search.csr.CsrUtils.gimmeRawCSR import com.egm.stellio.search.csr.model.* import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonUtils.serializeObject @@ -21,13 +22,6 @@ class ContextSourceCallerTests { private val apiaryId = "urn:ngsi-ld:Apiary:TEST" - private fun gimmeRawCSR() = ContextSourceRegistration( - id = "urn:ngsi-ld:ContextSourceRegistration:test".toUri(), - endpoint = "http://localhost:8089".toUri(), - information = emptyList(), - operations = listOf(Operation.FEDERATION_OPS), - createdAt = ngsiLdDateTime(), - ) private val emptyParams = LinkedMultiValueMap() private val entityWithSysAttrs = """ diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt index ead974e27..b566ef75f 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceUtilsTests.kt @@ -1,7 +1,9 @@ package com.egm.stellio.search.csr.service +import arrow.core.left import arrow.core.right import com.egm.stellio.search.csr.model.ContextSourceRegistration +import com.egm.stellio.search.csr.model.MiscellaneousWarning import com.egm.stellio.search.csr.model.Mode import com.egm.stellio.shared.model.CompactedAttributeInstance import com.egm.stellio.shared.model.CompactedEntity @@ -86,22 +88,6 @@ class ContextSourceUtilsTests { assertEquals(entityWithName + entityWithLastName + entityWithSurName, mergedEntity.getOrNull()) } - @Test - fun `merge entity should call mergeAttribute or mergeTypeOrScope when keys are equal`() = runTest { - mockkObject(ContextSourceUtils) { - every { ContextSourceUtils.mergeAttribute(any(), any(), any()) } returns listOf( - nameAttribute - ).right() - every { ContextSourceUtils.mergeTypeOrScope(any(), any()) } returns listOf("Beehive") - ContextSourceUtils.mergeEntities( - entityWithName, - listOf(entityWithName to auxiliaryCSR, entityWithName to inclusiveCSR) - ) - verify(exactly = 2) { ContextSourceUtils.mergeAttribute(any(), any(), any()) } - verify(exactly = 2) { ContextSourceUtils.mergeTypeOrScope(any(), any()) } - } - } - @Test fun `merge entity should merge the types correctly `() = runTest { val mergedEntity = ContextSourceUtils.mergeEntities( @@ -192,4 +178,38 @@ class ContextSourceUtilsTests { (mergedEntity.getOrNull()?.get(NGSILD_CREATED_AT_TERM)) ) } + + @Test + fun `merge entity should merge each entity using getMergeNewValues and return the received warnings`() = runTest { + val warning1 = MiscellaneousWarning("1", inclusiveCSR) + val warning2 = MiscellaneousWarning("2", inclusiveCSR) + mockkObject(ContextSourceUtils) { + every { ContextSourceUtils.getMergeNewValues(any(), any(), any()) } returns + warning1.left() andThen warning2.left() + + val (warnings, entity) = ContextSourceUtils.mergeEntities( + entityWithName, + listOf(entityWithName to inclusiveCSR, entityWithName to inclusiveCSR) + ).toPair() + verify(exactly = 2) { ContextSourceUtils.getMergeNewValues(any(), any(), any()) } + assertThat(warnings).hasSize(2).contains(warning1, warning2) + assertEquals(entityWithName, entity) + } + } + + @Test + fun `merge entity should call mergeAttribute or mergeTypeOrScope when keys are equal`() = runTest { + mockkObject(ContextSourceUtils) { + every { ContextSourceUtils.mergeAttribute(any(), any(), any()) } returns listOf( + nameAttribute + ).right() + every { ContextSourceUtils.mergeTypeOrScope(any(), any()) } returns listOf("Beehive") + ContextSourceUtils.mergeEntities( + entityWithName, + listOf(entityWithName to auxiliaryCSR, entityWithName to inclusiveCSR) + ) + verify(exactly = 2) { ContextSourceUtils.mergeAttribute(any(), any(), any()) } + verify(exactly = 2) { ContextSourceUtils.mergeTypeOrScope(any(), any()) } + } + } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt index 38d46b112..fe6a3a29a 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt @@ -3,6 +3,10 @@ package com.egm.stellio.search.entity.web import arrow.core.left import arrow.core.right import com.egm.stellio.search.common.config.SearchProperties +import com.egm.stellio.search.csr.CsrUtils.gimmeRawCSR +import com.egm.stellio.search.csr.model.MiscellaneousWarning +import com.egm.stellio.search.csr.model.NGSILDWarning +import com.egm.stellio.search.csr.service.ContextSourceCaller import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.entity.model.* import com.egm.stellio.search.entity.service.EntityQueryService @@ -804,6 +808,43 @@ class EntityHandlerTests { ) } + @Test + fun `get entity by id should return the warnings send by the csr and update the csr status`() { + val csr = gimmeRawCSR() + coEvery { + entityQueryService.queryEntity("urn:ngsi-ld:BeeHive:TEST".toUri(), sub.getOrNull()) + } returns ResourceNotFoundException("no entity").left() + + coEvery { + contextSourceRegistrationService + .getContextSourceRegistrations(any(), any(), any()) + } returns listOf(csr, csr) + + mockkObject(ContextSourceCaller) { + coEvery { + ContextSourceCaller.getDistributedInformation(any(), any(), any(), any()) + } returns MiscellaneousWarning( + "message\nwith\nline\nbreaks", + csr + ).left() andThen + MiscellaneousWarning("message", csr).left() + + coEvery { contextSourceRegistrationService.updateContextSourceStatus(any(), any()) } returns Unit + webClient.get() + .uri("/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:TEST") + .header(HttpHeaders.LINK, AQUAC_HEADER_LINK) + .exchange() + .expectStatus().isNotFound + .expectHeader().valueEquals( + NGSILDWarning.HEADER_NAME, + "199 urn:ngsi-ld:ContextSourceRegistration:test \"message with line breaks\"", + "199 urn:ngsi-ld:ContextSourceRegistration:test \"message\"" + ) + + coVerify(exactly = 2) { contextSourceRegistrationService.updateContextSourceStatus(any(), false) } + } + } + @Test fun `get entities by type should not include temporal properties if query param sysAttrs is not present`() { coEvery { entityQueryService.queryEntities(any(), any()) } returns Pair(