Merge branch 'develop'

api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt

@@ -52,6 +52,13 @@ class ApiGatewayApplication(
                     }
                     .uri("http://$searchServiceUrl:8083")
             }
+            .route { p ->
+                p.path("/ngsi-ld/v1/temporal/entityOperations/**")
+                    .filters {
+                        it.filter(filterFactory.apply())
+                    }
+                    .uri("http://$searchServiceUrl:8083")
+            }
             .route { p ->
                 p.path("/ngsi-ld/v1/subscriptions/**")
                     .filters {

build.gradle.kts

@@ -59,7 +59,7 @@ subprojects {
         // it provides support for JWT decoding and verification
         implementation("org.springframework.security:spring-security-oauth2-jose")
 
-        implementation("org.springframework.cloud:spring-cloud-starter-stream-kafka")
+        implementation("org.springframework.kafka:spring-kafka")
 
         implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
         implementation("com.github.jsonld-java:jsonld-java:0.13.2")

docker-compose.yml

@@ -25,7 +25,7 @@ services:
     depends_on:
       - zookeeper
   neo4j:
-    image: neo4j:4.0
+    image: neo4j:4.2
     container_name: stellio-neo4j
     volumes:
       - stellio-neo4j-storage:/data
@@ -34,7 +34,7 @@ services:
       - NEO4J_dbms_default__database=${NEO4J_DEFAULT_DATABASE}
       - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD}
       - "NEO4J_dbms_security_procedures_unrestricted=apoc.*"
-      - "NEO4J_dbms_security_procedures_whitelist=apoc.*"
+      - "NEO4J_dbms_security_procedures_allowlist=apoc.*"
       - NEO4JLABS_PLUGINS=["apoc"]
     ports:
       - 7474:7474

entity-service/config/detekt/baseline.xml

@@ -11,12 +11,11 @@
     <ID>LargeClass:StandaloneNeo4jSearchRepositoryTests.kt$StandaloneNeo4jSearchRepositoryTests</ID>
     <ID>LongMethod:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap&lt;String, String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:EntityServiceTests.kt$EntityServiceTests$@Test fun `it should create a new multi attribute property`()</ID>
-    <ID>LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( params: Map&lt;String, Any?&gt;, page: Int, limit: Int, contexts: List&lt;String&gt; ): String</ID>
-    <ID>LongMethod:StandaloneNeo4jSearchRepositoryTests.kt$StandaloneNeo4jSearchRepositoryTests$@Test fun `it should return an entity if type and combinations of time, string and relationship are correct`()</ID>
+    <ID>LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, page: Int, limit: Int, contexts: List&lt;String&gt; ): String</ID>
     <ID>LongParameterList:Attribute.kt$Attribute$( @Transient val attributeType: String, var observedAt: ZonedDateTime? = null, @JsonIgnore val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), @JsonIgnore var modifiedAt: ZonedDateTime? = null, @Convert(UriConverter::class) var datasetId: URI? = null, @Relationship(type = "HAS_VALUE") val properties: MutableList&lt;Property&gt; = mutableListOf(), @Relationship(type = "HAS_OBJECT") val relationships: MutableList&lt;com.egm.stellio.entity.model.Relationship&gt; = mutableListOf() )</ID>
     <ID>LongParameterList:Entity.kt$Entity$( @Id @JsonProperty("@id") @Convert(UriConverter::class) val id: URI, @Labels @JsonProperty("@type") val type: List&lt;String&gt;, @JsonIgnore val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), @JsonIgnore var modifiedAt: ZonedDateTime? = null, @JsonIgnore var location: String? = null, @Relationship(type = "HAS_VALUE") val properties: MutableList&lt;Property&gt; = mutableListOf(), @Relationship(type = "HAS_OBJECT") val relationships: MutableList&lt;com.egm.stellio.entity.model.Relationship&gt; = mutableListOf(), var contexts: List&lt;String&gt; = mutableListOf() )</ID>
-    <ID>LongParameterList:EntityService.kt$EntityService$( params: Map&lt;String, Any?&gt;, userSub: String, page: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean )</ID>
-    <ID>LongParameterList:EntityService.kt$EntityService$( params: Map&lt;String, Any?&gt;, userSub: String, page: Int, limit: Int, contexts: List&lt;String&gt;, includeSysAttrs: Boolean )</ID>
+    <ID>LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, page: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean )</ID>
+    <ID>LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, page: Int, limit: Int, contexts: List&lt;String&gt;, includeSysAttrs: Boolean )</ID>
     <ID>MaxLineLength:EntityHandlerTests.kt$EntityHandlerTests$ </ID>
     <ID>MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (a:</ID>
     <ID>MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (entity:</ID>

entity-service/docker-compose.yml

@@ -23,15 +23,15 @@ services:
     depends_on:
       - zookeeper
   neo4j:
-    image: neo4j:4.0
+    image: neo4j:4.2
     volumes:
       - stellio-entity-neo4j-storage:/data
     environment:
       - NEO4J_dbms_allow__upgrade=${NEO4J_ALLOW_UPGRADE}
       - NEO4J_dbms_default__database=${NEO4J_DEFAULT_DATABASE}
       - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD}
       - "NEO4J_dbms_security_procedures_unrestricted=apoc.*"
-      - "NEO4J_dbms_security_procedures_whitelist=apoc.*"
+      - "NEO4J_dbms_security_procedures_allowlist=apoc.*"
       - NEO4JLABS_PLUGINS=["apoc"]
     ports:
       - 7474:7474

entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt

@@ -1,8 +1,10 @@
 package com.egm.stellio.entity.authorization
 
+import com.egm.stellio.entity.authorization.AuthorizationService.Companion.CLIENT_LABEL
 import com.egm.stellio.entity.authorization.AuthorizationService.Companion.EGM_ROLES
 import com.egm.stellio.entity.authorization.AuthorizationService.Companion.R_CAN_ADMIN
 import com.egm.stellio.entity.authorization.AuthorizationService.Companion.SERVICE_ACCOUNT_ID
+import com.egm.stellio.entity.authorization.AuthorizationService.Companion.USER_LABEL
 import com.egm.stellio.entity.model.Property
 import com.egm.stellio.entity.model.Relationship
 import com.egm.stellio.shared.util.JsonLdUtils.EGM_SPECIFIC_ACCESS_POLICY
@@ -127,10 +129,18 @@ class Neo4jAuthorizationRepository(
     fun createAdminLinks(userId: URI, relationships: List<Relationship>, entitiesId: List<URI>): List<URI> {
         val query =
             """
+            CALL {
+                MATCH (user:Entity:`$USER_LABEL`)
+                WHERE user.id = ${'$'}userId
+                RETURN user
+                UNION
+                MATCH (user:Entity:`$CLIENT_LABEL`)
+                WHERE (user)-[:HAS_VALUE]->(:Property { name: "$SERVICE_ACCOUNT_ID", value: ${'$'}userId })
+                RETURN user
+            }
+            WITH user
             UNWIND ${'$'}relPropsAndTargets AS relPropAndTarget
-            MATCH (user:Entity), (target:Entity { id: relPropAndTarget.second })
-            WHERE (user.id = ${'$'}userId 
-                OR (user)-[:HAS_VALUE]->(:Property { name: "$SERVICE_ACCOUNT_ID", value: ${'$'}userId }))
+            MATCH (target:Entity { id: relPropAndTarget.second })
             CREATE (user)-[:HAS_OBJECT]->(r:Attribute:Relationship:`$R_CAN_ADMIN`)-[:rCanAdmin]->(target)
             SET r = relPropAndTarget.first
             RETURN r.id as id

entity-service/src/main/kotlin/com/egm/stellio/entity/config/KafkaConfig.kt

@@ -0,0 +1,23 @@
+package com.egm.stellio.entity.config
+
+import org.springframework.boot.autoconfigure.kafka.ConcurrentKafkaListenerContainerFactoryConfigurer
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory
+import org.springframework.kafka.core.ConsumerFactory
+import org.springframework.kafka.listener.SeekToCurrentErrorHandler
+
+@Configuration
+class KafkaConfig {
+
+    @Bean
+    fun kafkaListenerContainerFactory(
+        configurer: ConcurrentKafkaListenerContainerFactoryConfigurer,
+        kafkaConsumerFactory: ConsumerFactory<Any?, Any?>?
+    ): ConcurrentKafkaListenerContainerFactory<*, *>? {
+        val factory = ConcurrentKafkaListenerContainerFactory<Any, Any>()
+        configurer.configure(factory, kafkaConsumerFactory)
+        factory.setErrorHandler(SeekToCurrentErrorHandler())
+        return factory
+    }
+}

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt

@@ -2,6 +2,7 @@ package com.egm.stellio.entity.repository
 
 import com.egm.stellio.entity.authorization.AuthorizationService.Companion.USER_PREFIX
 import com.egm.stellio.entity.authorization.Neo4jAuthorizationService
+import com.egm.stellio.shared.model.QueryParams
 import com.egm.stellio.shared.util.toUri
 import org.neo4j.ogm.session.Session
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
@@ -16,21 +17,27 @@ class Neo4jSearchRepository(
 ) : SearchRepository {
 
     override fun getEntities(
-        params: Map<String, Any?>,
+        queryParams: QueryParams,
         userSub: String,
         page: Int,
         limit: Int,
         contexts: List<String>
     ): Pair<Int, List<URI>> {
         val query = if (neo4jAuthorizationService.userIsAdmin(userSub))
-            QueryUtils.prepareQueryForEntitiesWithoutAuthentication(params, page, limit, contexts)
+            QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, page, limit, contexts)
         else
-            QueryUtils.prepareQueryForEntitiesWithAuthentication(params, page, limit, contexts)
+            QueryUtils.prepareQueryForEntitiesWithAuthentication(queryParams, page, limit, contexts)
 
         val result = session.query(query, mapOf("userId" to USER_PREFIX + userSub), true)
-        return Pair(
-            (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0,
-            result.map { (it["id"] as String).toUri() }
-        )
+        return if (limit == 0)
+            Pair(
+                (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0,
+                emptyList()
+            )
+        else
+            Pair(
+                (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0,
+                result.map { (it["id"] as String).toUri() }
+            )
     }
 }

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt

@@ -7,25 +7,23 @@ import com.egm.stellio.entity.util.isDateTime
 import com.egm.stellio.entity.util.isFloat
 import com.egm.stellio.entity.util.isRelationshipTarget
 import com.egm.stellio.entity.util.isTime
+import com.egm.stellio.shared.model.QueryParams
 import com.egm.stellio.shared.util.JsonLdUtils.EGM_SPECIFIC_ACCESS_POLICY
 import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdKey
 import java.util.regex.Pattern
 
 object QueryUtils {
 
     fun prepareQueryForEntitiesWithAuthentication(
-        params: Map<String, Any?>,
+        queryParams: QueryParams,
         page: Int,
         limit: Int,
         contexts: List<String>
     ): String {
-        val ids = params["id"] as List<String>?
-        val type = params["type"] as String
-        val idPattern = params["idPattern"] as String?
-        val rawQuery = params["q"] as String
-        val formattedIds = ids?.map { "'$it'" }
+        val (id, expandedType, idPattern, q) = queryParams
+        val formattedIds = id?.map { "'$it'" }
         val pattern = Pattern.compile("([^();|]+)")
-        val innerQuery = buildInnerQuery(rawQuery, pattern, contexts)
+        val innerQuery = q?.let { buildInnerQuery(q, pattern, contexts) } ?: ""
 
         val matchUserClause =
             """
@@ -42,7 +40,7 @@ object QueryUtils {
             """.trimIndent()
 
         val idClause =
-            if (ids != null) "AND entity.id in $formattedIds"
+            if (id != null) "AND entity.id in $formattedIds"
             else ""
 
         val idPatternClause =
@@ -56,10 +54,10 @@ object QueryUtils {
             }
 
         val matchEntityClause =
-            if (type.isEmpty())
+            if (expandedType == null)
                 "(entity:Entity)"
             else
-                "(entity:Entity:`$type`)"
+                "(entity:Entity:`$expandedType`)"
 
         val matchEntitiesClause =
             """
@@ -95,13 +93,17 @@ object QueryUtils {
             return entity.id as entityId
             """.trimIndent()
 
-        val pagingClause =
+        val pagingClause = if (limit == 0)
             """
-            WITH collect(distinct entityId) as entityIds, count(entityId) as count
-            UNWIND entityIds as id
-            RETURN id, count
-            ORDER BY id
-            SKIP ${(page - 1) * limit} LIMIT $limit
+            RETURN count(entityId) as count
+            """.trimIndent()
+        else
+            """
+                WITH collect(distinct entityId) as entityIds, count(entityId) as count
+                UNWIND entityIds as id
+                RETURN id, count
+                ORDER BY id
+                SKIP ${(page - 1) * limit} LIMIT $limit
             """.trimIndent()
 
         return """
@@ -119,31 +121,28 @@ object QueryUtils {
     }
 
     fun prepareQueryForEntitiesWithoutAuthentication(
-        params: Map<String, Any?>,
+        queryParams: QueryParams,
         page: Int,
         limit: Int,
         contexts: List<String>
     ): String {
-        val ids = params["id"] as List<String>?
-        val type = params["type"] as String
-        val idPattern = params["idPattern"] as String?
-        val rawQuery = params["q"] as String
-        val formattedIds = ids?.map { "'$it'" }
+        val (id, expandedType, idPattern, q) = queryParams
+        val formattedIds = id?.map { "'$it'" }
         val pattern = Pattern.compile("([^();|]+)")
-        val innerQuery = buildInnerQuery(rawQuery, pattern, contexts)
+        val innerQuery = q?.let { buildInnerQuery(q, pattern, contexts) } ?: ""
 
         val matchClause =
-            if (type.isEmpty())
+            if (expandedType == null)
                 "MATCH (entity:Entity)"
             else
-                "MATCH (entity:`$type`)"
+                "MATCH (entity:`$expandedType`)"
 
         val whereClause =
-            if (innerQuery.isNotEmpty() || ids != null || idPattern != null) " WHERE "
+            if (innerQuery.isNotEmpty() || id != null || idPattern != null) " WHERE "
             else ""
 
         val idClause =
-            if (ids != null)
+            if (id != null)
                 """
                     entity.id in $formattedIds
                     ${if (idPattern != null || innerQuery.isNotEmpty()) " AND " else ""}
@@ -158,7 +157,11 @@ object QueryUtils {
                 """
             else ""
 
-        val pagingClause =
+        val pagingClause = if (limit == 0)
+            """
+            RETURN count(entity) as count
+            """.trimIndent()
+        else
             """
             WITH collect(entity) as entities, count(entity) as count
             UNWIND entities as entity

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt

@@ -1,13 +1,14 @@
 package com.egm.stellio.entity.repository
 
+import com.egm.stellio.shared.model.QueryParams
 import java.net.URI
 
 interface SearchRepository {
 
     /**
      * Searches the requested entities and applies permissions checks in authentication enabled mode.
      *
-     * @param params query parameters.
+     * @param queryParams query parameters.
      * @param userSub to be used in authentication enabled mode to apply permissions checks.
      * @param page page number for pagination.
      * @param limit limit number for pagination.
@@ -17,7 +18,7 @@ interface SearchRepository {
      *  @property second list of matching entities ids as requested by pagination sorted by entity id.
      */
     fun getEntities(
-        params: Map<String, Any?>,
+        queryParams: QueryParams,
         userSub: String,
         page: Int,
         limit: Int,

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt

@@ -1,5 +1,6 @@
 package com.egm.stellio.entity.repository
 
+import com.egm.stellio.shared.model.QueryParams
 import com.egm.stellio.shared.util.toUri
 import org.neo4j.ogm.session.Session
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
@@ -13,16 +14,20 @@ class StandaloneNeo4jSearchRepository(
 ) : SearchRepository {
 
     override fun getEntities(
-        params: Map<String, Any?>,
+        queryParams: QueryParams,
         userSub: String,
         page: Int,
         limit: Int,
         contexts: List<String>
     ): Pair<Int, List<URI>> {
-        val query = QueryUtils.prepareQueryForEntitiesWithoutAuthentication(params, page, limit, contexts)
+        val query = QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, page, limit, contexts)
         val result = session.query(query, emptyMap<String, Any>(), true)
-
-        return Pair(
+        return if (limit == 0)
+            Pair(
+                (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0,
+                emptyList()
+            )
+        else Pair(
             (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0,
             result.map { (it["id"] as String).toUri() }
         )