Merge branch 'develop'

.github/workflows/cla.yml

@@ -12,7 +12,7 @@ jobs:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
         # Beta Release
-        uses: cla-assistant/github-action@v2.1.3-beta
+        uses: cla-assistant/github-action@v2.2.1
         env: 
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret

api-gateway/build.gradle.kts

@@ -9,7 +9,7 @@ dependencies {
     implementation("org.springframework.cloud:spring-cloud-starter-gateway")
     implementation("org.springframework.boot:spring-boot-starter-oauth2-client")
 
-    detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.20.0")
+    detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.21.0")
 }
 
 springBoot {

build.gradle.kts

@@ -18,13 +18,13 @@ plugins {
     java // why did I have to add that ?!
     // only apply the plugin in the subprojects requiring it because it expects a Spring Boot app
     // and the shared lib is obviously not one
-    id("org.springframework.boot") version "2.7.1" apply false
-    id("io.spring.dependency-management") version "1.0.12.RELEASE" apply false
+    id("org.springframework.boot") version "2.7.3" apply false
+    id("io.spring.dependency-management") version "1.0.13.RELEASE" apply false
     kotlin("jvm") version "1.6.21" apply false
     kotlin("plugin.spring") version "1.6.21" apply false
-    id("org.jlleitschuh.gradle.ktlint") version "10.3.0"
-    id("com.google.cloud.tools.jib") version "3.2.1" apply false
-    id("io.gitlab.arturbosch.detekt") version "1.20.0" apply false
+    id("org.jlleitschuh.gradle.ktlint") version "11.0.0"
+    id("com.google.cloud.tools.jib") version "3.3.0" apply false
+    id("io.gitlab.arturbosch.detekt") version "1.21.0" apply false
     id("org.sonarqube") version "3.4.0.2513"
     jacoco
 }
@@ -82,15 +82,15 @@ subprojects {
             exclude(module = "mockito-core")
         }
         testImplementation("com.ninja-squad:springmockk:3.1.1")
-        testImplementation("io.mockk:mockk:1.12.4")
+        testImplementation("io.mockk:mockk:1.12.7")
         testImplementation("io.projectreactor:reactor-test")
         testImplementation("org.springframework.security:spring-security-test")
         testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test")
     }
 
     tasks.withType<KotlinCompile> {
         kotlinOptions {
-            freeCompilerArgs = listOf("-Xjsr305=strict")
+            freeCompilerArgs = listOf("-Xjsr305=strict", "-opt-in=kotlin.RequiresOptIn")
             jvmTarget = "17"
         }
     }

config/detekt/detekt.yml

@@ -78,6 +78,8 @@ style:
 #    active: true
 #  DataClassShouldBeImmutable:
 #    active: true
+  DestructuringDeclarationWithTooManyEntries:
+    active: false
   ExplicitCollectionElementAccessMethod:
     active: true
   ExpressionBodySyntax:

config/kafka/update_run.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Docker workaround: Remove check for KAFKA_ZOOKEEPER_CONNECT parameter
+sed -i '/KAFKA_ZOOKEEPER_CONNECT/d' /etc/confluent/docker/configure
+
+# Docker workaround: Ignore cub zk-ready
+sed -i 's/cub zk-ready/echo ignore zk-ready/' /etc/confluent/docker/ensure
+
+# KRaft required step: Format the storage directory with a new cluster ID
+echo "kafka-storage format --ignore-formatted -t $(kafka-storage random-uuid) -c /etc/kafka/kafka.properties" >> /etc/confluent/docker/ensure
+

docker-compose.yml

@@ -1,29 +1,26 @@
 version: '3.9'
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:5.5.7
-    container_name: stellio-zookeeper
-    ports:
-      - "2181:2181"
-    environment:
-      ZOOKEEPER_SERVER_ID: 1
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
   kafka:
-    image: confluentinc/cp-kafka:5.5.7
+    image: confluentinc/cp-kafka:7.2.0
     container_name: stellio-kafka
+    hostname: stellio-kafka
     ports:
       - "29092:29092"
     environment:
-      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://localhost:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_BROKER_ID: 1
-      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://stellio-kafka:9092,PLAINTEXT_HOST://localhost:29092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    depends_on:
-      - zookeeper
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@stellio-kafka:29093'
+      KAFKA_LISTENERS: CONTROLLER://stellio-kafka:29093,PLAINTEXT://stellio-kafka:9092,PLAINTEXT_HOST://0.0.0.0:29092
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
+    volumes:
+      - ./config/kafka/update_run.sh:/tmp/update_run.sh
+    command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'"
   neo4j:
     image: neo4j:4.4
     container_name: stellio-neo4j

entity-service/build.gradle.kts

@@ -13,11 +13,11 @@ plugins {
 
 dependencies {
     implementation("org.springframework.boot:spring-boot-starter-data-neo4j")
-    implementation("eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter:1.8.3")
+    implementation("eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter:1.10.1")
     implementation("org.springframework.boot:spring-boot-starter-cache")
     implementation(project(":shared"))
 
-    detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.20.0")
+    detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.21.0")
 
     developmentOnly("org.springframework.boot:spring-boot-devtools")
 

entity-service/config/detekt/baseline.xml

@@ -10,12 +10,14 @@
     <ID>LongMethod:EntityServiceTests.kt$EntityServiceTests$@Test fun `it should create a new multi attribute property`()</ID>
     <ID>LongMethod:Neo4jAuthorizationServiceTest.kt$Neo4jAuthorizationServiceTest$@Test fun `it should get and transform authorized entities into JSON-LD entities`()</ID>
     <ID>LongMethod:Neo4jRepositoryTests.kt$Neo4jRepositoryTests$@Test fun `it should retrieve details of attributes with one geo property`()</ID>
+    <ID>LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, geoQuery: GeoQuery, contexts: List&lt;String&gt; ): String</ID>
     <ID>LongParameterList:EntityAccessControlHandlerTests.kt$EntityAccessControlHandlerTests$( id: String, type: String, datasetId: String? = null, right: String, specificAccessPolicy: String? = null, rCanReadUser: URI? = null, createdAt: String? = null, modifiedAt: String? = null, context: String )</ID>
     <ID>LongParameterList:EntityEventService.kt$EntityEventService$( sub: String?, entityId: URI, attributeName: String, datasetId: URI? = null, deleteAll: Boolean, contexts: List&lt;String&gt; )</ID>
     <ID>LongParameterList:EntityEventService.kt$EntityEventService$( sub: String?, entityId: URI, jsonLdAttributes: Map&lt;String, Any&gt;, updateResult: UpdateResult, overwrite: Boolean, contexts: List&lt;String&gt; )</ID>
     <ID>LongParameterList:Neo4jRepositoryTests.kt$Neo4jRepositoryTests$( id: URI, types: List&lt;String&gt;, properties: MutableList&lt;Property&gt; = mutableListOf(), location: String? = null, operationSpace: String? = null, observationSpace: String? = null )</ID>
     <ID>MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (a:</ID>
     <ID>MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (entity:</ID>
+    <ID>NestedBlockDepth:Neo4jSearchRepository.kt$Neo4jSearchRepository$override fun getEntities( queryParams: QueryParams, sub: Option&lt;Sub&gt;, contexts: List&lt;String&gt; ): Pair&lt;Int, List&lt;URI&gt;&gt;</ID>
     <ID>TooManyFunctions:AuthorizationService.kt$AuthorizationService</ID>
     <ID>TooManyFunctions:EntityService.kt$EntityService</ID>
     <ID>TooManyFunctions:Neo4jAuthorizationService.kt$Neo4jAuthorizationService : AuthorizationService</ID>

entity-service/docker-compose.yml

@@ -1,27 +1,26 @@
 version: '3.5'
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:5.5.7
-    ports:
-      - "2181:2181"
-    environment:
-      ZOOKEEPER_SERVER_ID: 1
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
   kafka:
-    image: confluentinc/cp-kafka:5.5.7
+    image: confluentinc/cp-kafka:7.2.0
+    container_name: stellio-kafka
+    hostname: stellio-kafka
     ports:
       - "29092:29092"
     environment:
-      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL://localhost:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_BROKER_ID: 1
-      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://stellio-kafka:9092,PLAINTEXT_HOST://localhost:29092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    depends_on:
-      - zookeeper
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@stellio-kafka:29093'
+      KAFKA_LISTENERS: CONTROLLER://stellio-kafka:29093,PLAINTEXT://stellio-kafka:9092,PLAINTEXT_HOST://0.0.0.0:29092
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
+    volumes:
+      - ../config/kafka/update_run.sh:/tmp/update_run.sh
+    command: "bash -c 'if [ ! -f /tmp/update_run.sh ]; then echo \"ERROR: Did you forget the update_run.sh file that came with this docker-compose.yml file?\" && exit 1 ; else /tmp/update_run.sh && /etc/confluent/docker/run ; fi'"
   neo4j:
     image: neo4j:4.4
     volumes:

entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt

@@ -407,9 +407,7 @@ class Neo4jAuthorizationRepository(
                 rCanReadUsers = usersRightsOnEntity.valuesForRight(AccessRight.R_CAN_READ),
                 createdAt = (entityNode.get("createdAt") as DateTimeValue).asZonedDateTime(),
                 modifiedAt = (entityNode.get("modifiedAt") as? DateTimeValue)?.asZonedDateTime(),
-                specificAccessPolicy = specificAccessPolicy?.let { it ->
-                    AuthContextModel.SpecificAccessPolicy.valueOf(it)
-                }
+                specificAccessPolicy = specificAccessPolicy?.let { AuthContextModel.SpecificAccessPolicy.valueOf(it) }
             )
         }.toSet()
 
@@ -464,7 +462,8 @@ class Neo4jAuthorizationRepository(
                 (result.firstOrNull()?.get("groups") as List<Map<String, Any>>).map {
                     Group(
                         id = (it["groupId"] as String).toUri(),
-                        name = it["groupName"] as String
+                        name = it["groupName"] as String,
+                        isMember = true
                     )
                 }
             )

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt

@@ -275,31 +275,6 @@ class Neo4jRepository(
         return neo4jClient.query(relationshipTypeQuery).bindAll(parameters).run().counters().containsUpdates()
     }
 
-    fun updateTargetOfRelationship(
-        attributeId: URI,
-        relationshipType: String,
-        oldRelationshipObjectId: URI,
-        newRelationshipObjectId: URI
-    ): Int {
-        val relationshipTypeQuery =
-            """
-            MATCH (a:Attribute { id: ${'$'}attributeId })-[v:$relationshipType]
-                ->(e { id: ${'$'}oldRelationshipObjectId })
-            MERGE (target { id: ${'$'}newRelationshipObjectId })
-            ON CREATE SET target:PartialEntity
-            DETACH DELETE v
-            MERGE (a)-[:$relationshipType]->(target)
-            SET a.objectId = ${'$'}newRelationshipObjectId
-            """.trimIndent()
-
-        val parameters = mapOf(
-            "attributeId" to attributeId.toString(),
-            "oldRelationshipObjectId" to oldRelationshipObjectId.toString(),
-            "newRelationshipObjectId" to newRelationshipObjectId.toString()
-        )
-        return neo4jClient.query(relationshipTypeQuery).bindAll(parameters).run().counters().nodesDeleted()
-    }
-
     fun updateGeoPropertyOfEntity(entityId: URI, propertyKey: String, geoProperty: NgsiLdGeoPropertyInstance): Int {
         val query =
             """
@@ -313,6 +288,19 @@ class Neo4jRepository(
     }
 
     fun deleteEntity(entityId: URI): Pair<Int, Int> {
+        // first update incoming relationships to point to a partial entity with the same id as the deleted one
+        val relationshipTypeQuery =
+            """
+            MATCH (r:Relationship)-[relType]->(e { id: ${'$'}entityId })
+            MERGE (target:PartialEntity { id: ${'$'}entityId })
+            WITH relType, target
+            CALL apoc.refactor.to(relType, target)
+            YIELD input, output
+            RETURN input, output
+            """.trimIndent()
+
+        neo4jClient.query(relationshipTypeQuery).bind(entityId.toString()).to("entityId").run()
+
         /**
          * Delete :
          *
@@ -339,9 +327,8 @@ class Neo4jRepository(
             OPTIONAL MATCH (rel)-[:HAS_VALUE]->(propOfRel:Property)
             WITH n, prop, relOfProp, propOfProp, rel, propOfRel
             OPTIONAL MATCH (rel)-[:HAS_OBJECT]->(relOfRel:Relationship)
-            WITH n, prop, relOfProp, propOfProp, rel, propOfRel, relOfRel            
-            OPTIONAL MATCH (inRel:Relationship)-[]->(n)
-            DETACH DELETE n, prop, relOfProp, propOfProp, rel, propOfRel, relOfRel, inRel
+            WITH n, prop, relOfProp, propOfProp, rel, propOfRel, relOfRel
+            DETACH DELETE n, prop, relOfProp, propOfProp, rel, propOfRel, relOfRel
             """.trimIndent()
 
         val queryStatistics = neo4jClient.query(query).bind(entityId.toString()).to("entityId").run().counters()

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt

@@ -21,20 +21,21 @@ class Neo4jSearchRepository(
         sub: Option<Sub>,
         contexts: List<String>
     ): Pair<Int, List<URI>> {
+        val userAndGroupIds = neo4jAuthorizationService.getSubjectGroups(sub)
+            .plus(neo4jAuthorizationService.getSubjectUri(sub))
+            .map { it.toString() }
+
         val query = if (neo4jAuthorizationService.userIsAdmin(sub))
             QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, contexts)
         else
             QueryUtils.prepareQueryForEntitiesWithAuthentication(queryParams, contexts)
 
-        val userAndGroupIds = neo4jAuthorizationService.getSubjectGroups(sub)
-            .plus(neo4jAuthorizationService.getSubjectUri(sub))
-            .map { it.toString() }
-
         val result = neo4jClient
             .query(query)
             .bind(userAndGroupIds).to("userAndGroupIds")
             .fetch()
             .all()
+
         return prepareResults(queryParams.limit, result)
     }
 }