From d5ecbbe10d316911ee8854a10bbf4f14b956b2d8 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 1 Jul 2021 07:53:52 +0200 Subject: [PATCH 01/56] refactor: avoid cartesian product when filtering entities an user has an access to --- .../entity/authorization/Neo4jAuthorizationRepository.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt index 867396e1d..5f271bde5 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt @@ -29,10 +29,13 @@ class Neo4jAuthorizationRepository( ): List { val query = """ - MATCH (userEntity:Entity), (entity:Entity) - WHERE entity.id IN ${'$'}entitiesId - AND (userEntity.id = ${'$'}userId + MATCH (userEntity:Entity) + WHERE (userEntity.id = ${'$'}userId OR (userEntity)-[:HAS_VALUE]->(:Property { name: "$SERVICE_ACCOUNT_ID", value: ${'$'}userId })) + WITH userEntity + MATCH (entity:Entity) + WHERE entity.id IN ${'$'}entitiesId + WITH userEntity, entity MATCH (userEntity)-[:HAS_OBJECT]->(right:Attribute:Relationship)-[]->(entity:Entity) WHERE size([label IN labels(right) WHERE label IN ${'$'}rights]) > 0 return entity.id as id From 3c4ef75d8aeae67b737454c65e095a93669f5f48 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Mon, 12 Jul 2021 12:15:53 +0200 Subject: [PATCH 02/56] fix(entity): do not fail parsing parameters if type is not provided when querying entities #444 (#446) --- .../egm/stellio/entity/web/EntityHandler.kt | 2 +- .../StandaloneNeo4jSearchRepositoryTests.kt | 22 +++++++++++++++++++ .../stellio/entity/web/EntityHandlerTests.kt | 15 +++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt index f2bfec53b..439bedcfe 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt @@ -135,7 +135,7 @@ class EntityHandler( * with the right parameters values */ val countAndEntities = entityService.searchEntities( - QueryParams(ids, expandJsonLdKey(type as String, contextLink), idPattern, q?.decode()), + QueryParams(ids, type?.let { expandJsonLdKey(type, contextLink) }, idPattern, q?.decode()), userId, page, limit, diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt index 3999ddd2d..ec7caba68 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt @@ -513,6 +513,28 @@ class StandaloneNeo4jSearchRepositoryTests { neo4jRepository.deleteEntity(secondEntity.id) } + @Test + fun `it should return an entity matching given id without specifying a type`() { + val firstEntity = createEntity( + beekeeperUri, + listOf("Beekeeper") + ) + val secondEntity = createEntity( + "urn:ngsi-ld:Beekeeper:1231".toUri(), + listOf("Beekeeper") + ) + val entities = searchRepository.getEntities( + QueryParams(id = listOf("urn:ngsi-ld:Beekeeper:1231")), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).second + + assertFalse(entities.contains(firstEntity.id)) + assertTrue(entities.contains(secondEntity.id)) + } + @Test fun `it should return entities matching given type and ids`() { val firstEntity = createEntity( diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt index 49ff88bf9..c172ceb8a 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt @@ -659,6 +659,21 @@ class EntityHandlerTests { ) } + @Test + fun `get entities should allow a query not including a type request parameter`() { + every { entityService.exists(any()) } returns true + every { entityService.searchEntities(any(), any(), any(), any(), any(), false) } returns Pair( + 0, + emptyList() + ) + + webClient.get() + .uri("/ngsi-ld/v1/entities/?attrs=myProp") + .exchange() + .expectStatus().isOk + .expectBody().json("[]") + } + @Test fun `get entity by id should correctly serialize properties of type DateTime and display sysAttrs asked`() { every { entityService.exists(any()) } returns true From d36d233915b3d2377ced8b39db95d535234dfffc Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Mon, 12 Jul 2021 13:44:16 +0200 Subject: [PATCH 03/56] feat(entity): allow to query on createdAt and modifiedAt properties #445 (#447) --- .../stellio/entity/repository/QueryUtils.kt | 19 ++++--- .../StandaloneNeo4jSearchRepositoryTests.kt | 54 +++++++++++++++++++ 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt index d12decfde..fd8400d98 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt @@ -204,13 +204,18 @@ object QueryUtils { comparablePropertyPath[1] else "value" - """ - EXISTS { - MATCH (entity)-[:HAS_VALUE]->(p:Property) - WHERE p.name = '${expandJsonLdKey(comparablePropertyPath[0], contexts)!!}' - AND p.$comparablePropertyName ${parsedQueryTerm.second} $comparableValue - } - """.trimIndent() + if (listOf("createdAt", "modifiedAt").contains(parsedQueryTerm.first)) + """ + entity.${parsedQueryTerm.first} ${parsedQueryTerm.second} $comparableValue + """.trimIndent() + else + """ + EXISTS { + MATCH (entity)-[:HAS_VALUE]->(p:Property) + WHERE p.name = '${expandJsonLdKey(comparablePropertyPath[0], contexts)!!}' + AND p.$comparablePropertyName ${parsedQueryTerm.second} $comparableValue + } + """.trimIndent() } } .replace(";", " AND ") diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt index ec7caba68..a839aa26d 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt @@ -18,8 +18,10 @@ import org.springframework.context.annotation.Import import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.TestPropertySource import java.net.URI +import java.time.Instant import java.time.LocalDate import java.time.LocalTime +import java.time.ZoneOffset import java.time.ZonedDateTime @SpringBootTest @@ -533,6 +535,58 @@ class StandaloneNeo4jSearchRepositoryTests { assertFalse(entities.contains(firstEntity.id)) assertTrue(entities.contains(secondEntity.id)) + + neo4jRepository.deleteEntity(firstEntity.id) + neo4jRepository.deleteEntity(secondEntity.id) + } + + @Test + fun `it should return an entity matching creation date and an id (without specifying a type)`() { + val firstEntity = createEntity( + beekeeperUri, + listOf("Beekeeper") + ) + val secondEntity = createEntity( + "urn:ngsi-ld:Beekeeper:1231".toUri(), + listOf("Beekeeper") + ) + val entitiesCount = searchRepository.getEntities( + QueryParams(id = listOf("urn:ngsi-ld:Beekeeper:1231"), q = "createdAt>2021-07-10T00:00:00Z"), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).first + + assertEquals(1, entitiesCount) + + neo4jRepository.deleteEntity(firstEntity.id) + neo4jRepository.deleteEntity(secondEntity.id) + } + + @Test + fun `it should return entities matching creation date only (without specifying a type)`() { + val now = Instant.now().atOffset(ZoneOffset.UTC) + val firstEntity = createEntity( + beekeeperUri, + listOf("Beekeeper") + ) + val secondEntity = createEntity( + "urn:ngsi-ld:Beekeeper:1231".toUri(), + listOf("Beekeeper") + ) + val entitiesCount = searchRepository.getEntities( + QueryParams(q = "createdAt>$now"), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).first + + assertEquals(2, entitiesCount) + + neo4jRepository.deleteEntity(firstEntity.id) + neo4jRepository.deleteEntity(secondEntity.id) } @Test From 0a564bce15642624fd15f2b430eba0fb607bd128 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 2 Jul 2021 16:39:39 +0200 Subject: [PATCH 04/56] feat(common): migrate to Spring Boot 2.5 - migrate all Neo4j model and APIs to use only Spring Data Neo4j 6.x - improve some Cypher queries - use a bit more the Neo4jRepository to remove hand made Cypher queries - upgrade to detekt 1.17.1 (required for Gradle 7 / Kotlin 1.5 compatibility) - upgrade to ktlint 10.1.0 (required for Gradle 7 / Kotlin 1.5 compatibility) - upgrade to neo4j-migrations 0.1.4 - upgrade to Gradle 7.x (default for Spring Boot 2.5) - upgrade to Kotlin 1.5 - upgrade to r2dbc 1.2 - remove jcenter repository - integrate Neo4j Testcontainers (instead of relying on external docker-compose file) - integrate Timescale Testcontainers (instead of relying on external docker-compose file) - add a Timescale container interface to be implemented by tests needing it - in each service as the Testcontainers r2dbc dependency tries to set a r2dbc connexion if in shared module - better use of tests slices from Spring Boot to avoid loading the whole app context when not necessary --- api-gateway/build.gradle.kts | 1 - .../apigateway/ApiGatewayApplication.kt | 17 +- build.gradle.kts | 26 +- entity-service/build.gradle.kts | 7 +- entity-service/config/detekt/baseline.xml | 13 +- .../Neo4jAuthorizationRepository.kt | 70 +++--- .../egm/stellio/entity/config/Neo4jConfig.kt | 21 ++ .../entity/config/Neo4jConfiguration.kt | 38 --- .../config/Neo4jUriPropertyConverter.kt | 18 ++ .../config/Neo4jValuePropertyConverter.kt | 27 ++ .../com/egm/stellio/entity/model/Attribute.kt | 108 +------- .../com/egm/stellio/entity/model/Entity.kt | 21 +- .../egm/stellio/entity/model/PartialEntity.kt | 13 +- .../com/egm/stellio/entity/model/Property.kt | 110 ++++++-- .../egm/stellio/entity/model/Relationship.kt | 110 +++++++- .../egm/stellio/entity/model/UriConverter.kt | 18 -- .../entity/repository/EntityRepository.kt | 16 +- .../entity/repository/Neo4jRepository.kt | 236 ++++++------------ .../repository/Neo4jSearchRepository.kt | 8 +- .../entity/repository/PropertyRepository.kt | 17 +- .../repository/RelationshipRepository.kt | 18 +- .../entity/repository/SearchRepository.kt | 2 + .../StandaloneNeo4jSearchRepository.kt | 6 +- .../entity/service/EntityAttributeService.kt | 34 ++- .../stellio/entity/service/EntityService.kt | 42 ++-- .../service/SubscriptionHandlerService.kt | 6 +- .../egm/stellio/entity/util/ParsingUtils.kt | 7 +- .../resources/application-docker.properties | 2 +- .../src/main/resources/application.properties | 18 +- .../Neo4jAuthorizationRepositoryTest.kt | 87 ++----- .../config/TestContainersConfiguration.kt | 34 --- .../entity/config/WebSecurityTestConfig.kt | 3 +- .../entity/config/WithNeo4jContainer.kt | 36 +++ .../repository/EntityRepositoryTests.kt | 6 +- .../entity/repository/Neo4jRepositoryTests.kt | 160 ++++-------- .../repository/Neo4jSearchRepositoryTests.kt | 46 +--- .../StandaloneNeo4jSearchRepositoryTests.kt | 70 ++---- .../service/EntityAttributeServiceTests.kt | 88 +++---- .../entity/service/EntityServiceTests.kt | 22 +- .../SubscriptionHandlerServiceTests.kt | 34 +-- .../stellio/entity/web/EntityHandlerTests.kt | 2 +- .../src/test/resources/logback-test.xml | 4 +- gradle/wrapper/gradle-wrapper.properties | 2 +- search-service/build.gradle.kts | 5 +- search-service/config/detekt/baseline.xml | 3 +- .../egm/stellio/search/model/EntityPayload.kt | 8 + .../search/model/TemporalEntityAttribute.kt | 2 + .../egm/stellio/search/model/TemporalQuery.kt | 2 +- .../service/AttributeInstanceService.kt | 33 ++- .../service/EntityEventListenerService.kt | 11 +- .../stellio/search/service/EntityService.kt | 10 +- .../service/TemporalEntityAttributeService.kt | 84 ++++--- .../search/web/TemporalEntityHandler.kt | 68 +++-- .../src/main/resources/logback-spring.xml | 2 +- .../config/TestContainersConfiguration.kt | 45 ---- .../search/config/TimescaleBasedTests.kt | 17 -- .../search/config/WebSecurityTestConfig.kt | 3 +- .../service/AttributeInstanceServiceTests.kt | 30 +-- .../search/service/EntityServiceTests.kt | 17 +- .../search/service/QueryServiceTests.kt | 8 +- .../TemporalEntityAttributeServiceTests.kt | 29 +-- .../service/TemporalEntityServiceTests.kt | 2 +- .../search/support/WithTimescaleContainer.kt | 45 ++++ .../search/web/TemporalEntityHandlerTests.kt | 21 +- .../resources/application-test.properties | 10 - .../src/test/resources/logback-test.xml | 28 +++ shared/build.gradle.kts | 2 - shared/config/detekt/baseline.xml | 1 + .../egm/stellio/shared/model/NgsiLdEntity.kt | 32 +-- .../egm/stellio/shared/util/JsonLdUtils.kt | 12 +- .../com/egm/stellio/shared/util/UriUtils.kt | 2 +- .../stellio/shared/util/JsonLdUtilsTests.kt | 3 +- .../egm/stellio/shared/util/UriUtilsTests.kt | 37 +++ .../com/egm/stellio/shared/TestContainers.kt | 23 -- subscription-service/build.gradle.kts | 4 +- .../config/detekt/baseline.xml | 10 +- .../stellio/subscription/model/GeoQuery.kt | 3 + .../service/NotificationService.kt | 11 +- .../service/SubscriptionService.kt | 97 ++++--- .../subscription/utils/ParsingUtils.kt | 2 +- .../stellio/subscription/utils/QueryUtils.kt | 3 - .../SubscriptionServiceApplicationTests.kt | 13 - .../config/TestContainersConfiguration.kt | 45 ---- .../config/TimescaleBasedTests.kt | 17 -- .../config/WebSecurityTestConfig.kt | 3 +- .../service/NotificationServiceTests.kt | 29 +-- .../service/SubscriptionServiceTests.kt | 79 +++--- .../support/WithTimescaleContainer.kt | 45 ++++ .../web/SubscriptionHandlerTests.kt | 2 +- .../resources/application-test.properties | 4 - 90 files changed, 1202 insertions(+), 1384 deletions(-) create mode 100644 entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfig.kt delete mode 100644 entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfiguration.kt create mode 100644 entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jUriPropertyConverter.kt create mode 100644 entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt delete mode 100644 entity-service/src/main/kotlin/com/egm/stellio/entity/model/UriConverter.kt delete mode 100644 entity-service/src/test/kotlin/com/egm/stellio/entity/config/TestContainersConfiguration.kt create mode 100644 entity-service/src/test/kotlin/com/egm/stellio/entity/config/WithNeo4jContainer.kt create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/model/EntityPayload.kt delete mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/config/TestContainersConfiguration.kt delete mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/config/TimescaleBasedTests.kt create mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt create mode 100644 search-service/src/test/resources/logback-test.xml create mode 100644 shared/src/test/kotlin/com/egm/stellio/shared/util/UriUtilsTests.kt delete mode 100644 shared/src/testFixtures/kotlin/com/egm/stellio/shared/TestContainers.kt delete mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/SubscriptionServiceApplicationTests.kt delete mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TestContainersConfiguration.kt delete mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TimescaleBasedTests.kt create mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt diff --git a/api-gateway/build.gradle.kts b/api-gateway/build.gradle.kts index 22c1fd307..b1dcced3b 100644 --- a/api-gateway/build.gradle.kts +++ b/api-gateway/build.gradle.kts @@ -5,7 +5,6 @@ plugins { dependencies { implementation("org.springframework.cloud:spring-cloud-starter-gateway") - implementation("org.springframework.cloud:spring-cloud-starter-security") implementation("org.springframework.boot:spring-boot-starter-oauth2-client") } diff --git a/api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt b/api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt index 3eceec540..3251068c0 100644 --- a/api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt +++ b/api-gateway/src/main/kotlin/com/egm/stellio/apigateway/ApiGatewayApplication.kt @@ -5,13 +5,10 @@ import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication import org.springframework.cloud.gateway.route.RouteLocator import org.springframework.cloud.gateway.route.builder.RouteLocatorBuilder -import org.springframework.cloud.security.oauth2.gateway.TokenRelayGatewayFilterFactory import org.springframework.context.annotation.Bean @SpringBootApplication -class ApiGatewayApplication( - private val filterFactory: TokenRelayGatewayFilterFactory -) { +class ApiGatewayApplication { @Value("\${application.entity-service.url:entity-service}") private val entityServiceUrl: String = "" @@ -27,42 +24,42 @@ class ApiGatewayApplication( .route { p -> p.path("/ngsi-ld/v1/entities/**") .filters { - it.filter(filterFactory.apply()) + it.tokenRelay() } .uri("http://$entityServiceUrl:8082") } .route { p -> p.path("/ngsi-ld/v1/entityOperations/**") .filters { - it.filter(filterFactory.apply()) + it.tokenRelay() } .uri("http://$entityServiceUrl:8082") } .route { p -> p.path("/ngsi-ld/v1/types/**") .filters { - it.filter(filterFactory.apply()) + it.tokenRelay() } .uri("http://$entityServiceUrl:8082") } .route { p -> p.path("/ngsi-ld/v1/temporal/entities/**") .filters { - it.filter(filterFactory.apply()) + it.tokenRelay() } .uri("http://$searchServiceUrl:8083") } .route { p -> p.path("/ngsi-ld/v1/temporal/entityOperations/**") .filters { - it.filter(filterFactory.apply()) + it.tokenRelay() } .uri("http://$searchServiceUrl:8083") } .route { p -> p.path("/ngsi-ld/v1/subscriptions/**") .filters { - it.filter(filterFactory.apply()) + it.tokenRelay() } .uri("http://$subscriptionServiceUrl:8084") } diff --git a/build.gradle.kts b/build.gradle.kts index 55d5a0b64..6f47db286 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,28 +5,27 @@ import org.jlleitschuh.gradle.ktlint.reporter.ReporterType val detektConfigFile = file("$rootDir/config/detekt/detekt.yml") -extra["springCloudVersion"] = "Hoxton.SR11" -extra["testcontainersVersion"] = "1.15.1" +extra["springCloudVersion"] = "2020.0.3" +extra["testcontainersVersion"] = "1.15.3" plugins { java // why did I have to add that ?! // only apply the plugin in the subprojects requiring it because it expects a Spring Boot app // and the shared lib is obviously not one - id("org.springframework.boot") version "2.3.10.RELEASE" apply false + id("org.springframework.boot") version "2.5.2" apply false id("io.spring.dependency-management") version "1.0.11.RELEASE" apply false - kotlin("jvm") version "1.3.72" apply false - kotlin("plugin.spring") version "1.3.72" apply false - id("org.jlleitschuh.gradle.ktlint") version "9.3.0" + kotlin("jvm") version "1.5.20" apply false + kotlin("plugin.spring") version "1.5.20" apply false + id("org.jlleitschuh.gradle.ktlint") version "10.1.0" id("com.google.cloud.tools.jib") version "2.5.0" apply false - kotlin("kapt") version "1.3.61" apply false - id("io.gitlab.arturbosch.detekt") version "1.11.2" apply false + kotlin("kapt") version "1.5.20" apply false + id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false id("org.sonarqube") version "3.1.1" } subprojects { repositories { mavenCentral() - jcenter() maven { url = uri("https://dl.bintray.com/arrow-kt/arrow-kt/") } } @@ -71,7 +70,7 @@ subprojects { "kapt"("io.arrow-kt:arrow-meta:0.10.4") - "detektPlugins"("io.gitlab.arturbosch.detekt:detekt-formatting:1.11.2") + "detektPlugins"("io.gitlab.arturbosch.detekt:detekt-formatting:1.17.1") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") @@ -79,15 +78,14 @@ subprojects { runtimeOnly("io.micrometer:micrometer-registry-prometheus") testImplementation("org.springframework.boot:spring-boot-starter-test") { - exclude(group = "org.junit.vintage", module = "junit-vintage-engine") // to ensure we are using mocks and spies from springmockk lib instead exclude(module = "mockito-core") } testImplementation("com.ninja-squad:springmockk:2.0.0") testImplementation("io.projectreactor:reactor-test") - testImplementation("org.springframework.cloud:spring-cloud-stream-test-support") testImplementation("org.springframework.security:spring-security-test") testImplementation("org.testcontainers:testcontainers") + testImplementation("org.testcontainers:junit-jupiter") testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test") } @@ -106,7 +104,6 @@ subprojects { } ktlint { - enableExperimentalRules.set(true) disabledRules.set(setOf("experimental:multiline-if-else", "no-wildcard-imports")) reporters { reporter(ReporterType.CHECKSTYLE) @@ -115,7 +112,7 @@ subprojects { } detekt { - toolVersion = "1.11.2" + toolVersion = "1.17.1" input = files("src/main/kotlin", "src/test/kotlin") config = files(detektConfigFile) buildUponDefaultConfig = true @@ -158,6 +155,5 @@ allprojects { repositories { mavenCentral() maven { url = uri("https://repo.spring.io/milestone") } - jcenter() } } diff --git a/entity-service/build.gradle.kts b/entity-service/build.gradle.kts index e3fc3073a..12bdb7a1f 100644 --- a/entity-service/build.gradle.kts +++ b/entity-service/build.gradle.kts @@ -13,13 +13,13 @@ plugins { dependencies { implementation("org.springframework.boot:spring-boot-starter-data-neo4j") - implementation("org.neo4j:neo4j-ogm-bolt-native-types") - implementation("eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter:0.0.13") + implementation("eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter:0.1.4") implementation(project(":shared")) developmentOnly("org.springframework.boot:spring-boot-devtools") testImplementation("org.hamcrest:hamcrest:2.2") + testImplementation("org.testcontainers:neo4j") testImplementation(testFixtures(project(":shared"))) } @@ -32,7 +32,8 @@ tasks.bootRun { jib.from.image = "adoptopenjdk/openjdk11:alpine-jre" jib.to.image = "stellio/stellio-entity-service" jib.container.entrypoint = listOf( - "/bin/sh", "-c", + "/bin/sh", + "-c", "/database/wait-for-neo4j.sh neo4j:7687 -t \$NEO4J_WAIT_TIMEOUT -- " + "java " + (project.ext["jibContainerJvmFlags"] as List).joinToString(" ") + diff --git a/entity-service/config/detekt/baseline.xml b/entity-service/config/detekt/baseline.xml index d043ca7be..0c03ac340 100644 --- a/entity-service/config/detekt/baseline.xml +++ b/entity-service/config/detekt/baseline.xml @@ -7,22 +7,19 @@ LargeClass:EntityHandlerTests.kt$EntityHandlerTests LargeClass:EntityOperationHandlerTests.kt$EntityOperationHandlerTests LargeClass:EntityServiceTests.kt$EntityServiceTests - LargeClass:Neo4jRepositoryTests.kt$Neo4jRepositoryTests - LargeClass:StandaloneNeo4jSearchRepositoryTests.kt$StandaloneNeo4jSearchRepositoryTests + LargeClass:Neo4jRepositoryTests.kt$Neo4jRepositoryTests : WithNeo4jContainer + LargeClass:StandaloneNeo4jSearchRepositoryTests.kt$StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer LongMethod:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap<String, String> ): ResponseEntity<*> LongMethod:EntityServiceTests.kt$EntityServiceTests$@Test fun `it should create a new multi attribute property`() LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, page: Int, limit: Int, contexts: List<String> ): String - LongParameterList:Attribute.kt$Attribute$( @Transient val attributeType: String, var observedAt: ZonedDateTime? = null, @JsonIgnore val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), @JsonIgnore var modifiedAt: ZonedDateTime? = null, @Convert(UriConverter::class) var datasetId: URI? = null, @Relationship(type = "HAS_VALUE") val properties: MutableList<Property> = mutableListOf(), @Relationship(type = "HAS_OBJECT") val relationships: MutableList<com.egm.stellio.entity.model.Relationship> = mutableListOf() ) - LongParameterList:Entity.kt$Entity$( @Id @JsonProperty("@id") @Convert(UriConverter::class) val id: URI, @Labels @JsonProperty("@type") val type: List<String>, @JsonIgnore val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), @JsonIgnore var modifiedAt: ZonedDateTime? = null, @JsonIgnore var location: String? = null, @Relationship(type = "HAS_VALUE") val properties: MutableList<Property> = mutableListOf(), @Relationship(type = "HAS_OBJECT") val relationships: MutableList<com.egm.stellio.entity.model.Relationship> = mutableListOf(), var contexts: List<String> = mutableListOf() ) LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, page: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean ) LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, page: Int, limit: Int, contexts: List<String>, includeSysAttrs: Boolean ) MaxLineLength:EntityHandlerTests.kt$EntityHandlerTests$ MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (a: MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (entity: - MaxLineLength:WebSecurityTestConfig.kt$WebSecurityTestConfig : WebSecurityConfig ReturnCount:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap<String, String> ): ResponseEntity<*> - TooGenericExceptionCaught:ParsingUtils.kt$e: Exception - TooManyFunctions:EntityService.kt$EntityService$EntityService - TooManyFunctions:Neo4jRepository.kt$Neo4jRepository$Neo4jRepository + TooManyFunctions:EntityService.kt$EntityService + TooManyFunctions:Neo4jRepository.kt$Neo4jRepository + UnusedPrivateMember:EntityServiceTests.kt$EntityServiceTests$@MockkBean private lateinit var searchRepository: SearchRepository diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt index 5f271bde5..d354ebfac 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepository.kt @@ -5,23 +5,19 @@ import com.egm.stellio.entity.authorization.AuthorizationService.Companion.EGM_R import com.egm.stellio.entity.authorization.AuthorizationService.Companion.R_CAN_ADMIN import com.egm.stellio.entity.authorization.AuthorizationService.Companion.SERVICE_ACCOUNT_ID import com.egm.stellio.entity.authorization.AuthorizationService.Companion.USER_LABEL -import com.egm.stellio.entity.model.Property import com.egm.stellio.entity.model.Relationship import com.egm.stellio.shared.util.JsonLdUtils.EGM_SPECIFIC_ACCESS_POLICY import com.egm.stellio.shared.util.toListOfString import com.egm.stellio.shared.util.toUri -import org.neo4j.ogm.session.Session -import org.slf4j.LoggerFactory +import org.springframework.data.neo4j.core.Neo4jClient import org.springframework.stereotype.Component import java.net.URI @Component class Neo4jAuthorizationRepository( - private val session: Session + private val neo4jClient: Neo4jClient ) { - private val logger = LoggerFactory.getLogger(javaClass) - fun filterEntitiesUserHasOneOfGivenRights( userId: URI, entitiesId: List, @@ -52,9 +48,9 @@ class Neo4jAuthorizationRepository( "rights" to rights ) - return session.query(query, parameters).map { - (it["id"] as String).toUri() - } + return neo4jClient.query(query).bindAll(parameters) + .fetch().all() + .map { (it["id"] as String).toUri() } } fun filterEntitiesWithSpecificAccessPolicy( @@ -75,9 +71,9 @@ class Neo4jAuthorizationRepository( "specificAccessPolicies" to specificAccessPolicies ) - return session.query(query, parameters).queryResults().map { - (it["id"] as String).toUri() - } + return neo4jClient.query(query).bindAll(parameters) + .fetch().all() + .map { (it["id"] as String).toUri() } } fun getUserRoles(userId: URI): Set { @@ -87,48 +83,33 @@ class Neo4jAuthorizationRepository( OPTIONAL MATCH (userEntity)-[:HAS_VALUE]->(p:Property { name:"$EGM_ROLES" }) OPTIONAL MATCH (userEntity)-[:HAS_OBJECT]-(r:Attribute:Relationship)- [:isMemberOf]->(group:Entity)-[:HAS_VALUE]->(pgroup:Property { name: "$EGM_ROLES" }) - RETURN p, pgroup + RETURN apoc.coll.union(collect(p.value), collect(pgroup.value)) as roles UNION MATCH (client:Entity)-[:HAS_VALUE]->(sid:Property { name: "$SERVICE_ACCOUNT_ID", value: ${'$'}userId }) OPTIONAL MATCH (client)-[:HAS_VALUE]->(p:Property { name:"$EGM_ROLES" }) OPTIONAL MATCH (client)-[:HAS_OBJECT]-(r:Attribute:Relationship)- [:isMemberOf]->(group:Entity)-[:HAS_VALUE]->(pgroup:Property { name: "$EGM_ROLES" }) - RETURN p, pgroup + RETURN apoc.coll.union(collect(p.value), collect(pgroup.value)) as roles """.trimIndent() val parameters = mapOf( "userId" to userId.toString() ) - val result = session.query(query, parameters) - - if (result.toSet().isEmpty()) { - return emptySet() - } + val result = neo4jClient.query(query).bindAll(parameters).fetch().all() return result .flatMap { - listOfNotNull( - propertyToListOfRoles(it["p"]), - propertyToListOfRoles(it["pgroup"]) - ).flatten() + val roles = it["roles"] as List<*> + when { + roles.isEmpty() -> emptyList() + roles[0] is String -> listOf(roles[0] as String) + else -> (roles[0] as List) + } } .toSet() } - private fun propertyToListOfRoles(property: Any?): List { - val rolesProperty = property as Property? ?: return emptyList() - - return when (rolesProperty.value) { - is String -> listOf(rolesProperty.value as String) - is List<*> -> rolesProperty.value as List - else -> { - logger.warn("Unknown value type for roles property: ${rolesProperty.value}") - emptyList() - } - } - } - fun createAdminLinks(userId: URI, relationships: List, entitiesId: List): List { val query = """ @@ -143,19 +124,24 @@ class Neo4jAuthorizationRepository( } WITH user UNWIND ${'$'}relPropsAndTargets AS relPropAndTarget - MATCH (target:Entity { id: relPropAndTarget.second }) + MATCH (target:Entity { id: relPropAndTarget.targetEntityId }) CREATE (user)-[:HAS_OBJECT]->(r:Attribute:Relationship:`$R_CAN_ADMIN`)-[:rCanAdmin]->(target) - SET r = relPropAndTarget.first + SET r = relPropAndTarget.props RETURN r.id as id """ val parameters = mapOf( - "relPropsAndTargets" to relationships.map { it.nodeProperties() }.zip(entitiesId.toListOfString()), + "relPropsAndTargets" to relationships + .map { it.nodeProperties() } + .zip(entitiesId.toListOfString()) + .map { + mapOf("props" to it.first, "targetEntityId" to it.second) + }, "userId" to userId.toString() ) - return session.query(query, parameters).map { - (it["id"] as String).toUri() - } + return neo4jClient.query(query).bindAll(parameters) + .fetch().all() + .map { (it["id"] as String).toUri() } } } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfig.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfig.kt new file mode 100644 index 000000000..3faffba0b --- /dev/null +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfig.kt @@ -0,0 +1,21 @@ +package com.egm.stellio.entity.config + +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.data.auditing.DateTimeProvider +import org.springframework.data.neo4j.config.EnableNeo4jAuditing +import java.time.Instant +import java.time.ZoneOffset +import java.util.Optional + +@Configuration +@EnableNeo4jAuditing( + dateTimeProviderRef = "fixedDateTimeProvider" +) +class Neo4jConfig { + + @Bean + fun fixedDateTimeProvider(): DateTimeProvider { + return DateTimeProvider { Optional.of(Instant.now().atZone(ZoneOffset.UTC)) } + } +} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfiguration.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfiguration.kt deleted file mode 100644 index d32841b97..000000000 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jConfiguration.kt +++ /dev/null @@ -1,38 +0,0 @@ -package com.egm.stellio.entity.config - -import org.neo4j.ogm.session.SessionFactory -import org.springframework.boot.autoconfigure.data.neo4j.Neo4jProperties -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.Profile -import org.springframework.data.neo4j.repository.config.EnableNeo4jRepositories -import org.springframework.data.neo4j.transaction.Neo4jTransactionManager -import org.springframework.transaction.annotation.EnableTransactionManagement - -@Profile("!test") -@Configuration -@EnableNeo4jRepositories(basePackages = ["com.egm.stellio.entity.repository"]) -@EnableTransactionManagement -class Neo4jConfiguration { - - @Bean - fun sessionFactory(properties: Neo4jProperties): SessionFactory { - return SessionFactory(ogmConfiguration(properties), "com.egm.stellio.entity.model") - } - - @Bean - fun ogmConfiguration(properties: Neo4jProperties): org.neo4j.ogm.config.Configuration { - return org.neo4j.ogm.config.Configuration.Builder() - .uri(properties.uri) - .credentials(properties.username, properties.password) - .useNativeTypes() - .database("stellio") - .verifyConnection(true) - .build() - } - - @Bean - fun transactionManager(properties: Neo4jProperties): Neo4jTransactionManager { - return Neo4jTransactionManager(sessionFactory(properties)) - } -} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jUriPropertyConverter.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jUriPropertyConverter.kt new file mode 100644 index 000000000..8a267e82b --- /dev/null +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jUriPropertyConverter.kt @@ -0,0 +1,18 @@ +package com.egm.stellio.entity.config + +import com.egm.stellio.shared.util.toUri +import org.neo4j.driver.Value +import org.neo4j.driver.internal.value.StringValue +import org.springframework.data.neo4j.core.convert.Neo4jPersistentPropertyConverter +import java.net.URI + +class Neo4jUriPropertyConverter : Neo4jPersistentPropertyConverter { + + override fun write(source: URI): Value { + return StringValue(source.toString()) + } + + override fun read(source: Value): URI { + return source.asString().toUri() + } +} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt new file mode 100644 index 000000000..125aea2ee --- /dev/null +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt @@ -0,0 +1,27 @@ +package com.egm.stellio.entity.config + +import org.neo4j.driver.Value +import org.neo4j.driver.Values +import org.neo4j.driver.internal.value.ListValue +import org.neo4j.driver.internal.value.StringValue +import org.springframework.data.neo4j.core.convert.Neo4jPersistentPropertyConverter +import java.net.URI + +class Neo4jValuePropertyConverter : Neo4jPersistentPropertyConverter { + + @Suppress("SpreadOperator") + override fun write(source: Any): Value { + return when (source) { + is List<*> -> ListValue(*source.map { Values.value(it) }.toTypedArray()) + is URI -> StringValue(source.toString()) + else -> Values.value(source) + } + } + + override fun read(source: Value): Any { + return when (source) { + is ListValue -> source.asList() + else -> Values.ofObject().apply(source) + } + } +} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Attribute.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Attribute.kt index d7c6886d2..2f064ce7e 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Attribute.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Attribute.kt @@ -1,107 +1,11 @@ package com.egm.stellio.entity.model -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_KW -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_PROPERTY -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_PROPERTY -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATE_TIME_TYPE -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY -import com.egm.stellio.shared.util.toNgsiLdFormat -import com.egm.stellio.shared.util.toUri -import com.fasterxml.jackson.annotation.JsonIgnore -import org.neo4j.ogm.annotation.Id -import org.neo4j.ogm.annotation.NodeEntity -import org.neo4j.ogm.annotation.Relationship -import org.neo4j.ogm.annotation.Transient -import org.neo4j.ogm.annotation.typeconversion.Convert +import org.springframework.data.neo4j.core.schema.Node import java.net.URI -import java.time.Instant -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.util.UUID -@NodeEntity -open class Attribute( - - @Transient - val attributeType: String, - - var observedAt: ZonedDateTime? = null, - - @JsonIgnore - val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), - - @JsonIgnore - var modifiedAt: ZonedDateTime? = null, - - @Convert(UriConverter::class) - var datasetId: URI? = null, - - @Relationship(type = "HAS_VALUE") - val properties: MutableList = mutableListOf(), - - @Relationship(type = "HAS_OBJECT") - val relationships: MutableList = mutableListOf() - -) { - @Id - @Convert(UriConverter::class) - val id: URI = "urn:ngsi-ld:$attributeType:${UUID.randomUUID()}".toUri() - - open fun serializeCoreProperties(includeSysAttrs: Boolean): MutableMap { - val resultEntity = mutableMapOf() - if (includeSysAttrs) { - resultEntity[NGSILD_CREATED_AT_PROPERTY] = mapOf( - JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, - JSONLD_VALUE_KW to createdAt.toNgsiLdFormat() - ) - - modifiedAt?.run { - resultEntity[NGSILD_MODIFIED_AT_PROPERTY] = mapOf( - JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, - JSONLD_VALUE_KW to this.toNgsiLdFormat() - ) - } - } - observedAt?.run { - resultEntity[NGSILD_OBSERVED_AT_PROPERTY] = mapOf( - JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, - JSONLD_VALUE_KW to this.toNgsiLdFormat() - ) - } - - datasetId?.run { - resultEntity[NGSILD_DATASET_ID_PROPERTY] = mapOf( - JSONLD_ID to this.toString() - ) - } - - return resultEntity - } - - /** - * Return a map of the properties to store with the Property node in neo4j - */ - open fun nodeProperties(): MutableMap { - val nodeProperties = mutableMapOf( - "id" to id, - "createdAt" to createdAt - ) - - modifiedAt?.run { - nodeProperties["modifiedAt"] = this - } - - observedAt?.run { - nodeProperties["observedAt"] = this - } - - datasetId?.run { - nodeProperties["datasetId"] = this - } - - return nodeProperties - } +@Node +interface Attribute { + fun serializeCoreProperties(includeSysAttrs: Boolean): MutableMap + fun nodeProperties(): MutableMap + fun id(): URI } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt index bdb1a328a..c7f12927f 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt @@ -1,5 +1,6 @@ package com.egm.stellio.entity.model +import com.egm.stellio.entity.config.Neo4jUriPropertyConverter import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_KW @@ -14,25 +15,26 @@ import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import org.locationtech.jts.io.WKTReader -import org.neo4j.ogm.annotation.Id -import org.neo4j.ogm.annotation.Labels -import org.neo4j.ogm.annotation.NodeEntity -import org.neo4j.ogm.annotation.Relationship -import org.neo4j.ogm.annotation.typeconversion.Convert +import org.springframework.data.annotation.LastModifiedDate +import org.springframework.data.neo4j.core.convert.ConvertWith +import org.springframework.data.neo4j.core.schema.DynamicLabels +import org.springframework.data.neo4j.core.schema.Id +import org.springframework.data.neo4j.core.schema.Node +import org.springframework.data.neo4j.core.schema.Relationship import java.net.URI import java.time.Instant import java.time.ZoneOffset import java.time.ZonedDateTime -@NodeEntity +@Node @JsonIgnoreProperties(ignoreUnknown = true) -class Entity( +data class Entity( @Id @JsonProperty("@id") - @Convert(UriConverter::class) + @ConvertWith(converter = Neo4jUriPropertyConverter::class) val id: URI, - @Labels + @DynamicLabels @JsonProperty("@type") val type: List, @@ -40,6 +42,7 @@ class Entity( val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), @JsonIgnore + @LastModifiedDate var modifiedAt: ZonedDateTime? = null, @JsonIgnore diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/PartialEntity.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/PartialEntity.kt index 02f44f738..45408f8cb 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/PartialEntity.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/PartialEntity.kt @@ -1,14 +1,15 @@ package com.egm.stellio.entity.model -import org.neo4j.ogm.annotation.Id -import org.neo4j.ogm.annotation.NodeEntity -import org.neo4j.ogm.annotation.typeconversion.Convert +import com.egm.stellio.entity.config.Neo4jUriPropertyConverter +import org.springframework.data.neo4j.core.convert.ConvertWith +import org.springframework.data.neo4j.core.schema.Id +import org.springframework.data.neo4j.core.schema.Node import java.net.URI -@NodeEntity -class PartialEntity( +@Node +data class PartialEntity( @Id - @Convert(UriConverter::class) + @ConvertWith(converter = Neo4jUriPropertyConverter::class) val id: URI ) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt index 58b87feba..04164ed05 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt @@ -1,5 +1,7 @@ package com.egm.stellio.entity.model +import com.egm.stellio.entity.config.Neo4jUriPropertyConverter +import com.egm.stellio.entity.config.Neo4jValuePropertyConverter import com.egm.stellio.shared.model.NgsiLdPropertyInstance import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE @@ -12,28 +14,55 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_UNIT_CODE_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMap import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsDateTime import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsString +import com.egm.stellio.shared.util.toNgsiLdFormat +import com.egm.stellio.shared.util.toUri +import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonRawValue -import org.neo4j.ogm.annotation.Index -import org.neo4j.ogm.annotation.NodeEntity +import org.springframework.data.annotation.LastModifiedDate +import org.springframework.data.neo4j.core.convert.ConvertWith +import org.springframework.data.neo4j.core.schema.Id +import org.springframework.data.neo4j.core.schema.Node +import org.springframework.data.neo4j.core.schema.Relationship import java.net.URI +import java.time.Instant import java.time.LocalDate import java.time.LocalTime +import java.time.ZoneOffset import java.time.ZonedDateTime +import java.util.UUID -@NodeEntity -class Property( - @Index - val name: String, - var unitCode: String? = null, +@Node +data class Property( + @Id + @ConvertWith(converter = Neo4jUriPropertyConverter::class) + val id: URI = "urn:ngsi-ld:Property:${UUID.randomUUID()}".toUri(), - @JsonRawValue - var value: Any, + var observedAt: ZonedDateTime? = null, + + @JsonIgnore + val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), - observedAt: ZonedDateTime? = null, + @JsonIgnore + @LastModifiedDate + val modifiedAt: ZonedDateTime? = null, - datasetId: URI? = null + @ConvertWith(converter = Neo4jUriPropertyConverter::class) + val datasetId: URI? = null, -) : Attribute(attributeType = "Property", observedAt = observedAt, datasetId = datasetId) { + @Relationship(type = "HAS_VALUE") + val properties: MutableList = mutableListOf(), + + @Relationship(type = "HAS_OBJECT") + val relationships: MutableList = mutableListOf(), + + val name: String, + + var unitCode: String? = null, + + @JsonRawValue + @ConvertWith(converter = Neo4jValuePropertyConverter::class) + var value: Any +) : Attribute { constructor(name: String, ngsiLdPropertyInstance: NgsiLdPropertyInstance) : this( @@ -45,7 +74,32 @@ class Property( ) override fun serializeCoreProperties(includeSysAttrs: Boolean): MutableMap { - val resultEntity = super.serializeCoreProperties(includeSysAttrs) + val resultEntity = mutableMapOf() + if (includeSysAttrs) { + resultEntity[JsonLdUtils.NGSILD_CREATED_AT_PROPERTY] = mapOf( + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, + JSONLD_VALUE_KW to createdAt.toNgsiLdFormat() + ) + + modifiedAt?.run { + resultEntity[JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY] = mapOf( + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, + JSONLD_VALUE_KW to this.toNgsiLdFormat() + ) + } + } + observedAt?.run { + resultEntity[JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY] = mapOf( + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, + JSONLD_VALUE_KW to this.toNgsiLdFormat() + ) + } + + datasetId?.run { + resultEntity[JsonLdUtils.NGSILD_DATASET_ID_PROPERTY] = mapOf( + JsonLdUtils.JSONLD_ID to this.toString() + ) + } resultEntity[JSONLD_TYPE] = JsonLdUtils.NGSILD_PROPERTY_TYPE.uri resultEntity[NGSILD_PROPERTY_VALUE] = when (value) { is ZonedDateTime -> mapOf(JSONLD_TYPE to NGSILD_DATE_TIME_TYPE, JSONLD_VALUE_KW to value) @@ -61,18 +115,36 @@ class Property( } override fun nodeProperties(): MutableMap { - val propsMap = super.nodeProperties() - propsMap["name"] = name - propsMap["value"] = value + val nodeProperties = mutableMapOf( + "id" to id.toString(), + "createdAt" to createdAt + ) + + modifiedAt?.run { + nodeProperties["modifiedAt"] = this + } + + observedAt?.run { + nodeProperties["observedAt"] = this + } + + datasetId?.run { + nodeProperties["datasetId"] = this.toString() + } + + nodeProperties["name"] = name + nodeProperties["value"] = value unitCode?.run { - propsMap["unitCode"] = this + nodeProperties["unitCode"] = this } - return propsMap + return nodeProperties } - fun updateValues(unitCode: String?, value: Any?, observedAt: ZonedDateTime?): Property { + override fun id(): URI = id + + private fun updateValues(unitCode: String?, value: Any?, observedAt: ZonedDateTime?): Property { unitCode?.let { this.unitCode = unitCode } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt index 322c65a01..99c5bda77 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt @@ -1,27 +1,55 @@ package com.egm.stellio.entity.model +import com.egm.stellio.entity.config.Neo4jUriPropertyConverter import com.egm.stellio.shared.model.NgsiLdRelationshipInstance import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsDateTime import com.egm.stellio.shared.util.extractShortTypeFromExpanded +import com.egm.stellio.shared.util.toNgsiLdFormat +import com.egm.stellio.shared.util.toUri +import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonProperty -import org.neo4j.ogm.annotation.Labels -import org.neo4j.ogm.annotation.NodeEntity +import org.springframework.data.annotation.LastModifiedDate +import org.springframework.data.neo4j.core.convert.ConvertWith +import org.springframework.data.neo4j.core.schema.DynamicLabels +import org.springframework.data.neo4j.core.schema.Id +import org.springframework.data.neo4j.core.schema.Node import java.net.URI +import java.time.Instant +import java.time.ZoneOffset import java.time.ZonedDateTime +import java.util.UUID -@NodeEntity -class Relationship( +@Node +data class Relationship( - @Labels - @JsonProperty("@type") - val type: List, + @Id + @ConvertWith(converter = Neo4jUriPropertyConverter::class) + val id: URI = "urn:ngsi-ld:Relationship:${UUID.randomUUID()}".toUri(), + + var observedAt: ZonedDateTime? = null, + + @JsonIgnore + val createdAt: ZonedDateTime = Instant.now().atZone(ZoneOffset.UTC), - observedAt: ZonedDateTime? = null, + @JsonIgnore + @LastModifiedDate + val modifiedAt: ZonedDateTime? = null, - datasetId: URI? = null + @ConvertWith(converter = Neo4jUriPropertyConverter::class) + val datasetId: URI? = null, -) : Attribute(attributeType = "Relationship", observedAt = observedAt, datasetId = datasetId) { + @org.springframework.data.neo4j.core.schema.Relationship(type = "HAS_VALUE") + val properties: MutableList = mutableListOf(), + + @org.springframework.data.neo4j.core.schema.Relationship(type = "HAS_OBJECT") + val relationships: MutableList = mutableListOf(), + + @DynamicLabels + @JsonProperty("@type") + val type: List, + +) : Attribute { constructor(type: String, ngsiLdRelationshipInstance: NgsiLdRelationshipInstance) : this( @@ -30,7 +58,63 @@ class Relationship( datasetId = ngsiLdRelationshipInstance.datasetId ) - fun updateValues(observedAt: ZonedDateTime?): Relationship { + override fun serializeCoreProperties(includeSysAttrs: Boolean): MutableMap { + val resultEntity = mutableMapOf() + if (includeSysAttrs) { + resultEntity[JsonLdUtils.NGSILD_CREATED_AT_PROPERTY] = mapOf( + JsonLdUtils.JSONLD_TYPE to JsonLdUtils.NGSILD_DATE_TIME_TYPE, + JsonLdUtils.JSONLD_VALUE_KW to createdAt.toNgsiLdFormat() + ) + + modifiedAt?.run { + resultEntity[JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY] = mapOf( + JsonLdUtils.JSONLD_TYPE to JsonLdUtils.NGSILD_DATE_TIME_TYPE, + JsonLdUtils.JSONLD_VALUE_KW to this.toNgsiLdFormat() + ) + } + } + observedAt?.run { + resultEntity[JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY] = mapOf( + JsonLdUtils.JSONLD_TYPE to JsonLdUtils.NGSILD_DATE_TIME_TYPE, + JsonLdUtils.JSONLD_VALUE_KW to this.toNgsiLdFormat() + ) + } + + datasetId?.run { + resultEntity[JsonLdUtils.NGSILD_DATASET_ID_PROPERTY] = mapOf( + JsonLdUtils.JSONLD_ID to this.toString() + ) + } + + resultEntity[JsonLdUtils.JSONLD_TYPE] = JsonLdUtils.NGSILD_RELATIONSHIP_TYPE.uri + + return resultEntity + } + + override fun nodeProperties(): MutableMap { + val nodeProperties = mutableMapOf( + "id" to id.toString(), + "createdAt" to createdAt + ) + + modifiedAt?.run { + nodeProperties["modifiedAt"] = this + } + + observedAt?.run { + nodeProperties["observedAt"] = this + } + + datasetId?.run { + nodeProperties["datasetId"] = this.toString() + } + + return nodeProperties + } + + override fun id(): URI = id + + private fun updateValues(observedAt: ZonedDateTime?): Relationship { observedAt?.let { this.observedAt = observedAt } @@ -39,6 +123,10 @@ class Relationship( fun updateValues(updateFragment: Map>): Relationship = updateValues(getPropertyValueFromMapAsDateTime(updateFragment, JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY)) + + // neo4j forces us to have a list but we know we have only one dynamic label + fun relationshipType(): String = + type.first() } fun String.toRelationshipTypeName(): String = diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/UriConverter.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/UriConverter.kt deleted file mode 100644 index 2159955cc..000000000 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/UriConverter.kt +++ /dev/null @@ -1,18 +0,0 @@ -package com.egm.stellio.entity.model - -import com.egm.stellio.shared.util.toUri -import org.neo4j.ogm.typeconversion.AttributeConverter -import java.net.URI - -class UriConverter : AttributeConverter { - - override fun toGraphProperty(value: URI?): String? { - return value?.toString() - } - - override fun toEntityAttribute(value: String?): URI? { - return value?.let { - value.toUri() - } - } -} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt index 04b1da812..74558ccb5 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt @@ -1,8 +1,8 @@ package com.egm.stellio.entity.repository import com.egm.stellio.entity.model.Entity -import org.springframework.data.neo4j.annotation.Query import org.springframework.data.neo4j.repository.Neo4jRepository +import org.springframework.data.neo4j.repository.query.Query import org.springframework.stereotype.Repository import java.net.URI @@ -32,4 +32,18 @@ interface EntityRepository : Neo4jRepository { " relOfRel, type(or) as relOfRelType, relOfRelObject.id as relOfRelObjectId" ) fun getEntityRelationships(id: String): List> + + @Query( + "MATCH ({ id: \$subjectId })-[:HAS_OBJECT]->(r:Relationship { datasetId: \$datasetId }) " + + "-[:`:#{literal(#relationshipType)}`]->(e:Entity)" + + " RETURN e" + ) + fun getRelationshipTargetOfSubject(subjectId: URI, relationshipType: String, datasetId: URI): Entity? + + @Query( + "MATCH ({ id: \$subjectId })-[:HAS_OBJECT]->(r:Relationship) " + + "-[:`:#{literal(#relationshipType)}`]->(e:Entity)" + + " RETURN e" + ) + fun getRelationshipTargetOfSubject(subjectId: URI, relationshipType: String): Entity? } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt index b7ccbfccc..d5384f9f6 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt @@ -1,7 +1,6 @@ package com.egm.stellio.entity.repository import com.egm.stellio.entity.authorization.AuthorizationService -import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.model.Property import com.egm.stellio.entity.model.Relationship import com.egm.stellio.entity.model.toRelationshipTypeName @@ -11,15 +10,12 @@ import com.egm.stellio.shared.model.NgsiLdPropertyInstance import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LOCATION_PROPERTY import com.egm.stellio.shared.util.toListOfString import com.egm.stellio.shared.util.toUri -import org.neo4j.ogm.session.Session -import org.neo4j.ogm.session.SessionFactory -import org.neo4j.ogm.session.event.Event -import org.neo4j.ogm.session.event.EventListenerAdapter +import org.springframework.data.neo4j.core.Neo4jClient +import org.springframework.data.neo4j.core.mappedBy import org.springframework.stereotype.Component import java.net.URI import java.time.Instant import java.time.ZoneOffset -import javax.annotation.PostConstruct sealed class SubjectNodeInfo(val id: URI, val label: String) class EntitySubjectNode(id: URI) : SubjectNodeInfo(id, "Entity") @@ -27,14 +23,15 @@ class AttributeSubjectNode(id: URI) : SubjectNodeInfo(id, "Attribute") @Component class Neo4jRepository( - private val session: Session, - private val sessionFactory: SessionFactory + private val neo4jClient: Neo4jClient ) { fun mergePartialWithNormalEntity(id: URI): Int { val query = """ - MATCH (e:Entity { id: ${'$'}id }), (pe:PartialEntity { id: ${'$'}id }) + MATCH (pe:PartialEntity { id: ${'$'}id }) + WITH pe + MATCH (e:Entity { id: ${'$'}id }) WITH head(collect([e,pe])) as nodes CALL apoc.refactor.mergeNodes(nodes, { properties:"discard" }) YIELD node @@ -42,11 +39,7 @@ class Neo4jRepository( RETURN node """ - val parameters = mapOf( - "id" to id - ) - - return session.query(query, parameters).queryStatistics().nodesDeleted + return neo4jClient.query(query).bind(id.toString()).to("id").run().counters().nodesDeleted() } fun createPropertyOfSubject(subjectNodeInfo: SubjectNodeInfo, property: Property): Boolean { @@ -59,9 +52,9 @@ class Neo4jRepository( val parameters = mapOf( "props" to property.nodeProperties(), - "subjectId" to subjectNodeInfo.id + "subjectId" to subjectNodeInfo.id.toString() ) - return session.query(query, parameters).queryStatistics().containsUpdates() + return neo4jClient.query(query).bindAll(parameters).run().counters().containsUpdates() } fun createRelationshipOfSubject( @@ -82,11 +75,11 @@ class Neo4jRepository( val parameters = mapOf( "props" to relationship.nodeProperties(), - "subjectId" to subjectNodeInfo.id, - "targetId" to targetId + "subjectId" to subjectNodeInfo.id.toString(), + "targetId" to targetId.toString() ) - return session.query(query, parameters).queryStatistics().containsUpdates() + return neo4jClient.query(query).bindAll(parameters).run().counters().containsUpdates() } /** @@ -99,10 +92,9 @@ class Neo4jRepository( ON MATCH SET subject.location = "${toWktFormat(geoProperty.geoPropertyType, geoProperty.coordinates)}" """ - val parameters = mapOf( - "subjectId" to subjectId - ) - return session.query(query, parameters).queryStatistics().propertiesSet + return neo4jClient.query(query) + .bind(subjectId.toString()).to("subjectId") + .run().counters().propertiesSet() } fun updateEntityPropertyInstance( @@ -121,13 +113,13 @@ class Neo4jRepository( val matchQuery = if (datasetId == null) """ MATCH (entity:${subjectNodeInfo.label} { id: ${'$'}entityId })-[:HAS_VALUE] - ->(attribute:Property { name: ${'$'}propertyName }) - WHERE NOT EXISTS (attribute.datasetId) - """.trimIndent() + ->(attribute:Property { name: ${'$'}propertyName }) + WHERE attribute.datasetId IS NULL + """ else """ MATCH (entity:${subjectNodeInfo.label} { id: ${'$'}entityId })-[:HAS_VALUE] - ->(attribute:Property { name: ${'$'}propertyName, datasetId: ${'$'}datasetId}) + ->(attribute:Property { name: ${'$'}propertyName, datasetId: ${'$'}datasetId}) """.trimIndent() var createAttributeQuery = @@ -137,7 +129,7 @@ class Neo4jRepository( """ val parameters = mutableMapOf( - "entityId" to subjectNodeInfo.id, + "entityId" to subjectNodeInfo.id.toString(), "propertyName" to propertyName, "datasetId" to datasetId?.toString(), "props" to Property(propertyName, newPropertyInstance).nodeProperties(), @@ -164,14 +156,15 @@ class Neo4jRepository( MERGE (target { id: "${ngsiLdRelationship.instances[0].objectId}" }) ON CREATE SET target:PartialEntity CREATE (newAttribute)-[:HAS_OBJECT] - ->(r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}relationshipOfProperty_$index) - -[:${relationship.type[0].toRelationshipTypeName()}]->(target) + ->(r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}relationshipOfProperty_$index) + -[:${relationship.type[0].toRelationshipTypeName()}]->(target) """ ) } - return session.query(matchQuery + deleteAttributeQuery + createAttributeQuery, parameters) - .queryStatistics().nodesDeleted + return neo4jClient.query(matchQuery + deleteAttributeQuery + createAttributeQuery) + .bindAll(parameters) + .run().counters().nodesDeleted() } fun updateEntityModifiedDate(entityId: URI): Int { @@ -181,7 +174,7 @@ class Neo4jRepository( ON MATCH SET entity.modifiedAt = datetime("${Instant.now().atZone(ZoneOffset.UTC)}") """ - return session.query(query, mapOf("entityId" to entityId)).queryStatistics().propertiesSet + return neo4jClient.query(query).bind(entityId.toString()).to("entityId").run().counters().propertiesSet() } fun hasRelationshipOfType(subjectNodeInfo: SubjectNodeInfo, relationshipType: String): Boolean { @@ -191,10 +184,9 @@ class Neo4jRepository( RETURN a.id """.trimIndent() - val parameters = mapOf( - "attributeId" to subjectNodeInfo.id - ) - return session.query(query, parameters, true).toList().isNotEmpty() + return neo4jClient.query(query) + .bind(subjectNodeInfo.id.toString()).to("attributeId") + .fetch().first().isPresent } fun hasPropertyOfName(subjectNodeInfo: SubjectNodeInfo, propertyName: String): Boolean { @@ -205,10 +197,10 @@ class Neo4jRepository( """.trimIndent() val parameters = mapOf( - "attributeId" to subjectNodeInfo.id, + "attributeId" to subjectNodeInfo.id.toString(), "propertyName" to propertyName ) - return session.query(query, parameters, true).toList().isNotEmpty() + return neo4jClient.query(query).bindAll(parameters).fetch().first().isPresent } fun hasGeoPropertyOfName(subjectNodeInfo: SubjectNodeInfo, geoPropertyName: String): Boolean { @@ -218,17 +210,16 @@ class Neo4jRepository( RETURN a.id """.trimIndent() - val parameters = mapOf( - "attributeId" to subjectNodeInfo.id - ) - return session.query(query, parameters, true).toList().isNotEmpty() + return neo4jClient.query(query) + .bind(subjectNodeInfo.id.toString()).to("attributeId") + .fetch().first().isPresent } fun hasPropertyInstance(subjectNodeInfo: SubjectNodeInfo, propertyName: String, datasetId: URI? = null): Boolean { val query = if (datasetId == null) """ MATCH (a:${subjectNodeInfo.label} { id: ${'$'}attributeId })-[:HAS_VALUE]->(property:Property { name: ${'$'}propertyName }) - WHERE NOT EXISTS (property.datasetId) + WHERE property.datasetId IS NULL RETURN a.id """.trimIndent() else @@ -238,12 +229,12 @@ class Neo4jRepository( """.trimIndent() val parameters = mapOf( - "attributeId" to subjectNodeInfo.id, + "attributeId" to subjectNodeInfo.id.toString(), "propertyName" to propertyName, "datasetId" to datasetId?.toString() ) - return session.query(query, parameters, true).toList().isNotEmpty() + return neo4jClient.query(query).bindAll(parameters).fetch().first().isPresent } fun hasRelationshipInstance( @@ -264,10 +255,10 @@ class Neo4jRepository( """.trimIndent() val parameters = mapOf( - "attributeId" to subjectNodeInfo.id, + "attributeId" to subjectNodeInfo.id.toString(), "datasetId" to datasetId?.toString() ) - return session.query(query, parameters, true).toList().isNotEmpty() + return neo4jClient.query(query).bindAll(parameters).fetch().first().isPresent } fun updateRelationshipTargetOfSubject( @@ -294,12 +285,12 @@ class Neo4jRepository( """.trimIndent() val parameters = mapOf( - "subjectId" to subjectId, - "newRelationshipObjectId" to newRelationshipObjectId, + "subjectId" to subjectId.toString(), + "newRelationshipObjectId" to newRelationshipObjectId.toString(), "datasetId" to datasetId?.toString() ) - return session.query(relationshipTypeQuery, parameters).queryStatistics().containsUpdates() + return neo4jClient.query(relationshipTypeQuery).bindAll(parameters).run().counters().containsUpdates() } fun updateTargetOfRelationship( @@ -319,11 +310,11 @@ class Neo4jRepository( """.trimIndent() val parameters = mapOf( - "attributeId" to attributeId, - "oldRelationshipObjectId" to oldRelationshipObjectId, - "newRelationshipObjectId" to newRelationshipObjectId + "attributeId" to attributeId.toString(), + "oldRelationshipObjectId" to oldRelationshipObjectId.toString(), + "newRelationshipObjectId" to newRelationshipObjectId.toString() ) - return session.query(relationshipTypeQuery, parameters).queryStatistics().nodesDeleted + return neo4jClient.query(relationshipTypeQuery).bindAll(parameters).run().counters().nodesDeleted() } fun updateLocationPropertyOfEntity(entityId: URI, geoProperty: NgsiLdGeoPropertyInstance): Int { @@ -332,7 +323,7 @@ class Neo4jRepository( MERGE (entity:Entity { id: "$entityId" }) ON MATCH SET entity.location = "${toWktFormat(geoProperty.geoPropertyType, geoProperty.coordinates)}" """ - return session.query(query, emptyMap()).queryStatistics().propertiesSet + return neo4jClient.query(query).run().counters().propertiesSet() } fun deleteEntity(entityId: URI): Pair { @@ -367,11 +358,8 @@ class Neo4jRepository( DETACH DELETE n, prop, relOfProp, propOfProp, rel, propOfRel, relOfRel, inRel """.trimIndent() - val parameters = mapOf( - "entityId" to entityId - ) - val queryStatistics = session.query(query, parameters).queryStatistics() - return Pair(queryStatistics.nodesDeleted, queryStatistics.relationshipsDeleted) + val queryStatistics = neo4jClient.query(query).bind(entityId.toString()).to("entityId").run().counters() + return Pair(queryStatistics.nodesDeleted(), queryStatistics.relationshipsDeleted()) } fun deleteEntityAttributes(entityId: URI): Pair { @@ -402,11 +390,8 @@ class Neo4jRepository( DETACH DELETE prop, relOfProp, propOfProp, rel, propOfRel, relOfRel """.trimIndent() - val parameters = mapOf( - "entityId" to entityId - ) - val queryStatistics = session.query(query, parameters).queryStatistics() - return Pair(queryStatistics.nodesDeleted, queryStatistics.relationshipsDeleted) + val queryStatistics = neo4jClient.query(query).bind(entityId.toString()).to("entityId").run().counters() + return Pair(queryStatistics.nodesDeleted(), queryStatistics.relationshipsDeleted()) } fun deleteEntityProperty( @@ -429,20 +414,22 @@ class Neo4jRepository( else if (datasetId == null) """ MATCH (entity:${subjectNodeInfo.label} { id: ${'$'}entityId })-[:HAS_VALUE]->(attribute:Property { name: ${'$'}propertyName }) - WHERE NOT EXISTS (attribute.datasetId) - """.trimIndent() + WHERE attribute.datasetId IS NULL + """ else """ MATCH (entity:${subjectNodeInfo.label} { id: ${'$'}entityId })-[:HAS_VALUE]->(attribute:Property { name: ${'$'}propertyName, datasetId: ${'$'}datasetId}) """.trimIndent() val parameters = mapOf( - "entityId" to subjectNodeInfo.id, + "entityId" to subjectNodeInfo.id.toString(), "propertyName" to propertyName, "datasetId" to datasetId?.toString() ) - return session.query(matchQuery + deleteAttributeQuery, parameters).queryStatistics().nodesDeleted + return neo4jClient.query(matchQuery + deleteAttributeQuery) + .bindAll(parameters) + .run().counters().nodesDeleted() } fun deleteEntityRelationship( @@ -465,7 +452,7 @@ class Neo4jRepository( else if (datasetId == null) """ MATCH (entity:${subjectNodeInfo.label} { id: ${'$'}entityId })-[:HAS_OBJECT]->(attribute:Relationship)-[:$relationshipType]->() - WHERE NOT EXISTS (attribute.datasetId) + WHERE attribute.datasetId IS NULL """.trimIndent() else """ @@ -473,11 +460,11 @@ class Neo4jRepository( """.trimIndent() val parameters = mapOf( - "entityId" to subjectNodeInfo.id, + "entityId" to subjectNodeInfo.id.toString(), "datasetId" to datasetId?.toString() ) - return session.query(matchQuery + deleteAttributeQuery, parameters).queryStatistics().nodesDeleted + return neo4jClient.query(matchQuery + deleteAttributeQuery).bindAll(parameters).run().counters().nodesDeleted() } fun getEntityTypeAttributesInformation(expandedType: String): Map { @@ -496,15 +483,15 @@ class Neo4jRepository( reduce(output = [], r IN collect(distinct labels(rel)) | output + r) as relationshipNames """.trimIndent() - val result = session.query(query, emptyMap(), true).toList() + val result = neo4jClient.query(query).fetch().all() if (result.isEmpty()) return emptyMap() val entityCount = (result.first()["entityCount"] as Long).toInt() val entityWithLocationCount = (result.first()["entityWithLocationCount"] as Long).toInt() return mapOf( - "properties" to (result.first()["propertyNames"] as Array).toSet(), - "relationships" to (result.first()["relationshipNames"] as Array) + "properties" to (result.first()["propertyNames"] as List).toSet(), + "relationships" to (result.first()["relationshipNames"] as List) .filter { it !in listOf("Attribute", "Relationship") }.toSet(), "geoProperties" to if (entityWithLocationCount > 0) setOf(NGSILD_LOCATION_PROPERTY) else emptySet(), @@ -525,16 +512,16 @@ class Neo4jRepository( count(entity.location) as entityWithLocationCount """.trimIndent() - val result = session.query(query, emptyMap(), true).toList() + val result = neo4jClient.query(query).fetch().all() return result.map { rowResult -> val entityWithLocationCount = (rowResult["entityWithLocationCount"] as Long).toInt() - val entityTypes = (rowResult["entityType"] as Array) + val entityTypes = (rowResult["entityType"] as List) .filter { !authorizationEntitiesTypes.plus("Entity").contains(it) } entityTypes.map { entityType -> mapOf( "entityType" to entityType, - "properties" to (rowResult["propertyNames"] as Array).toSet(), - "relationships" to (rowResult["relationshipNames"] as Array) + "properties" to (rowResult["propertyNames"] as List).toSet(), + "relationships" to (rowResult["relationshipNames"] as List) .filter { it !in listOf("Attribute", "Relationship") }.toSet(), "geoProperties" to if (entityWithLocationCount > 0) setOf(NGSILD_LOCATION_PROPERTY) else emptySet() @@ -550,9 +537,9 @@ class Neo4jRepository( RETURN DISTINCT(labels(entity)) as entityType """.trimIndent() - val result = session.query(query, emptyMap(), true).toList() + val result = neo4jClient.query(query).fetch().all() return result.map { - (it["entityType"] as Array) + (it["entityType"] as List) .filter { !authorizationEntitiesTypes.plus("Entity").contains(it) } }.flatten() } @@ -564,62 +551,10 @@ class Neo4jRepository( val query = "MATCH (entity:Entity) WHERE entity.id IN \$entitiesIds RETURN entity.id as id" - return session.query(query, mapOf("entitiesIds" to entitiesIds.toListOfString()), true) - .map { (it["id"] as String).toUri() } - } - - fun getPropertyOfSubject(subjectId: URI, propertyName: String, datasetId: URI? = null): Property { - val query = if (datasetId == null) - """ - MATCH ({ id: '$subjectId' })-[:HAS_VALUE]->(p:Property { name: "$propertyName" }) - WHERE NOT EXISTS (p.datasetId) - RETURN p - """.trimIndent() - else - """ - MATCH ({ id: '$subjectId' })-[:HAS_VALUE]->(p:Property { name: "$propertyName", datasetId: "$datasetId" }) - RETURN p - """.trimIndent() - - return session.query(query, emptyMap(), true).toMutableList() - .map { it["p"] as Property } - .first() - } - - fun getRelationshipOfSubject(subjectId: URI, relationshipType: String, datasetId: URI? = null): Relationship { - val query = if (datasetId == null) - """ - MATCH ({ id: '$subjectId' })-[:HAS_OBJECT]->(r:Relationship)-[:$relationshipType]->() - RETURN r - """.trimIndent() - else - """ - MATCH ({ id: '$subjectId' })-[:HAS_OBJECT]->(r:Relationship { datasetId: "$datasetId" }) - -[:$relationshipType]->() - RETURN r - """.trimIndent() - - return session.query(query, emptyMap(), true).toMutableList() - .map { it["r"] as Relationship } - .first() - } - - fun getRelationshipTargetOfSubject(subjectId: URI, relationshipType: String, datasetId: URI? = null): Entity? { - val query = if (datasetId == null) - """ - MATCH ({ id: '$subjectId' })-[:HAS_OBJECT]->(r:Relationship)-[:$relationshipType]->(e: Entity) - RETURN e - """.trimIndent() - else - """ - MATCH ({ id: '$subjectId' })-[:HAS_OBJECT]->(r:Relationship { datasetId: "$datasetId" }) - -[:$relationshipType]->(e: Entity) - RETURN e - """.trimIndent() - - return session.query(query, emptyMap(), true).toMutableList() - .map { it["e"] as Entity } - .firstOrNull() + return neo4jClient.query(query).bind(entitiesIds.toListOfString()).to("entitiesIds") + .mappedBy { _, record -> (record["id"].asString()).toUri() } + .all() + .toList() } private val deleteAttributeQuery = @@ -635,29 +570,4 @@ class Neo4jRepository( AuthorizationService.AUTHORIZATION_ONTOLOGY + "Client", AuthorizationService.AUTHORIZATION_ONTOLOGY + "Group" ) - - @PostConstruct - fun addEventListenerToSessionFactory() { - val eventListener = PreSaveEventListener() - sessionFactory.register(eventListener) - } -} - -class PreSaveEventListener : EventListenerAdapter() { - override fun onPreSave(event: Event) { - when (event.getObject()) { - is Entity -> { - val entity = event.getObject() as Entity - entity.modifiedAt = Instant.now().atZone(ZoneOffset.UTC) - } - is Property -> { - val property = event.getObject() as Property - property.modifiedAt = Instant.now().atZone(ZoneOffset.UTC) - } - is Relationship -> { - val relationship = event.getObject() as Relationship - relationship.modifiedAt = Instant.now().atZone(ZoneOffset.UTC) - } - } - } } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt index b8daee24e..46535e672 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt @@ -4,16 +4,16 @@ import com.egm.stellio.entity.authorization.AuthorizationService.Companion.USER_ import com.egm.stellio.entity.authorization.Neo4jAuthorizationService import com.egm.stellio.shared.model.QueryParams import com.egm.stellio.shared.util.toUri -import org.neo4j.ogm.session.Session import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.data.neo4j.core.Neo4jClient import org.springframework.stereotype.Component import java.net.URI @Component @ConditionalOnProperty("application.authentication.enabled") class Neo4jSearchRepository( - private val session: Session, - private val neo4jAuthorizationService: Neo4jAuthorizationService + private val neo4jAuthorizationService: Neo4jAuthorizationService, + private val neo4jClient: Neo4jClient ) : SearchRepository { override fun getEntities( @@ -28,7 +28,7 @@ class Neo4jSearchRepository( else QueryUtils.prepareQueryForEntitiesWithAuthentication(queryParams, page, limit, contexts) - val result = session.query(query, mapOf("userId" to USER_PREFIX + userSub), true) + val result = neo4jClient.query(query).bind(USER_PREFIX + userSub).to("userId").fetch().all() return if (limit == 0) Pair( (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0, diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/PropertyRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/PropertyRepository.kt index b9681de6c..ba841fbf7 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/PropertyRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/PropertyRepository.kt @@ -2,8 +2,23 @@ package com.egm.stellio.entity.repository import com.egm.stellio.entity.model.Property import org.springframework.data.neo4j.repository.Neo4jRepository +import org.springframework.data.neo4j.repository.query.Query import org.springframework.stereotype.Repository import java.net.URI @Repository -interface PropertyRepository : Neo4jRepository +interface PropertyRepository : Neo4jRepository { + + @Query( + "MATCH ({ id: \$subjectId })-[:HAS_VALUE]->(p:Property { name: \$propertyName }) " + + "WHERE p.datasetId IS NULL " + + "RETURN p" + ) + fun getPropertyOfSubject(subjectId: URI, propertyName: String): Property + + @Query( + "MATCH ({ id: \$subjectId })-[:HAS_VALUE]->(p:Property { name: \$propertyName, datasetId: \$datasetId })" + + "RETURN p" + ) + fun getPropertyOfSubject(subjectId: URI, propertyName: String, datasetId: URI): Property +} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/RelationshipRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/RelationshipRepository.kt index 43ac6deb2..dc8b9124c 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/RelationshipRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/RelationshipRepository.kt @@ -2,8 +2,24 @@ package com.egm.stellio.entity.repository import com.egm.stellio.entity.model.Relationship import org.springframework.data.neo4j.repository.Neo4jRepository +import org.springframework.data.neo4j.repository.query.Query import org.springframework.stereotype.Repository import java.net.URI @Repository -interface RelationshipRepository : Neo4jRepository +interface RelationshipRepository : Neo4jRepository { + + @Query( + "MATCH ({ id: \$subjectId })-[:HAS_OBJECT]->(r:Relationship)-[:`:#{literal(#relationshipType)}`]->() " + + "WHERE r.datasetId IS NULL " + + "RETURN r" + ) + fun getRelationshipOfSubject(subjectId: URI, relationshipType: String): Relationship + + @Query( + "MATCH ({ id: \$subjectId })-[:HAS_OBJECT]->(r:Relationship { datasetId: \$datasetId })" + + " -[:`:#{literal(#relationshipType)}`]->()" + + "RETURN r" + ) + fun getRelationshipOfSubject(subjectId: URI, relationshipType: String, datasetId: URI): Relationship +} diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt index 23d0627be..5f7e90159 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt @@ -1,6 +1,7 @@ package com.egm.stellio.entity.repository import com.egm.stellio.shared.model.QueryParams +import org.springframework.transaction.annotation.Transactional import java.net.URI interface SearchRepository { @@ -17,6 +18,7 @@ interface SearchRepository { * @property first count of all matching entities in the database. * @property second list of matching entities ids as requested by pagination sorted by entity id. */ + @Transactional(readOnly = true) fun getEntities( queryParams: QueryParams, userSub: String, diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt index 09335bb8d..31d96c7c4 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt @@ -2,15 +2,15 @@ package com.egm.stellio.entity.repository import com.egm.stellio.shared.model.QueryParams import com.egm.stellio.shared.util.toUri -import org.neo4j.ogm.session.Session import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.data.neo4j.core.Neo4jClient import org.springframework.stereotype.Component import java.net.URI @Component @ConditionalOnProperty("application.authentication.enabled", havingValue = "false") class StandaloneNeo4jSearchRepository( - private val session: Session + private val neo4jClient: Neo4jClient ) : SearchRepository { override fun getEntities( @@ -21,7 +21,7 @@ class StandaloneNeo4jSearchRepository( contexts: List ): Pair> { val query = QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, page, limit, contexts) - val result = session.query(query, emptyMap(), true) + val result = neo4jClient.query(query).fetch().all() return if (limit == 0) Pair( (result.firstOrNull()?.get("count") as Long?)?.toInt() ?: 0, diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt index 61b31f1d1..9aca339fa 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt @@ -84,7 +84,11 @@ class EntityAttributeService( datasetId: URI?, contexts: List ): UpdateAttributeResult { - val relationship = neo4jRepository.getRelationshipOfSubject(entityId, relationshipType, datasetId) + val relationship = + if (datasetId != null) + relationshipRepository.getRelationshipOfSubject(entityId, relationshipType, datasetId) + else + relationshipRepository.getRelationshipOfSubject(entityId, relationshipType) val relationshipUpdates = partialUpdateRelationshipOfAttribute( relationship, entityId, @@ -118,7 +122,11 @@ class EntityAttributeService( datasetId: URI?, contexts: List ): UpdateAttributeResult { - val property = neo4jRepository.getPropertyOfSubject(entityId, expandedPropertyName, datasetId) + val property = + if (datasetId != null) + propertyRepository.getPropertyOfSubject(entityId, expandedPropertyName, datasetId) + else + propertyRepository.getPropertyOfSubject(entityId, expandedPropertyName) val propertyUpdates = partialUpdatePropertyOfAttribute( property, propertyValues @@ -144,7 +152,7 @@ class EntityAttributeService( contexts: List ): Boolean { return attributeValues.filterKeys { - if (attribute.attributeType == "Relationship") + if (attribute is Relationship) !NGSILD_RELATIONSHIPS_CORE_MEMBERS.contains(it) else !NGSILD_PROPERTIES_CORE_MEMBERS.contains(it) @@ -155,27 +163,27 @@ class EntityAttributeService( val attributeOfAttributeName = it.key when { neo4jRepository.hasRelationshipInstance( - AttributeSubjectNode(attribute.id), + AttributeSubjectNode(attribute.id()), attributeOfAttributeName.toRelationshipTypeName() ) -> { - val relationshipOfAttribute = neo4jRepository.getRelationshipOfSubject( - attribute.id, + val relationshipOfAttribute = relationshipRepository.getRelationshipOfSubject( + attribute.id(), attributeOfAttributeName.toRelationshipTypeName() ) partialUpdateRelationshipOfAttribute( relationshipOfAttribute, - attribute.id, + attribute.id(), attributeOfAttributeName.toRelationshipTypeName(), null, it.value ) } neo4jRepository.hasPropertyInstance( - AttributeSubjectNode(attribute.id), + AttributeSubjectNode(attribute.id()), expandJsonLdKey(attributeOfAttributeName, contexts)!! ) -> { - val propertyOfAttribute = neo4jRepository.getPropertyOfSubject( - attribute.id, + val propertyOfAttribute = propertyRepository.getPropertyOfSubject( + attribute.id(), expandJsonLdKey(attributeOfAttributeName, contexts)!! ) partialUpdatePropertyOfAttribute( @@ -186,16 +194,16 @@ class EntityAttributeService( else -> { if (isAttributeOfType(it.value, JsonLdUtils.NGSILD_RELATIONSHIP_TYPE)) { val ngsiLdRelationship = NgsiLdRelationship( - attributeOfAttributeName.toRelationshipTypeName(), + attributeOfAttributeName, listOf(it.value) ) - entityService.createAttributeRelationships(attribute.id, listOf(ngsiLdRelationship)) + entityService.createAttributeRelationships(attribute.id(), listOf(ngsiLdRelationship)) } else if (isAttributeOfType(it.value, JsonLdUtils.NGSILD_PROPERTY_TYPE)) { val ngsiLdProperty = NgsiLdProperty( expandJsonLdKey(attributeOfAttributeName, contexts)!!, listOf(it.value) ) - entityService.createAttributeProperties(attribute.id, listOf(ngsiLdProperty)) + entityService.createAttributeProperties(attribute.id(), listOf(ngsiLdProperty)) } else false } } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 6cf2c5a2e..3bbf70127 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -374,10 +374,10 @@ class EntityService( ): UpdateAttributeResult { val relationshipTypeName = ngsiLdRelationship.name.extractShortTypeFromExpanded() return if (!neo4jRepository.hasRelationshipInstance( - EntitySubjectNode(entityId), - relationshipTypeName, - ngsiLdRelationshipInstance.datasetId - ) + EntitySubjectNode(entityId), + relationshipTypeName, + ngsiLdRelationshipInstance.datasetId + ) ) { createEntityRelationship( entityId, @@ -431,10 +431,10 @@ class EntityService( disallowOverwrite: Boolean ): UpdateAttributeResult { return if (!neo4jRepository.hasPropertyInstance( - EntitySubjectNode(entityId), - ngsiLdProperty.name, - ngsiLdPropertyInstance.datasetId - ) + EntitySubjectNode(entityId), + ngsiLdProperty.name, + ngsiLdPropertyInstance.datasetId + ) ) { createEntityProperty(entityId, ngsiLdProperty.name, ngsiLdPropertyInstance) UpdateAttributeResult( @@ -477,9 +477,9 @@ class EntityService( disallowOverwrite: Boolean ): UpdateAttributeResult { return if (!neo4jRepository.hasGeoPropertyOfName( - EntitySubjectNode(entityId), - ngsiLdGeoProperty.name.extractShortTypeFromExpanded() - ) + EntitySubjectNode(entityId), + ngsiLdGeoProperty.name.extractShortTypeFromExpanded() + ) ) { createLocationProperty( entityId, @@ -548,10 +548,10 @@ class EntityService( ngsiLdRelationshipInstance: NgsiLdRelationshipInstance ): UpdateAttributeResult = if (neo4jRepository.hasRelationshipInstance( - EntitySubjectNode(entityId), - ngsiLdRelationship.name.toRelationshipTypeName(), - ngsiLdRelationshipInstance.datasetId - ) + EntitySubjectNode(entityId), + ngsiLdRelationship.name.toRelationshipTypeName(), + ngsiLdRelationshipInstance.datasetId + ) ) { deleteEntityAttributeInstance(entityId, ngsiLdRelationship.name, ngsiLdRelationshipInstance.datasetId) createEntityRelationship( @@ -584,8 +584,8 @@ class EntityService( ngsiLdPropertyInstance: NgsiLdPropertyInstance ): UpdateAttributeResult = if (neo4jRepository.hasPropertyInstance( - EntitySubjectNode(entityId), ngsiLdProperty.name, ngsiLdPropertyInstance.datasetId - ) + EntitySubjectNode(entityId), ngsiLdProperty.name, ngsiLdPropertyInstance.datasetId + ) ) { updateEntityAttributeInstance(entityId, ngsiLdProperty.name, ngsiLdPropertyInstance) UpdateAttributeResult( @@ -650,8 +650,8 @@ class EntityService( subjectNodeInfo = EntitySubjectNode(entityId), propertyName = expandedAttributeName, deleteAll = true ) >= 1 else if (neo4jRepository.hasRelationshipOfType( - EntitySubjectNode(entityId), expandedAttributeName.toRelationshipTypeName() - ) + EntitySubjectNode(entityId), expandedAttributeName.toRelationshipTypeName() + ) ) return neo4jRepository.deleteEntityRelationship( subjectNodeInfo = EntitySubjectNode(entityId), @@ -669,8 +669,8 @@ class EntityService( expandedAttributeName, datasetId ) >= 1 else if (neo4jRepository.hasRelationshipInstance( - EntitySubjectNode(entityId), expandedAttributeName.toRelationshipTypeName(), datasetId - ) + EntitySubjectNode(entityId), expandedAttributeName.toRelationshipTypeName(), datasetId + ) ) return neo4jRepository.deleteEntityRelationship( EntitySubjectNode(entityId), diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt index 6a2ef3470..a3be5d179 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt @@ -53,7 +53,7 @@ class SubscriptionHandlerService( fun deleteSubscriptionEntity(id: URI) { // Delete the last notification of the subscription - val lastNotification = neo4jRepository.getRelationshipTargetOfSubject( + val lastNotification = entityRepository.getRelationshipTargetOfSubject( id, JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) @@ -87,14 +87,14 @@ class SubscriptionHandlerService( entityRepository.save(notification) // Find the last notification of the subscription - val lastNotification = neo4jRepository.getRelationshipTargetOfSubject( + val lastNotification = entityRepository.getRelationshipTargetOfSubject( subscriptionId, JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) // Create relationship between the subscription and the new notification if (lastNotification != null) { - val relationship = neo4jRepository.getRelationshipOfSubject( + val relationship = relationshipRepository.getRelationshipOfSubject( subscriptionId, JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt index 58f526588..468fb1f6c 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt @@ -5,6 +5,7 @@ import java.net.URLDecoder import java.time.LocalDate import java.time.LocalTime import java.time.ZonedDateTime +import java.time.format.DateTimeParseException fun splitQueryTermOnOperator(queryTerm: String): List = queryTerm.split("==", "!=", ">=", ">", "<=", "<") @@ -50,7 +51,7 @@ fun String.isDateTime(): Boolean = try { ZonedDateTime.parse(this) true - } catch (e: Exception) { + } catch (e: DateTimeParseException) { false } @@ -58,7 +59,7 @@ fun String.isDate(): Boolean = try { LocalDate.parse(this) true - } catch (e: Exception) { + } catch (e: DateTimeParseException) { false } @@ -66,6 +67,6 @@ fun String.isTime(): Boolean = try { LocalTime.parse(this) true - } catch (e: Exception) { + } catch (e: DateTimeParseException) { false } diff --git a/entity-service/src/main/resources/application-docker.properties b/entity-service/src/main/resources/application-docker.properties index a0f179234..c9356614f 100644 --- a/entity-service/src/main/resources/application-docker.properties +++ b/entity-service/src/main/resources/application-docker.properties @@ -1,4 +1,4 @@ -spring.data.neo4j.uri = bolt://neo4j:7687 +spring.neo4j.uri = bolt://neo4j:7687 org.neo4j.driver.uri=bolt://neo4j:7687 diff --git a/entity-service/src/main/resources/application.properties b/entity-service/src/main/resources/application.properties index 815e6acf0..262426ebd 100644 --- a/entity-service/src/main/resources/application.properties +++ b/entity-service/src/main/resources/application.properties @@ -1,7 +1,10 @@ -spring.data.neo4j.use-native-types=true -spring.data.neo4j.uri = bolt://localhost:7687 -spring.data.neo4j.username = neo4j -spring.data.neo4j.password = neo4j_password +spring.neo4j.uri = bolt://localhost:7687 +spring.neo4j.authentication.username = neo4j +spring.neo4j.authentication.password = neo4j_password +spring.data.neo4j.database = stellio + +org.neo4j.migrations.database = stellio +org.neo4j.migrations.check-location=false spring.kafka.bootstrap-servers = localhost:29092 # To ensure we get all past messages when dynamically joining a new topic based on our "cim.observation.*" pattern @@ -27,13 +30,6 @@ spring.mvc.log-request-details = true # application.graylog.port = 12201 # application.graylog.source = stellio-int -# Neo4j migrations configuration -org.neo4j.driver.authentication.username=neo4j -org.neo4j.driver.authentication.password=neo4j_password -org.neo4j.driver.uri=bolt://localhost:7687 -# Disable the check if the location exists -org.neo4j.migrations.check-location=false - # Pagination config for query resources endpoints application.pagination.limit-default = 30 application.pagination.limit-max = 100 diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt index 1ba912597..e1c937901 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt @@ -9,7 +9,7 @@ import com.egm.stellio.entity.authorization.AuthorizationService.Companion.R_CAN import com.egm.stellio.entity.authorization.AuthorizationService.Companion.R_IS_MEMBER_OF import com.egm.stellio.entity.authorization.AuthorizationService.Companion.SERVICE_ACCOUNT_ID import com.egm.stellio.entity.authorization.AuthorizationService.Companion.USER_LABEL -import com.egm.stellio.entity.config.TestContainersConfiguration +import com.egm.stellio.entity.config.WithNeo4jContainer import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.model.Property import com.egm.stellio.entity.model.Relationship @@ -19,18 +19,17 @@ import com.egm.stellio.entity.repository.Neo4jRepository import com.egm.stellio.entity.repository.SubjectNodeInfo import com.egm.stellio.shared.util.JsonLdUtils.EGM_SPECIFIC_ACCESS_POLICY import com.egm.stellio.shared.util.toUri +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.context.annotation.Import import org.springframework.test.context.ActiveProfiles import java.net.URI @SpringBootTest @ActiveProfiles("test") -@Import(TestContainersConfiguration::class) -class Neo4jAuthorizationRepositoryTest { +class Neo4jAuthorizationRepositoryTest : WithNeo4jContainer { @Autowired private lateinit var neo4jAuthorizationRepository: Neo4jAuthorizationRepository @@ -48,6 +47,11 @@ class Neo4jAuthorizationRepositoryTest { private val apiaryUri = "urn:ngsi-ld:Apiary:01".toUri() private val apiary02Uri = "urn:ngsi-ld:Apiary:02".toUri() + @AfterEach + fun cleanData() { + entityRepository.deleteAll() + } + @Test fun `it should filter entities authorized for user with given rights`() { val userEntity = createEntity(userUri, listOf(USER_LABEL), mutableListOf()) @@ -62,10 +66,7 @@ class Neo4jAuthorizationRepositoryTest { setOf(R_CAN_READ, R_CAN_WRITE) ) - assert(availableRightsForEntities == listOf(apiaryUri)) - - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(apiaryUri) + assertEquals(listOf(apiaryUri), availableRightsForEntities) } @Test @@ -83,9 +84,6 @@ class Neo4jAuthorizationRepositoryTest { ) assert(availableRightsForEntities.isEmpty()) - - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(apiaryUri) } @Test @@ -108,11 +106,7 @@ class Neo4jAuthorizationRepositoryTest { setOf(R_CAN_READ, R_CAN_WRITE) ) - assert(authorizedEntitiesId == listOf(apiaryUri, apiary02Uri)) - - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(apiaryUri) - neo4jRepository.deleteEntity(apiary02Uri) + assertEquals(listOf(apiaryUri, apiary02Uri), authorizedEntitiesId) } @Test @@ -138,9 +132,6 @@ class Neo4jAuthorizationRepositoryTest { ) assert(availableRightsForEntities.isEmpty()) - - neo4jRepository.deleteEntity(clientUri) - neo4jRepository.deleteEntity(apiaryUri) } @Test @@ -165,10 +156,7 @@ class Neo4jAuthorizationRepositoryTest { setOf(R_CAN_READ, R_CAN_WRITE) ) - assert(availableRightsForEntities == listOf(apiaryUri)) - - neo4jRepository.deleteEntity(clientUri) - neo4jRepository.deleteEntity(apiaryUri) + assertEquals(listOf(apiaryUri), availableRightsForEntities) } @Test @@ -190,10 +178,7 @@ class Neo4jAuthorizationRepositoryTest { listOf(SpecificAccessPolicy.AUTH_READ.name) ) - assert(authorizedEntities == listOf(apiaryUri)) - - neo4jRepository.deleteEntity(apiaryUri) - neo4jRepository.deleteEntity(apiary02Uri) + assertEquals(listOf(apiaryUri), authorizedEntities) } @Test @@ -214,9 +199,7 @@ class Neo4jAuthorizationRepositoryTest { listOf(SpecificAccessPolicy.AUTH_WRITE.name, SpecificAccessPolicy.AUTH_READ.name) ) - assert(authorizedEntities == listOf(apiaryUri)) - - neo4jRepository.deleteEntity(apiaryUri) + assertEquals(listOf(apiaryUri), authorizedEntities) } @Test @@ -246,10 +229,7 @@ class Neo4jAuthorizationRepositoryTest { listOf(SpecificAccessPolicy.AUTH_WRITE.name) ) - assert(authorizedEntities == listOf(apiaryUri)) - - neo4jRepository.deleteEntity(apiaryUri) - neo4jRepository.deleteEntity(apiary02Uri) + assertEquals(listOf(apiaryUri), authorizedEntities) } @Test @@ -267,9 +247,7 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles(userUri) - assert(roles == setOf("admin", "creator")) - - neo4jRepository.deleteEntity(userUri) + assertEquals(setOf("admin", "creator"), roles) } @Test @@ -291,9 +269,7 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles(serviceAccountUri) - assert(roles == setOf("admin", "creator")) - - neo4jRepository.deleteEntity(clientUri) + assertEquals(setOf("admin", "creator"), roles) } @Test @@ -311,9 +287,7 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles(userUri) - assert(roles == setOf("admin")) - - neo4jRepository.deleteEntity(userUri) + assertEquals(setOf("admin"), roles) } @Test @@ -335,10 +309,7 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles(userUri) - assert(roles == setOf("admin")) - - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(groupUri) + assertEquals(setOf("admin"), roles) } @Test @@ -360,10 +331,7 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles(userUri) - assert(roles == setOf("admin")) - - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(groupUri) + assertEquals(setOf("admin"), roles) } @Test @@ -373,8 +341,6 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles(userUri) assert(roles.isEmpty()) - - neo4jRepository.deleteEntity(userUri) } @Test @@ -393,8 +359,6 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles("urn:ngsi-ld:User:unknown".toUri()) assert(roles.isEmpty()) - - neo4jRepository.deleteEntity(clientUri) } @Test @@ -417,8 +381,6 @@ class Neo4jAuthorizationRepositoryTest { val roles = neo4jAuthorizationRepository.getUserRoles("urn:ngsi-ld:User:unknown".toUri()) assert(roles.isEmpty()) - - neo4jRepository.deleteEntity(clientUri) } @Test @@ -441,16 +403,13 @@ class Neo4jAuthorizationRepositoryTest { ) assertEquals(2, createdRelations.size) - - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(apiaryUri) - neo4jRepository.deleteEntity(apiary02Uri) } @Test fun `it should create admin links to entities for a client`() { createEntity( - clientUri, listOf(CLIENT_LABEL), + clientUri, + listOf(CLIENT_LABEL), mutableListOf( Property( name = SERVICE_ACCOUNT_ID, @@ -475,10 +434,6 @@ class Neo4jAuthorizationRepositoryTest { ) assertEquals(2, createdRelations.size) - - neo4jRepository.deleteEntity(clientUri) - neo4jRepository.deleteEntity(apiaryUri) - neo4jRepository.deleteEntity(apiary02Uri) } fun createEntity(id: URI, type: List, properties: MutableList): Entity { diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/config/TestContainersConfiguration.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/config/TestContainersConfiguration.kt deleted file mode 100644 index da87e2a71..000000000 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/config/TestContainersConfiguration.kt +++ /dev/null @@ -1,34 +0,0 @@ -package com.egm.stellio.entity.config - -import com.egm.stellio.shared.TestContainers -import org.springframework.boot.test.context.TestConfiguration -import org.springframework.context.annotation.Bean - -@TestConfiguration -class TestContainersConfiguration { - - private val DB_USER = "neo4j" - private val DB_PASSWORD = "neo4j_password" - - object EntityServiceTestContainers : TestContainers("neo4j", 7687) { - - fun getNeo4jUri(): String { - return "bolt://" + instance.getServiceHost(serviceName, servicePort) + ":" + instance.getServicePort( - serviceName, - servicePort - ) - } - } - - @Bean - fun connectionFactory(): org.neo4j.ogm.config.Configuration { - EntityServiceTestContainers.startContainers() - - return org.neo4j.ogm.config.Configuration.Builder() - .uri(EntityServiceTestContainers.getNeo4jUri()) - .credentials(DB_USER, DB_PASSWORD) - .useNativeTypes() - .database("stellio") - .build() - } -} diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WebSecurityTestConfig.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WebSecurityTestConfig.kt index 9c3cf4fd1..714255248 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WebSecurityTestConfig.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WebSecurityTestConfig.kt @@ -7,7 +7,8 @@ import org.springframework.security.oauth2.jwt.ReactiveJwtDecoder import org.springframework.security.oauth2.jwt.ReactiveJwtDecoders /** - * This configuration class was added since handlers tests (when importing a customSecurityConfiguration) requires defining a bean of type ReactiveJwtDecoder. + * This configuration class was added since handlers tests (when importing a customSecurityConfiguration) + * requires defining a bean of type ReactiveJwtDecoder. */ @TestConfiguration class WebSecurityTestConfig : WebSecurityConfig() { diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WithNeo4jContainer.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WithNeo4jContainer.kt new file mode 100644 index 000000000..d3325f170 --- /dev/null +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/config/WithNeo4jContainer.kt @@ -0,0 +1,36 @@ +package com.egm.stellio.entity.config + +import ac.simons.neo4j.migrations.springframework.boot.autoconfigure.MigrationsAutoConfiguration +import org.springframework.boot.autoconfigure.ImportAutoConfiguration +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.testcontainers.containers.Neo4jContainer +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers + +@Testcontainers +@ImportAutoConfiguration(MigrationsAutoConfiguration::class) +interface WithNeo4jContainer { + + companion object { + + @Container + val neo4jContainer = Neo4jContainer("neo4j:4.3").apply { + withNeo4jConfig("dbms.default_database", "stellio") + withEnv("NEO4JLABS_PLUGINS", "[\"apoc\"]") + withAdminPassword("neo4j_password") + } + + @JvmStatic + @DynamicPropertySource + fun properties(registry: DynamicPropertyRegistry) { + registry.add("spring.neo4j.uri") { neo4jContainer.boltUrl } + registry.add("spring.neo4j.authentication.username") { "neo4j" } + registry.add("spring.neo4j.authentication.password") { neo4jContainer.adminPassword } + } + + init { + neo4jContainer.start() + } + } +} diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/EntityRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/EntityRepositoryTests.kt index 678e0a49f..575dd6e13 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/EntityRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/EntityRepositoryTests.kt @@ -1,6 +1,6 @@ package com.egm.stellio.entity.repository -import com.egm.stellio.entity.config.TestContainersConfiguration +import com.egm.stellio.entity.config.WithNeo4jContainer import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.model.Property import com.egm.stellio.shared.util.toUri @@ -8,14 +8,12 @@ import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.context.annotation.Import import org.springframework.test.context.ActiveProfiles import java.net.URI @SpringBootTest @ActiveProfiles("test") -@Import(TestContainersConfiguration::class) -class EntityRepositoryTests { +class EntityRepositoryTests : WithNeo4jContainer { @Autowired private lateinit var entityRepository: EntityRepository diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt index 8cd4c6145..7ac7c02e1 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt @@ -1,6 +1,6 @@ package com.egm.stellio.entity.repository -import com.egm.stellio.entity.config.TestContainersConfiguration +import com.egm.stellio.entity.config.WithNeo4jContainer import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.model.PartialEntity import com.egm.stellio.entity.model.Property @@ -19,18 +19,18 @@ import junit.framework.TestCase.assertNotNull import junit.framework.TestCase.assertNull import junit.framework.TestCase.assertTrue import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.context.annotation.Import +import org.springframework.dao.EmptyResultDataAccessException import org.springframework.test.context.ActiveProfiles import java.net.URI @SpringBootTest @ActiveProfiles("test") -@Import(TestContainersConfiguration::class) -class Neo4jRepositoryTests { +class Neo4jRepositoryTests : WithNeo4jContainer { @Autowired private lateinit var neo4jRepository: Neo4jRepository @@ -50,6 +50,13 @@ class Neo4jRepositoryTests { private val beekeeperUri = "urn:ngsi-ld:Beekeeper:1230".toUri() private val partialTargetEntityUri = "urn:ngsi-ld:Entity:4567".toUri() + @AfterEach + fun cleanData() { + entityRepository.deleteAll() + propertyRepository.deleteAll() + relationshipRepository.deleteAll() + } + @Test fun `it should merge a partial entity to a normal entity`() { val partialEntity = PartialEntity(beekeeperUri) @@ -62,8 +69,6 @@ class Neo4jRepositoryTests { assertTrue(entityRepository.existsById(beekeeperUri)) val beekeeper = entityRepository.findById(beekeeperUri) assertEquals(listOf("Beekeeper"), beekeeper.get().type) - - neo4jRepository.deleteEntity(beekeeperUri) } @Test @@ -79,8 +84,6 @@ class Neo4jRepositoryTests { val relsOfProp = propertyRepository.findById(property.id).get().relationships assertEquals(1, relsOfProp.size) assertEquals(EGM_OBSERVED_BY, relsOfProp[0].type[0]) - - neo4jRepository.deleteEntity(beekeeperUri) } @Test @@ -119,16 +122,14 @@ class Neo4jRepositoryTests { assertEquals( 300L, - neo4jRepository.getPropertyOfSubject(entity.id, "https://uri.etsi.org/ngsi-ld/size").value + propertyRepository.getPropertyOfSubject(entity.id, "https://uri.etsi.org/ngsi-ld/size").value ) assertEquals( 200L, - neo4jRepository.getPropertyOfSubject( + propertyRepository.getPropertyOfSubject( entity.id, "https://uri.etsi.org/ngsi-ld/size", "urn:ngsi-ld:Dataset:size:1".toUri() ).value ) - - neo4jRepository.deleteEntity(entity.id) } @Test @@ -168,16 +169,14 @@ class Neo4jRepositoryTests { assertEquals( 100L, - neo4jRepository.getPropertyOfSubject(entity.id, "https://uri.etsi.org/ngsi-ld/size").value + propertyRepository.getPropertyOfSubject(entity.id, "https://uri.etsi.org/ngsi-ld/size").value ) assertEquals( 300L, - neo4jRepository.getPropertyOfSubject( + propertyRepository.getPropertyOfSubject( entity.id, "https://uri.etsi.org/ngsi-ld/size", "urn:ngsi-ld:Dataset:size:1".toUri() ).value ) - - neo4jRepository.deleteEntity(entity.id) } @Test @@ -215,12 +214,10 @@ class Neo4jRepositoryTests { newProperty.instances[0] ) - val updatedPropertyId = neo4jRepository.getPropertyOfSubject( + val updatedPropertyId = propertyRepository.getPropertyOfSubject( entity.id, "https://uri.etsi.org/ngsi-ld/name" ).id - assertEquals(propertyRepository.findById(updatedPropertyId).get().properties[0].value, "English") - - neo4jRepository.deleteEntity(entity.id) + assertEquals("English", propertyRepository.findById(updatedPropertyId).get().properties[0].value) } @Test @@ -275,15 +272,13 @@ class Neo4jRepositoryTests { newProperty.instances[0] ) - val updatedPropertyId = neo4jRepository.getPropertyOfSubject( + val updatedPropertyId = propertyRepository.getPropertyOfSubject( entity.id, "https://uri.etsi.org/ngsi-ld/temperature" ).id assertEquals( propertyRepository.findById(updatedPropertyId).get().relationships[0].type[0], "https://uri.etsi.org/ngsi-ld/default-context/newRel" ) - - neo4jRepository.deleteEntity(entity.id) } @Test @@ -291,7 +286,6 @@ class Neo4jRepositoryTests { val property = Property(name = "name", value = "Scalpa") propertyRepository.save(property) assertNotNull(propertyRepository.findById(property.id).get().modifiedAt) - propertyRepository.deleteById(property.id) } @Test @@ -299,12 +293,11 @@ class Neo4jRepositoryTests { val relationship = Relationship(type = listOf("connectsTo")) relationshipRepository.save(relationship) assertNotNull(relationshipRepository.findById(relationship.id).get().modifiedAt) - relationshipRepository.deleteById(relationship.id) } @Test fun `it should update modifiedAt value when updating an entity`() { - val entity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:1233".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = "name", value = "Scalpa")) @@ -316,8 +309,8 @@ class Neo4jRepositoryTests { mutableListOf(Property(name = "name", value = "Demha")) ) val updatedModifiedAt = entityRepository.findById("urn:ngsi-ld:Beekeeper:1233".toUri()).get().modifiedAt + assertNotNull(updatedModifiedAt) assertThat(updatedModifiedAt).isAfter(modifiedAt) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -328,7 +321,6 @@ class Neo4jRepositoryTests { mutableListOf(Property(name = "name", value = "Scalpa")) ) assertTrue(entity.createdAt.toString().endsWith("Z")) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -339,12 +331,11 @@ class Neo4jRepositoryTests { mutableListOf(Property(name = "name", value = "Scalpa")) ) assertTrue(entity.modifiedAt.toString().endsWith("Z")) - neo4jRepository.deleteEntity(entity.id) } @Test fun `it should filter existing entityIds`() { - val entity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:1233".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = "name", value = "Scalpa")) @@ -356,7 +347,6 @@ class Neo4jRepositoryTests { listOf("urn:ngsi-ld:Beekeeper:1233".toUri()), neo4jRepository.filterExistingEntitiesAsIds(entitiesIds) ) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -376,8 +366,7 @@ class Neo4jRepositoryTests { ) neo4jRepository.deleteEntityProperty(EntitySubjectNode(entity.id), "firstName", null, true) - assertEquals(entityRepository.findById(entity.id).get().properties.size, 1) - neo4jRepository.deleteEntity(entity.id) + assertEquals(1, entityRepository.findById(entity.id).get().properties.size) } @Test @@ -413,7 +402,6 @@ class Neo4jRepositoryTests { it.datasetId == "urn:ngsi-ld:Dataset:firstName:2".toUri() } ) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -453,7 +441,6 @@ class Neo4jRepositoryTests { it.datasetId == null } ) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -473,8 +460,6 @@ class Neo4jRepositoryTests { ) assertEquals(entityRepository.findById(sensor.id).get().relationships.size, 0) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -504,9 +489,6 @@ class Neo4jRepositoryTests { it.datasetId == "urn:ngsi-ld:Dataset:observedBy:01".toUri() } ) - - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -540,8 +522,6 @@ class Neo4jRepositoryTests { it.datasetId == null } ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -561,8 +541,6 @@ class Neo4jRepositoryTests { assertTrue(propertyRepository.findById(originProperty.id).isEmpty) assertTrue(propertyRepository.findById(lastNameProperty.id).isEmpty) assertEquals(entityRepository.findById(entity.id).get().properties.size, 1) - - neo4jRepository.deleteEntity(entity.id) } @Test @@ -580,9 +558,6 @@ class Neo4jRepositoryTests { val entity = entityRepository.findById(sensor.id).get() assertEquals(entity.relationships.size, 0) assertEquals(entity.properties.size, 0) - - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -595,8 +570,6 @@ class Neo4jRepositoryTests { val entity = entityRepository.findById(device.id).get() assertEquals(entity.relationships.size, 0) - - neo4jRepository.deleteEntity(device.id) } @Test @@ -607,8 +580,7 @@ class Neo4jRepositoryTests { mutableListOf(Property(name = "name", value = 100L)) ) - assertNotNull(neo4jRepository.getPropertyOfSubject(entity.id, "name", null)) - neo4jRepository.deleteEntity(entity.id) + assertNotNull(propertyRepository.getPropertyOfSubject(entity.id, "name")) } @Test @@ -619,10 +591,9 @@ class Neo4jRepositoryTests { mutableListOf(Property(name = "name", value = 100L, datasetId = "urn:ngsi-ld:Dataset:name:1".toUri())) ) - assertThrows("List is empty") { - neo4jRepository.getPropertyOfSubject(entity.id, "name", null) + assertThrows("List is empty") { + propertyRepository.getPropertyOfSubject(entity.id, "name") } - neo4jRepository.deleteEntity(entity.id) } @Test @@ -634,13 +605,12 @@ class Neo4jRepositoryTests { ) assertNotNull( - neo4jRepository.getPropertyOfSubject( + propertyRepository.getPropertyOfSubject( entity.id, "name", datasetId = "urn:ngsi-ld:Dataset:name:1".toUri() ) ) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -651,14 +621,13 @@ class Neo4jRepositoryTests { mutableListOf(Property(name = "name", value = 100L, datasetId = "urn:ngsi-ld:Dataset:name:1".toUri())) ) - assertThrows("List is empty") { - neo4jRepository.getPropertyOfSubject( + assertThrows("List is empty") { + propertyRepository.getPropertyOfSubject( entity.id, "name", datasetId = "urn:ngsi-ld:Dataset:name:2".toUri() ) } - neo4jRepository.deleteEntity(entity.id) } @Test @@ -670,7 +639,6 @@ class Neo4jRepositoryTests { ) assertTrue(neo4jRepository.hasPropertyInstance(EntitySubjectNode(entity.id), "name", null)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -682,7 +650,6 @@ class Neo4jRepositoryTests { ) assertFalse(neo4jRepository.hasPropertyInstance(EntitySubjectNode(entity.id), "name", null)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -700,7 +667,6 @@ class Neo4jRepositoryTests { "urn:ngsi-ld:Dataset:name:1".toUri() ) ) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -718,7 +684,6 @@ class Neo4jRepositoryTests { "urn:ngsi-ld:Dataset:name:2".toUri() ) ) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -733,8 +698,6 @@ class Neo4jRepositoryTests { EGM_OBSERVED_BY.toRelationshipTypeName() ) ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -753,8 +716,6 @@ class Neo4jRepositoryTests { EntitySubjectNode(sensor.id), EGM_OBSERVED_BY.toRelationshipTypeName() ) ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -775,8 +736,6 @@ class Neo4jRepositoryTests { "urn:ngsi-ld:Dataset:observedBy:01".toUri() ) ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -797,8 +756,6 @@ class Neo4jRepositoryTests { "urn:ngsi-ld:Dataset:observedBy:0002".toUri() ) ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) } @Test @@ -819,12 +776,9 @@ class Neo4jRepositoryTests { ) assertEquals( - neo4jRepository.getRelationshipTargetOfSubject(sensor.id, EGM_OBSERVED_BY.toRelationshipTypeName())!!.id, + entityRepository.getRelationshipTargetOfSubject(sensor.id, EGM_OBSERVED_BY.toRelationshipTypeName())!!.id, newDevice.id ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) - neo4jRepository.deleteEntity(newDevice.id) } @Test @@ -848,19 +802,16 @@ class Neo4jRepositoryTests { ) assertEquals( - neo4jRepository.getRelationshipTargetOfSubject( + entityRepository.getRelationshipTargetOfSubject( sensor.id, EGM_OBSERVED_BY.toRelationshipTypeName(), datasetId )!!.id, newDevice.id ) - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) - neo4jRepository.deleteEntity(newDevice.id) } @Test fun `it should return an emptyMap for unknown type`() { - val entity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTC".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -873,13 +824,11 @@ class Neo4jRepositoryTests { .getEntityTypeAttributesInformation("https://ontology.eglobalmark.com/apic#unknownType") assertTrue(attributesInformation.isEmpty()) - - neo4jRepository.deleteEntity(entity.id) } @Test fun `it should retrieve entity type attributes information for two entities`() { - val firstEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTC".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -887,7 +836,7 @@ class Neo4jRepositoryTests { Property(name = "humidity", value = 65) ) ) - val secondEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTB".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -901,19 +850,16 @@ class Neo4jRepositoryTests { val propertiesInformation = attributesInformation["properties"] as Set<*> - assertEquals(propertiesInformation.size, 3) + assertEquals(3, propertiesInformation.size) assertTrue(propertiesInformation.containsAll(listOf("humidity", "temperature", "incoming"))) assertEquals(attributesInformation["relationships"], emptySet()) assertEquals(attributesInformation["geoProperties"], emptySet()) assertEquals(attributesInformation["entityCount"], 2) - - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test fun `it should retrieve entity type attributes information for two entities with relationships`() { - val firstEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTC".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -941,14 +887,11 @@ class Neo4jRepositoryTests { assertEquals(attributesInformation["relationships"], setOf("observedBy")) assertEquals(attributesInformation["geoProperties"], emptySet()) assertEquals(attributesInformation["entityCount"], 2) - - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test fun `it should retrieve entity type attributes information for three entities with location`() { - val firstEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTC".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -957,7 +900,7 @@ class Neo4jRepositoryTests { ), NgsiLdGeoPropertyInstance.toWktFormat(GeoPropertyType.Point, listOf(24.30623, 60.07966)) ) - val secondEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTB".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -965,7 +908,7 @@ class Neo4jRepositoryTests { Property(name = "humidity", value = 61) ) ) - val thirdEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTD".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -984,10 +927,6 @@ class Neo4jRepositoryTests { assertEquals(attributesInformation["relationships"], emptySet()) assertEquals(attributesInformation["geoProperties"], setOf("https://uri.etsi.org/ngsi-ld/location")) assertEquals(attributesInformation["entityCount"], 3) - - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test @@ -1008,7 +947,7 @@ class Neo4jRepositoryTests { Property(name = "isContainedIn", value = 61) ) ) - val thirdEntity = createEntity( + createEntity( "urn:ngsi-ld:Sensor:TESTB".toUri(), listOf("https://ontology.eglobalmark.com/apic#Sensor"), mutableListOf( @@ -1027,10 +966,6 @@ class Neo4jRepositoryTests { listOf("https://ontology.eglobalmark.com/apic#Beehive", "https://ontology.eglobalmark.com/apic#Sensor") ) ) - - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test @@ -1043,7 +978,7 @@ class Neo4jRepositoryTests { Property(name = "humidity", value = 65) ) ) - val secondEntity = createEntity( + createEntity( "urn:ngsi-ld:Beehive:TESTB".toUri(), listOf("https://ontology.eglobalmark.com/apic#Beehive"), mutableListOf( @@ -1063,7 +998,7 @@ class Neo4jRepositoryTests { val entityTypes = neo4jRepository.getEntityTypes() - assertEquals(entityTypes.size, 2) + assertEquals(2, entityTypes.size) assertTrue( entityTypes.containsAll( listOf( @@ -1082,10 +1017,6 @@ class Neo4jRepositoryTests { ) ) ) - - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test @@ -1099,18 +1030,13 @@ class Neo4jRepositoryTests { "urn:ngsi-ld:Dataset:observedBy:01".toUri() ) - val relationship = neo4jRepository.getRelationshipOfSubject( + val relationship = relationshipRepository.getRelationshipOfSubject( sensor.id, EGM_OBSERVED_BY.toRelationshipTypeName(), "urn:ngsi-ld:Dataset:observedBy:01".toUri() ) - assertEquals( - relationship.type, listOf(EGM_OBSERVED_BY) - ) - - neo4jRepository.deleteEntity(sensor.id) - neo4jRepository.deleteEntity(device.id) + assertEquals(relationship.type, listOf(EGM_OBSERVED_BY)) } fun createEntity( diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt index 810a8d0fe..90f926909 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt @@ -8,7 +8,7 @@ import com.egm.stellio.entity.authorization.AuthorizationService.Companion.R_CAN import com.egm.stellio.entity.authorization.AuthorizationService.Companion.R_IS_MEMBER_OF import com.egm.stellio.entity.authorization.AuthorizationService.Companion.SERVICE_ACCOUNT_ID import com.egm.stellio.entity.authorization.Neo4jAuthorizationService -import com.egm.stellio.entity.config.TestContainersConfiguration +import com.egm.stellio.entity.config.WithNeo4jContainer import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.model.Property import com.egm.stellio.entity.model.Relationship @@ -20,12 +20,12 @@ import com.egm.stellio.shared.util.toUri import com.ninjasquad.springmockk.MockkBean import io.mockk.every import junit.framework.TestCase.* +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.context.annotation.Import import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.TestPropertySource import java.net.URI @@ -33,8 +33,7 @@ import java.net.URI @SpringBootTest @ActiveProfiles("test") @TestPropertySource(properties = ["application.authentication.enabled=true"]) -@Import(TestContainersConfiguration::class) -class Neo4jSearchRepositoryTests { +class Neo4jSearchRepositoryTests : WithNeo4jContainer { @Autowired private lateinit var searchRepository: SearchRepository @@ -57,6 +56,11 @@ class Neo4jSearchRepositoryTests { private val page = 1 private val limit = 20 + @AfterEach + fun cleanData() { + entityRepository.deleteAll() + } + @Test fun `it should return matching entities that user can access`() { val userEntity = createEntity(userUri, listOf(AuthorizationService.USER_LABEL), mutableListOf()) @@ -88,10 +92,6 @@ class Neo4jSearchRepositoryTests { ).second assertTrue(entities.containsAll(listOf(firstEntity.id, secondEntity.id, thirdEntity.id))) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test @@ -121,10 +121,6 @@ class Neo4jSearchRepositoryTests { ).second assertTrue(entities.containsAll(listOf(firstEntity.id, secondEntity.id))) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(groupEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test @@ -144,8 +140,6 @@ class Neo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(userUri) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -182,14 +176,11 @@ class Neo4jSearchRepositoryTests { ).second assertTrue(entities.containsAll(listOf(firstEntity.id, secondEntity.id))) - neo4jRepository.deleteEntity(clientEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test fun `it should return all matching entities for admin users`() { - val userEntity = createEntity( + createEntity( userUri, listOf(AuthorizationService.USER_LABEL), mutableListOf( @@ -221,14 +212,11 @@ class Neo4jSearchRepositoryTests { ).second assertTrue(entities.containsAll(listOf(firstEntity.id, secondEntity.id))) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test fun `it should return matching entities as the specific access policy`() { - val userEntity = createEntity(userUri, listOf(AuthorizationService.USER_LABEL), mutableListOf()) + createEntity(userUri, listOf(AuthorizationService.USER_LABEL), mutableListOf()) val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), @@ -256,9 +244,6 @@ class Neo4jSearchRepositoryTests { assertTrue(entities.contains(firstEntity.id)) assertFalse(entities.contains(secondEntity.id)) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test @@ -292,10 +277,6 @@ class Neo4jSearchRepositoryTests { ).first assertEquals(entitiesCount, 2) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test @@ -324,9 +305,6 @@ class Neo4jSearchRepositoryTests { assertEquals(countAndEntities.first, 1) assertEquals(countAndEntities.second, emptyList()) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @ParameterizedTest @@ -366,10 +344,6 @@ class Neo4jSearchRepositoryTests { ).second assertTrue(entities.containsAll(expectedEntitiesIds)) - neo4jRepository.deleteEntity(userEntity.id) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } fun createEntity( diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt index a839aa26d..2d7ff17a3 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt @@ -1,6 +1,6 @@ package com.egm.stellio.entity.repository -import com.egm.stellio.entity.config.TestContainersConfiguration +import com.egm.stellio.entity.config.WithNeo4jContainer import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.model.Property import com.egm.stellio.entity.model.Relationship @@ -8,13 +8,15 @@ import com.egm.stellio.shared.model.QueryParams import com.egm.stellio.shared.util.DEFAULT_CONTEXTS import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdKey import com.egm.stellio.shared.util.toUri -import junit.framework.TestCase.* +import junit.framework.TestCase.assertEquals +import junit.framework.TestCase.assertFalse +import junit.framework.TestCase.assertTrue +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.context.annotation.Import import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.TestPropertySource import java.net.URI @@ -27,8 +29,7 @@ import java.time.ZonedDateTime @SpringBootTest @ActiveProfiles("test") @TestPropertySource(properties = ["application.authentication.enabled=false"]) -@Import(TestContainersConfiguration::class) -class StandaloneNeo4jSearchRepositoryTests { +class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { @Autowired private lateinit var searchRepository: SearchRepository @@ -46,6 +47,11 @@ class StandaloneNeo4jSearchRepositoryTests { private val page = 1 private val limit = 20 + @AfterEach + fun cleanData() { + entityRepository.deleteAll() + } + @Test fun `it should return an entity if type and string properties are correct`() { val entity = createEntity( @@ -63,7 +69,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -83,7 +88,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -103,7 +107,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -123,7 +126,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -143,7 +145,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -163,7 +164,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -182,7 +182,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -201,7 +200,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -220,7 +218,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -239,7 +236,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -266,7 +262,6 @@ class StandaloneNeo4jSearchRepositoryTests { DEFAULT_CONTEXTS ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -290,7 +285,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -314,7 +308,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -338,7 +331,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertFalse(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -362,7 +354,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -387,7 +378,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -422,8 +412,6 @@ class StandaloneNeo4jSearchRepositoryTests { DEFAULT_CONTEXTS ).second assertFalse(entities.contains(entity.id)) - - neo4jRepository.deleteEntity(entity.id) } @Test @@ -486,7 +474,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.contains(entity.id)) - neo4jRepository.deleteEntity(entity.id) } @Test @@ -511,8 +498,6 @@ class StandaloneNeo4jSearchRepositoryTests { assertFalse(entities.contains(firstEntity.id)) assertTrue(entities.contains(secondEntity.id)) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) } @Test @@ -620,9 +605,6 @@ class StandaloneNeo4jSearchRepositoryTests { assertFalse(entities.contains(firstEntity.id)) assertTrue(entities.contains(secondEntity.id)) assertTrue(entities.contains(thirdEntity.id)) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test @@ -653,24 +635,21 @@ class StandaloneNeo4jSearchRepositoryTests { assertFalse(entities.contains(firstEntity.id)) assertTrue(entities.contains(secondEntity.id)) assertFalse(entities.contains(thirdEntity.id)) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test fun `it should return no entity if given idPattern is not matching`() { - val firstEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) ) - val secondEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) ) - val thirdEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:11232".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) @@ -684,24 +663,21 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.isEmpty()) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @Test fun `it should return matching entities count`() { - val firstEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) ) - val secondEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) ) - val thirdEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:03432".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) @@ -715,9 +691,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).first assertEquals(entitiesCount, 2) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } @ParameterizedTest @@ -728,17 +701,17 @@ class StandaloneNeo4jSearchRepositoryTests { limit: Int, expectedEntitiesIds: List ) { - val firstEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) ) - val secondEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) ) - val thirdEntity = createEntity( + createEntity( "urn:ngsi-ld:Beekeeper:03432".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) @@ -753,9 +726,6 @@ class StandaloneNeo4jSearchRepositoryTests { ).second assertTrue(entities.containsAll(expectedEntitiesIds)) - neo4jRepository.deleteEntity(firstEntity.id) - neo4jRepository.deleteEntity(secondEntity.id) - neo4jRepository.deleteEntity(thirdEntity.id) } fun createEntity( diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt index d1767419e..7c38f6046 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt @@ -58,12 +58,12 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val property = Property(propertyName, "years", 0) + val property = Property(name = propertyName, unitCode = "years", value = 0) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(any(), any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any()) } returns property + every { propertyRepository.save(any()) } returns property entityAttributeService.partialUpdateEntityAttribute(fishUri, expandedPayload, listOf(AQUAC_COMPOUND_CONTEXT)) @@ -84,12 +84,12 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val property = Property(propertyName, "years", 0) + val property = Property(name = propertyName, unitCode = "years", value = 0) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(any(), any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any(), any()) } returns property + every { propertyRepository.save(any()) } returns property entityAttributeService.partialUpdateEntityAttribute(fishUri, expandedPayload, listOf(AQUAC_COMPOUND_CONTEXT)) @@ -116,13 +116,13 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val property = Property(propertyName, "months", 0) - val depthProperty = Property("depth", null, 0) + val property = Property(name = propertyName, unitCode = "months", value = 0) + val depthProperty = Property(name = "depth", value = 0) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(fishUri, any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any()) } returns property + every { propertyRepository.save(any()) } returns property every { neo4jRepository.hasPropertyInstance( match { it.label == "Attribute" }, @@ -132,7 +132,7 @@ class EntityAttributeServiceTests { } returns false every { entityService.createAttributeProperties(any(), any()) } returns true every { - neo4jRepository.getPropertyOfSubject( + propertyRepository.getPropertyOfSubject( property.id, "https://ontology.eglobalmark.com/egm#depth", any() @@ -173,17 +173,17 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val property = Property(propertyName, "years", 0) - val relationship = Relationship(listOf("measuredBy")) + val property = Property(name = propertyName, unitCode = "years", value = 0) + val relationship = Relationship(type = listOf("measuredBy")) every { neo4jRepository.hasRelationshipInstance(match { it.label == "Entity" }, any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(fishUri, any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(fishUri, any()) } returns property + every { propertyRepository.save(any()) } returns property every { neo4jRepository.hasRelationshipInstance(match { it.label == "Attribute" }, any(), any()) } returns true - every { neo4jRepository.getRelationshipOfSubject(property.id, any()) } returns relationship + every { relationshipRepository.getRelationshipOfSubject(property.id, any()) } returns relationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true - every { relationshipRepository.save(any()) } returns relationship + every { relationshipRepository.save(any()) } returns relationship entityAttributeService.partialUpdateEntityAttribute(fishUri, expandedPayload, listOf(AQUAC_COMPOUND_CONTEXT)) @@ -213,12 +213,12 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val property = Property(propertyName, "months", 0) + val property = Property(name = propertyName, unitCode = "months", value = 0) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(fishUri, any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(fishUri, any()) } returns property + every { propertyRepository.save(any()) } returns property every { neo4jRepository.hasPropertyInstance( match { it.id == property.id }, @@ -247,12 +247,12 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(listOf("isContainedIn")) + val relationship = Relationship(type = listOf("isContainedIn")) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true - every { neo4jRepository.getRelationshipOfSubject(any(), any()) } returns relationship + every { relationshipRepository.getRelationshipOfSubject(any(), any()) } returns relationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true - every { relationshipRepository.save(any()) } returns relationship + every { relationshipRepository.save(any()) } returns relationship entityAttributeService.partialUpdateEntityAttribute(fishUri, expandedPayload, listOf(AQUAC_COMPOUND_CONTEXT)) @@ -281,12 +281,12 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(listOf("isContainedIn")) + val relationship = Relationship(type = listOf("isContainedIn")) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true - every { neo4jRepository.getRelationshipOfSubject(any(), any()) } returns relationship + every { relationshipRepository.getRelationshipOfSubject(any(), any(), any()) } returns relationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true - every { relationshipRepository.save(any()) } returns relationship + every { relationshipRepository.save(any()) } returns relationship entityAttributeService.partialUpdateEntityAttribute(fishUri, expandedPayload, listOf(AQUAC_COMPOUND_CONTEXT)) @@ -320,17 +320,18 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(listOf("isContainedIn")) - val property = Property("depth", null, 0) + val relationship = Relationship(type = listOf("isContainedIn")) + val property = Property(name = "depth", value = 0) every { neo4jRepository.hasRelationshipInstance(match { it.label == "Entity" }, any(), any()) } returns true - every { neo4jRepository.getRelationshipOfSubject(any(), any()) } returns relationship + every { relationshipRepository.getRelationshipOfSubject(any(), any(), any()) } returns relationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true - every { relationshipRepository.save(any()) } returns relationship + every { relationshipRepository.save(any()) } returns relationship every { neo4jRepository.hasRelationshipInstance(match { it.label == "Attribute" }, any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(any(), any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any(), any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any()) } returns property + every { propertyRepository.save(any()) } returns property entityAttributeService.partialUpdateEntityAttribute(fishUri, expandedPayload, listOf(AQUAC_COMPOUND_CONTEXT)) @@ -365,14 +366,14 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(listOf("isContainedIn")) - val measuredByRelationship = Relationship(listOf("measuredBy")) + val relationship = Relationship(type = listOf("isContainedIn")) + val measuredByRelationship = Relationship(type = listOf("measuredBy")) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true - every { neo4jRepository.getRelationshipOfSubject(fishUri, any()) } returns relationship - every { relationshipRepository.save(any()) } returns relationship + every { relationshipRepository.getRelationshipOfSubject(fishUri, any()) } returns relationship + every { relationshipRepository.save(any()) } returns relationship every { neo4jRepository.hasRelationshipInstance(any(), "observedBy", any()) } returns false - every { neo4jRepository.getRelationshipOfSubject(relationship.id, any()) } returns measuredByRelationship + every { relationshipRepository.getRelationshipOfSubject(relationship.id, any()) } returns measuredByRelationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true every { entityService.createAttributeRelationships(any(), any()) } returns true @@ -442,12 +443,13 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val property = Property(propertyName, "years", 0) + val property = Property(name = propertyName, unitCode = "years", value = 0) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true - every { neo4jRepository.getPropertyOfSubject(any(), any(), any()) } returns property - every { propertyRepository.save(any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any()) } returns property + every { propertyRepository.getPropertyOfSubject(any(), any(), any()) } returns property + every { propertyRepository.save(any()) } returns property val result = entityAttributeService.partialUpdateEntityAttribute( fishUri, @@ -495,11 +497,11 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(listOf("isContainedIn")) + val relationship = Relationship(type = listOf("isContainedIn")) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true - every { neo4jRepository.getRelationshipOfSubject(any(), any()) } returns relationship + every { relationshipRepository.getRelationshipOfSubject(any(), any(), any()) } returns relationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true - every { relationshipRepository.save(any()) } returns relationship + every { relationshipRepository.save(any()) } returns relationship val result = entityAttributeService.partialUpdateEntityAttribute( fishUri, diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt index 6d31b433a..1affb1fa0 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt @@ -22,8 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles import java.net.URI -import java.time.Instant -import java.time.ZoneOffset import java.time.ZonedDateTime import java.util.Optional import java.util.UUID @@ -38,20 +36,14 @@ class EntityServiceTests { @MockkBean(relaxed = true) private lateinit var neo4jRepository: Neo4jRepository - @MockkBean(relaxed = true) - private lateinit var neo4jSearchRepository: Neo4jSearchRepository - @MockkBean(relaxed = true) private lateinit var entityRepository: EntityRepository - @MockkBean - private lateinit var propertyRepository: PropertyRepository - @MockkBean private lateinit var partialEntityRepository: PartialEntityRepository @MockkBean - private lateinit var entityEventService: EntityEventService + private lateinit var searchRepository: SearchRepository private val mortalityRemovalServiceUri = "urn:ngsi-ld:MortalityRemovalService:014YFA9Z".toUri() private val fishContainmentUri = "urn:ngsi-ld:FishContainment:1234".toUri() @@ -1107,16 +1099,4 @@ class EntityServiceTests { ) confirmVerified() } - - private fun gimmeAnObservation(): Observation { - return Observation( - attributeName = "incoming", - latitude = 43.12, - longitude = 65.43, - observedBy = "urn:ngsi-ld:Sensor:01XYZ".toUri(), - unitCode = "CEL", - value = 12.4, - observedAt = Instant.now().atZone(ZoneOffset.UTC) - ) - } } diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerServiceTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerServiceTests.kt index 0c2bb034a..47f3d0993 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerServiceTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerServiceTests.kt @@ -54,8 +54,8 @@ class SubscriptionHandlerServiceTests { val mockkedProperty = mockkClass(Property::class) every { entityService.exists(any()) } returns false - every { propertyRepository.save(any()) } returns mockkedProperty - every { entityRepository.save(any()) } returns mockkedSubscription + every { propertyRepository.save(any()) } returns mockkedProperty + every { entityRepository.save(any()) } returns mockkedSubscription subscriptionHandlerService.createSubscriptionEntity(subscriptionId, subscriptionType, properties) @@ -88,13 +88,13 @@ class SubscriptionHandlerServiceTests { fun `it should delete a subscription`() { val subscriptionId = "urn:ngsi-ld:Subscription:04".toUri() - every { neo4jRepository.getRelationshipTargetOfSubject(any(), any()) } returns null + every { entityRepository.getRelationshipTargetOfSubject(any(), any()) } returns null every { entityService.deleteEntity(any()) } returns Pair(2, 1) subscriptionHandlerService.deleteSubscriptionEntity(subscriptionId) verify { - neo4jRepository.getRelationshipTargetOfSubject( + entityRepository.getRelationshipTargetOfSubject( eq(subscriptionId), JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) @@ -109,7 +109,7 @@ class SubscriptionHandlerServiceTests { val subscriptionId = "urn:ngsi-ld:Subscription:04".toUri() val notificationId = "urn:ngsi-ld:Notification:01".toUri() - every { neo4jRepository.getRelationshipTargetOfSubject(any(), any()) } answers { + every { entityRepository.getRelationshipTargetOfSubject(any(), any()) } answers { mockkClass(Entity::class, relaxed = true) { every { id } returns notificationId } @@ -119,7 +119,7 @@ class SubscriptionHandlerServiceTests { subscriptionHandlerService.deleteSubscriptionEntity(subscriptionId) verify { - neo4jRepository.getRelationshipTargetOfSubject( + entityRepository.getRelationshipTargetOfSubject( eq(subscriptionId), JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) @@ -143,9 +143,9 @@ class SubscriptionHandlerServiceTests { val mockkedProperty = mockkClass(Property::class) every { entityRepository.getEntityCoreById(any()) } returns mockkedSubscription - every { propertyRepository.save(any()) } returns mockkedProperty - every { entityRepository.save(any()) } returns mockkedNotification - every { neo4jRepository.getRelationshipTargetOfSubject(any(), any()) } returns null + every { propertyRepository.save(any()) } returns mockkedProperty + every { entityRepository.save(any()) } returns mockkedNotification + every { entityRepository.getRelationshipTargetOfSubject(any(), any()) } returns null every { mockkedSubscription.id } returns subscriptionId every { neo4jRepository.createRelationshipOfSubject(any(), any(), any()) } returns true @@ -157,7 +157,7 @@ class SubscriptionHandlerServiceTests { verify { propertyRepository.save(any()) } verify { entityRepository.save(any()) } verify { - neo4jRepository.getRelationshipTargetOfSubject( + entityRepository.getRelationshipTargetOfSubject( subscriptionId, JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) @@ -183,15 +183,15 @@ class SubscriptionHandlerServiceTests { val mockkedRelationship = mockkClass(Relationship::class) every { entityRepository.getEntityCoreById(any()) } returns mockkedSubscription - every { propertyRepository.save(any()) } returns mockkedProperty - every { entityRepository.save(any()) } returns mockkedNotification - every { neo4jRepository.getRelationshipTargetOfSubject(any(), any()) } returns mockkedLastNotification - every { neo4jRepository.getRelationshipOfSubject(any(), any()) } returns mockkedRelationship + every { propertyRepository.save(any()) } returns mockkedProperty + every { entityRepository.save(any()) } returns mockkedNotification + every { entityRepository.getRelationshipTargetOfSubject(any(), any()) } returns mockkedLastNotification + every { relationshipRepository.getRelationshipOfSubject(any(), any()) } returns mockkedRelationship every { mockkedRelationship.id } returns relationshipId every { mockkedNotification.id } returns notificationId every { mockkedLastNotification.id } returns lastNotificationId every { neo4jRepository.updateTargetOfRelationship(any(), any(), any(), any()) } returns 1 - every { relationshipRepository.save(any()) } returns mockkedRelationship + every { relationshipRepository.save(any()) } returns mockkedRelationship every { entityService.deleteEntity(any()) } returns Pair(1, 1) subscriptionHandlerService.createNotificationEntity( @@ -202,13 +202,13 @@ class SubscriptionHandlerServiceTests { verify { propertyRepository.save(any()) } verify { entityRepository.save(any()) } verify { - neo4jRepository.getRelationshipTargetOfSubject( + entityRepository.getRelationshipTargetOfSubject( subscriptionId, JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) } verify { - neo4jRepository.getRelationshipOfSubject( + relationshipRepository.getRelationshipOfSubject( subscriptionId, JsonLdUtils.EGM_RAISED_NOTIFICATION.toRelationshipTypeName() ) diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt index c172ceb8a..3171e41fc 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt @@ -1679,7 +1679,7 @@ class EntityHandlerTests { { "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData", "title": "The request includes input data which does not meet the requirements of the operation", - "detail": "Unexpected error while parsing payload : loading remote context failed: http://easyglobalmarket.com/contexts/diat.jsonld" + "detail": "Unexpected error while parsing payload (cause was: com.github.jsonldjava.core.JsonLdError: loading remote context failed: http://easyglobalmarket.com/contexts/diat.jsonld)" } """.trimIndent() ) diff --git a/entity-service/src/test/resources/logback-test.xml b/entity-service/src/test/resources/logback-test.xml index d95c806ba..e6a9d7154 100644 --- a/entity-service/src/test/resources/logback-test.xml +++ b/entity-service/src/test/resources/logback-test.xml @@ -18,10 +18,8 @@ - - - + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 12d38de6a..0f80bbf51 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/search-service/build.gradle.kts b/search-service/build.gradle.kts index 67842228e..c09c1c35a 100644 --- a/search-service/build.gradle.kts +++ b/search-service/build.gradle.kts @@ -12,17 +12,20 @@ plugins { dependencies { implementation("org.springframework.boot:spring-boot-starter-data-r2dbc") // required for Flyway's direct access to the DB to apply migration scripts + // (https://github.com/flyway/flyway/issues/2502) implementation("org.springframework:spring-jdbc") implementation("org.flywaydb:flyway-core") - implementation(project(":shared")) // implementation (and not runtime) because we are using the native jsonb encoding provided by PG implementation("io.r2dbc:r2dbc-postgresql") + implementation(project(":shared")) developmentOnly("org.springframework.boot:spring-boot-devtools") runtimeOnly("org.postgresql:postgresql") testImplementation("com.github.tomakehurst:wiremock-standalone:2.25.1") + testImplementation("org.testcontainers:postgresql") + testImplementation("org.testcontainers:r2dbc") testImplementation(testFixtures(project(":shared"))) } diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml index f5d782df9..cc39b71f3 100644 --- a/search-service/config/detekt/baseline.xml +++ b/search-service/config/detekt/baseline.xml @@ -3,13 +3,12 @@ LargeClass:EntityEventListenerServiceTest.kt$EntityEventListenerServiceTest - LargeClass:TemporalEntityHandlerTests.kt$TemporalEntityHandlerTests LongMethod:ParameterizedTests.kt$ParameterizedTests.Companion$@JvmStatic fun rawResultsProvider(): Stream<Arguments> LongParameterList:AttributeInstance.kt$AttributeInstance.Companion$( temporalEntityAttribute: UUID, instanceId: URI? = null, observedAt: ZonedDateTime, value: String? = null, measuredValue: Double? = null, payload: Map<String, Any> ) LongParameterList:EntityEventListenerService.kt$EntityEventListenerService$( entityId: URI, expandedAttributeName: String, datasetId: URI?, attributeValuesNode: JsonNode, updatedEntity: String, contexts: List<String> ) - MaxLineLength:WebSecurityTestConfig.kt$WebSecurityTestConfig : WebSecurityConfig ReturnCount:EntityEventListenerService.kt$EntityEventListenerService$internal fun toTemporalAttributeMetadata(jsonNode: JsonNode): Validated<String, AttributeMetadata> ReturnCount:TemporalEntityAttributeService.kt$TemporalEntityAttributeService$internal fun toTemporalAttributeMetadata( ngsiLdAttributeInstance: NgsiLdAttributeInstance ): Validated<String, AttributeMetadata> + SwallowedException:TemporalEntityHandler.kt$catch (e: IllegalArgumentException) { "'timerel' is not valid, it should be one of 'before', 'between', or 'after'".left() } ThrowsCount:TemporalEntityHandler.kt$internal fun buildTemporalQuery(params: MultiValueMap<String, String>, contextLink: String): TemporalQuery UtilityClassWithPublicConstructor:ParameterizedTests.kt$ParameterizedTests UtilityClassWithPublicConstructor:QueryParameterizedTests.kt$QueryParameterizedTests diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/EntityPayload.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/EntityPayload.kt new file mode 100644 index 000000000..da849ce38 --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/EntityPayload.kt @@ -0,0 +1,8 @@ +package com.egm.stellio.search.model + +import java.net.URI + +data class EntityPayload( + val entityId: URI, + val entityPayload: String +) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntityAttribute.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntityAttribute.kt index e5fe556fa..d58c26b68 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntityAttribute.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntityAttribute.kt @@ -1,9 +1,11 @@ package com.egm.stellio.search.model +import org.springframework.data.annotation.Id import java.net.URI import java.util.UUID data class TemporalEntityAttribute( + @Id val id: UUID = UUID.randomUUID(), val entityId: URI, val type: String, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt index 72a8c83dc..a55220764 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt @@ -24,7 +24,7 @@ data class TemporalQuery( companion object { fun isSupportedAggregate(aggregate: String): Boolean = - values().toList().any { it.name == aggregate.toUpperCase() } + values().toList().any { it.name == aggregate.uppercase() } } } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index 7bd5dc225..69539e55c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -10,8 +10,8 @@ import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMap import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsDateTime import com.egm.stellio.shared.util.extractAttributeInstanceFromCompactedEntity import io.r2dbc.postgresql.codec.Json -import org.springframework.data.r2dbc.core.DatabaseClient -import org.springframework.data.r2dbc.core.bind +import org.springframework.r2dbc.core.DatabaseClient +import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import reactor.core.publisher.Mono @@ -27,7 +27,7 @@ class AttributeInstanceService( @Transactional fun create(attributeInstance: AttributeInstance): Mono = - databaseClient.execute( + databaseClient.sql( """ INSERT INTO attribute_instance (observed_at, measured_value, value, temporal_entity_attribute, instance_id, payload) @@ -60,12 +60,11 @@ class AttributeInstanceService( observedAt = getPropertyValueFromMapAsDateTime(attributeValues, EGM_OBSERVED_BY)!!, value = valueToStringOrNull(attributeValue), measuredValue = valueToDoubleOrNull(attributeValue), - payload = - extractAttributeInstanceFromCompactedEntity( - parsedPayload, - attributeKey, - null - ) + payload = extractAttributeInstanceFromCompactedEntity( + parsedPayload, + attributeKey, + null + ) ) return create(attributeInstance) } @@ -131,7 +130,7 @@ class AttributeInstanceService( if (temporalQuery.lastN != null) selectQuery = selectQuery.plus(" LIMIT ${temporalQuery.lastN}") - return databaseClient.execute(selectQuery) + return databaseClient.sql(selectQuery) .fetch() .all() .map { @@ -141,7 +140,7 @@ class AttributeInstanceService( } fun deleteAttributeInstancesOfEntity(entityId: URI): Mono = - databaseClient.execute( + databaseClient.sql( """ DELETE FROM attribute_instance WHERE temporal_entity_attribute IN ( SELECT id FROM temporal_entity_attribute WHERE entity_id = :entity_id @@ -153,7 +152,7 @@ class AttributeInstanceService( .rowsUpdated() fun deleteAttributeInstancesOfTemporalAttribute(entityId: URI, attributeName: String, datasetId: URI?): Mono = - databaseClient.execute( + databaseClient.sql( """ DELETE FROM attribute_instance WHERE temporal_entity_attribute IN ( SELECT id FROM temporal_entity_attribute WHERE @@ -173,7 +172,7 @@ class AttributeInstanceService( .rowsUpdated() fun deleteAllAttributeInstancesOfTemporalAttribute(entityId: URI, attributeName: String): Mono = - databaseClient.execute( + databaseClient.sql( """ DELETE FROM attribute_instance WHERE temporal_entity_attribute IN ( SELECT id FROM temporal_entity_attribute WHERE @@ -196,10 +195,10 @@ class AttributeInstanceService( SimplifiedAttributeInstanceResult( temporalEntityAttribute = (row["temporal_entity_attribute"] as UUID?)!!, value = row["value"]!!, - observedAt = - row["time_bucket"]?.let { ZonedDateTime.parse(it.toString()).toInstant().atZone(ZoneOffset.UTC) } - ?: row["observed_at"] - .let { ZonedDateTime.parse(it.toString()).toInstant().atZone(ZoneOffset.UTC) } + observedAt = row["time_bucket"]?.let { + ZonedDateTime.parse(it.toString()).toInstant().atZone(ZoneOffset.UTC) + } ?: row["observed_at"] + .let { ZonedDateTime.parse(it.toString()).toInstant().atZone(ZoneOffset.UTC) } ) else FullAttributeInstanceResult( temporalEntityAttribute = (row["temporal_entity_attribute"] as UUID?)!!, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt index f995226eb..b86513b95 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt @@ -275,12 +275,11 @@ class EntityEventListenerService( observedAt = attributeMetadata.observedAt, measuredValue = attributeMetadata.measuredValue, value = attributeMetadata.value, - payload = - extractAttributeInstanceFromCompactedEntity( - compactedJsonLdEntity, - compactTerm(expandedAttributeName, contexts), - attributeMetadata.datasetId - ) + payload = extractAttributeInstanceFromCompactedEntity( + compactedJsonLdEntity, + compactTerm(expandedAttributeName, contexts), + attributeMetadata.datasetId + ) ) temporalEntityAttributeService.create(temporalEntityAttribute).zipWhen { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityService.kt index 26be299b9..e5c864384 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityService.kt @@ -18,10 +18,12 @@ class EntityService( private val consumer: (ClientCodecConfigurer) -> Unit = { configurer -> configurer.defaultCodecs().enableLoggingRequestDetails(true) } - private var webClient = WebClient.builder() - .exchangeStrategies(ExchangeStrategies.builder().codecs(consumer).build()) - .baseUrl(applicationProperties.entity.serviceUrl.toString()) - .build() + private val webClient by lazy { + WebClient.builder() + .exchangeStrategies(ExchangeStrategies.builder().codecs(consumer).build()) + .baseUrl(applicationProperties.entity.serviceUrl.toString()) + .build() + } fun getEntityById(entityId: URI, bearerToken: String): Mono = webClient.get() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt index 401fc0499..f2fc64d64 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt @@ -23,8 +23,11 @@ import com.egm.stellio.shared.util.toUri import io.r2dbc.postgresql.codec.Json import io.r2dbc.spi.Row import org.slf4j.LoggerFactory -import org.springframework.data.r2dbc.core.DatabaseClient -import org.springframework.data.r2dbc.core.bind +import org.springframework.data.r2dbc.core.R2dbcEntityTemplate +import org.springframework.data.relational.core.query.Criteria.where +import org.springframework.data.relational.core.query.Query.query +import org.springframework.r2dbc.core.DatabaseClient +import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import reactor.core.publisher.Flux @@ -35,6 +38,7 @@ import java.util.UUID @Service class TemporalEntityAttributeService( private val databaseClient: DatabaseClient, + private val r2dbcEntityTemplate: R2dbcEntityTemplate, private val attributeInstanceService: AttributeInstanceService, private val applicationProperties: ApplicationProperties ) { @@ -43,7 +47,7 @@ class TemporalEntityAttributeService( @Transactional fun create(temporalEntityAttribute: TemporalEntityAttribute): Mono = - databaseClient.execute( + databaseClient.sql( """ INSERT INTO temporal_entity_attribute (id, entity_id, type, attribute_name, attribute_type, attribute_value_type, dataset_id) @@ -62,7 +66,7 @@ class TemporalEntityAttributeService( internal fun createEntityPayload(entityId: URI, entityPayload: String?): Mono = if (applicationProperties.entity.storePayloads) - databaseClient.execute( + databaseClient.sql( """ INSERT INTO entity_payload (entity_id, payload) VALUES (:entity_id, :payload) @@ -77,7 +81,7 @@ class TemporalEntityAttributeService( fun updateEntityPayload(entityId: URI, payload: String): Mono = if (applicationProperties.entity.storePayloads) - databaseClient.execute("UPDATE entity_payload SET payload = :payload WHERE entity_id = :entity_id") + databaseClient.sql("UPDATE entity_payload SET payload = :payload WHERE entity_id = :entity_id") .bind("payload", Json.of(payload)) .bind("entity_id", entityId) .fetch() @@ -86,7 +90,7 @@ class TemporalEntityAttributeService( Mono.just(1) fun deleteEntityPayload(entityId: URI): Mono = - databaseClient.execute("DELETE FROM entity_payload WHERE entity_id = :entity_id") + databaseClient.sql("DELETE FROM entity_payload WHERE entity_id = :entity_id") .bind("entity_id", entityId) .fetch() .rowsUpdated() @@ -132,12 +136,11 @@ class TemporalEntityAttributeService( observedAt = attributeMetadata.observedAt, measuredValue = attributeMetadata.measuredValue, value = attributeMetadata.value, - payload = - extractAttributeInstanceFromCompactedEntity( - parsedPayload, - compactTerm(expandedAttributeName, contexts), - attributeMetadata.datasetId - ) + payload = extractAttributeInstanceFromCompactedEntity( + parsedPayload, + compactTerm(expandedAttributeName, contexts), + attributeMetadata.datasetId + ) ) Pair(temporalEntityAttribute, attributeInstance) @@ -161,15 +164,14 @@ class TemporalEntityAttributeService( .map { it.t1 + it.t2 } fun deleteTemporalAttributesOfEntity(entityId: URI): Mono = - databaseClient.execute("DELETE FROM temporal_entity_attribute WHERE entity_id = :entity_id") - .bind("entity_id", entityId) - .fetch() - .rowsUpdated() + r2dbcEntityTemplate.delete(TemporalEntityAttribute::class.java) + .matching(query(where("entity_id").`is`(entityId))) + .all() fun deleteTemporalAttributeReferences(entityId: URI, attributeName: String, datasetId: URI?): Mono = attributeInstanceService.deleteAttributeInstancesOfTemporalAttribute(entityId, attributeName, datasetId) .zipWith( - databaseClient.execute( + databaseClient.sql( """ delete FROM temporal_entity_attribute WHERE entity_id = :entity_id @@ -191,7 +193,7 @@ class TemporalEntityAttributeService( fun deleteTemporalAttributeAllInstancesReferences(entityId: URI, attributeName: String): Mono = attributeInstanceService.deleteAllAttributeInstancesOfTemporalAttribute(entityId, attributeName) .zipWith( - databaseClient.execute( + databaseClient.sql( """ delete FROM temporal_entity_attribute WHERE entity_id = :entity_id @@ -246,34 +248,34 @@ class TemporalEntityAttributeService( fun getForEntities(ids: Set, types: Set, attrs: Set, withEntityPayload: Boolean = false): Mono> { - var selectQuery = if (withEntityPayload) - """ + var selectQuery = if (withEntityPayload) + """ SELECT id, temporal_entity_attribute.entity_id, type, attribute_name, attribute_type, attribute_value_type, payload::TEXT, dataset_id FROM temporal_entity_attribute LEFT JOIN entity_payload ON entity_payload.entity_id = temporal_entity_attribute.entity_id WHERE - """.trimIndent() - else - """ + """.trimIndent() + else + """ SELECT id, entity_id, type, attribute_name, attribute_type, attribute_value_type, dataset_id FROM temporal_entity_attribute WHERE - """.trimIndent() - - val formattedIds = ids.joinToString(",") { "'$it'" } - val formattedTypes = types.joinToString(",") { "'$it'" } - val formattedAttrs = attrs.joinToString(",") { "'$it'" } - if (ids.isNotEmpty()) selectQuery = "$selectQuery entity_id in ($formattedIds) AND" - if (types.isNotEmpty()) selectQuery = "$selectQuery type in ($formattedTypes) AND" - if (attrs.isNotEmpty()) selectQuery = "$selectQuery attribute_name in ($formattedAttrs) AND" - return databaseClient - .execute(selectQuery.removeSuffix("AND")) - .fetch() - .all() - .map { rowToTemporalEntityAttribute(it) } - .collectList() - } + """.trimIndent() + + val formattedIds = ids.joinToString(",") { "'$it'" } + val formattedTypes = types.joinToString(",") { "'$it'" } + val formattedAttrs = attrs.joinToString(",") { "'$it'" } + if (ids.isNotEmpty()) selectQuery = "$selectQuery entity_id in ($formattedIds) AND" + if (types.isNotEmpty()) selectQuery = "$selectQuery type in ($formattedTypes) AND" + if (attrs.isNotEmpty()) selectQuery = "$selectQuery attribute_name in ($formattedAttrs) AND" + return databaseClient + .sql(selectQuery.removeSuffix("AND")) + .fetch() + .all() + .map { rowToTemporalEntityAttribute(it) } + .collectList() + } fun getForEntity(id: URI, attrs: Set, withEntityPayload: Boolean = false): Flux { val selectQuery = if (withEntityPayload) @@ -300,7 +302,7 @@ class TemporalEntityAttributeService( selectQuery return databaseClient - .execute(finalQuery) + .sql(finalQuery) .bind("entity_id", id) .fetch() .all() @@ -316,7 +318,7 @@ class TemporalEntityAttributeService( """.trimIndent() return databaseClient - .execute(selectQuery) + .sql(selectQuery) .bind("entity_id", id) .map(rowToId) .first() @@ -333,7 +335,7 @@ class TemporalEntityAttributeService( """.trimIndent() return databaseClient - .execute(selectQuery) + .sql(selectQuery) .bind("entity_id", id) .bind("attribute_name", attributeName) .let { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt index b88888fae..797936d66 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt @@ -1,39 +1,56 @@ package com.egm.stellio.search.web -import arrow.core.* -import com.egm.stellio.search.model.TemporalEntityAttribute +import arrow.core.Either +import arrow.core.flatMap +import arrow.core.getOrHandle +import arrow.core.left +import arrow.core.right import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.service.AttributeInstanceService -import com.egm.stellio.search.service.EntityService import com.egm.stellio.search.service.QueryService import com.egm.stellio.search.service.TemporalEntityAttributeService -import com.egm.stellio.shared.model.* -import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.model.BadRequestDataException +import com.egm.stellio.shared.model.BadRequestDataResponse +import com.egm.stellio.shared.util.JSON_LD_CONTENT_TYPE import com.egm.stellio.shared.util.JsonLdUtils.addContextsToEntity -import com.egm.stellio.shared.util.JsonLdUtils.compact import com.egm.stellio.shared.util.JsonLdUtils.compactTerm -import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdFragment import com.egm.stellio.shared.util.JsonLdUtils.expandValueAsMap +import com.egm.stellio.shared.util.JsonUtils import com.egm.stellio.shared.util.JsonUtils.serializeObject +import com.egm.stellio.shared.util.OptionsParamValue +import com.egm.stellio.shared.util.buildGetSuccessResponse +import com.egm.stellio.shared.util.checkAndGetContext +import com.egm.stellio.shared.util.getApplicableMediaType +import com.egm.stellio.shared.util.getContextFromLinkHeaderOrDefault +import com.egm.stellio.shared.util.hasValueInOptionsParam +import com.egm.stellio.shared.util.parseAndExpandRequestParameter +import com.egm.stellio.shared.util.parseTimeParameter +import com.egm.stellio.shared.util.toUri import kotlinx.coroutines.reactive.awaitFirst import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus import org.springframework.http.MediaType import org.springframework.http.ResponseEntity import org.springframework.util.MultiValueMap -import org.springframework.web.bind.annotation.* +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestHeader +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController import reactor.core.publisher.Mono import java.net.URI import java.time.ZonedDateTime -import java.util.* +import java.util.Optional @RestController @RequestMapping("/ngsi-ld/v1/temporal/entities") class TemporalEntityHandler( private val attributeInstanceService: AttributeInstanceService, private val temporalEntityAttributeService: TemporalEntityAttributeService, - private val entityService: EntityService, private val queryService: QueryService ) { @@ -122,35 +139,6 @@ class TemporalEntityHandler( return buildGetSuccessResponse(mediaType, contextLink) .body(serializeObject(addContextsToEntity(temporalEntity, listOf(contextLink), mediaType))) } - - /** - * Get the entity payload from entity service if we don't have it locally (for legacy entries in DB) - */ - private fun loadEntityPayload( - temporalEntityAttribute: TemporalEntityAttribute, - bearerToken: String, - contextLink: String - ): Mono = - when { - temporalEntityAttribute.entityPayload == null -> - entityService.getEntityById(temporalEntityAttribute.entityId, bearerToken) - .doOnSuccess { - val entityPayload = compact(it, contextLink) - temporalEntityAttributeService.updateEntityPayload( - temporalEntityAttribute.entityId, - serializeObject(entityPayload) - ).subscribe() - } - temporalEntityAttribute.type != "https://uri.etsi.org/ngsi-ld/Subscription" -> Mono.just( - expandJsonLdEntity( - temporalEntityAttribute.entityPayload - ) - ) - else -> { - val parsedEntity = expandJsonLdEntity(temporalEntityAttribute.entityPayload) - Mono.just(parsedEntity) - } - } } internal fun buildTemporalQuery(params: MultiValueMap, contextLink: String): TemporalQuery { @@ -209,7 +197,7 @@ internal fun buildTimerelAndTime( Pair(null, null).right() } else if (timerelParam != null && timeParam != null) { val timeRelResult = try { - TemporalQuery.Timerel.valueOf(timerelParam.toUpperCase()).right() + TemporalQuery.Timerel.valueOf(timerelParam.uppercase()).right() } catch (e: IllegalArgumentException) { "'timerel' is not valid, it should be one of 'before', 'between', or 'after'".left() } diff --git a/search-service/src/main/resources/logback-spring.xml b/search-service/src/main/resources/logback-spring.xml index 3306a1151..cf44bbbd9 100644 --- a/search-service/src/main/resources/logback-spring.xml +++ b/search-service/src/main/resources/logback-spring.xml @@ -30,7 +30,7 @@ - + diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/config/TestContainersConfiguration.kt b/search-service/src/test/kotlin/com/egm/stellio/search/config/TestContainersConfiguration.kt deleted file mode 100644 index 1ef3bafea..000000000 --- a/search-service/src/test/kotlin/com/egm/stellio/search/config/TestContainersConfiguration.kt +++ /dev/null @@ -1,45 +0,0 @@ -package com.egm.stellio.search.config - -import com.egm.stellio.shared.TestContainers -import io.r2dbc.spi.ConnectionFactories -import io.r2dbc.spi.ConnectionFactory -import io.r2dbc.spi.ConnectionFactoryOptions -import org.springframework.boot.test.context.TestConfiguration -import org.springframework.context.annotation.Bean - -@TestConfiguration -class TestContainersConfiguration { - - private val DB_NAME = "stellio_search" - private val DB_USER = "stellio_search" - private val DB_PASSWORD = "stellio_search_db_password" - - object SearchServiceTestContainers : TestContainers("postgres", 5432) { - - fun getPostgresqlHost(): String { - return instance.getServiceHost(serviceName, servicePort) - } - - fun getPostgresqlPort(): Int { - return instance.getServicePort(serviceName, servicePort) - } - - fun getPostgresqlUri(): String { - return "jdbc:postgresql://" + getPostgresqlHost() + ":" + getPostgresqlPort() + '/' - } - } - - @Bean - fun connectionFactory(): ConnectionFactory { - val options = ConnectionFactoryOptions.builder() - .option(ConnectionFactoryOptions.DATABASE, DB_NAME) - .option(ConnectionFactoryOptions.HOST, SearchServiceTestContainers.getPostgresqlHost()) - .option(ConnectionFactoryOptions.PORT, SearchServiceTestContainers.getPostgresqlPort()) - .option(ConnectionFactoryOptions.USER, DB_USER) - .option(ConnectionFactoryOptions.PASSWORD, DB_PASSWORD) - .option(ConnectionFactoryOptions.DRIVER, "postgresql") - .build() - - return ConnectionFactories.get(options) - } -} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/config/TimescaleBasedTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/config/TimescaleBasedTests.kt deleted file mode 100644 index 7263ff919..000000000 --- a/search-service/src/test/kotlin/com/egm/stellio/search/config/TimescaleBasedTests.kt +++ /dev/null @@ -1,17 +0,0 @@ -package com.egm.stellio.search.config - -import org.flywaydb.core.Flyway -import org.springframework.context.annotation.Import - -@Import(TestContainersConfiguration::class) -open class TimescaleBasedTests { - - init { - val testContainers = TestContainersConfiguration.SearchServiceTestContainers - testContainers.startContainers() - Flyway.configure() - .dataSource(testContainers.getPostgresqlUri(), "stellio_search", "stellio_search_db_password") - .load() - .migrate() - } -} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/config/WebSecurityTestConfig.kt b/search-service/src/test/kotlin/com/egm/stellio/search/config/WebSecurityTestConfig.kt index 4449057be..6e72bfd76 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/config/WebSecurityTestConfig.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/config/WebSecurityTestConfig.kt @@ -7,7 +7,8 @@ import org.springframework.security.oauth2.jwt.ReactiveJwtDecoder import org.springframework.security.oauth2.jwt.ReactiveJwtDecoders /** - * This configuration class was added since handlers tests (when importing a customSecurityConfiguration) requires defining a bean of type ReactiveJwtDecoder. + * This configuration class was added since handlers tests (when importing a customSecurityConfiguration) + * requires defining a bean of type ReactiveJwtDecoder. */ @TestConfiguration class WebSecurityTestConfig : WebSecurityConfig() { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt index 99cf82df2..e13920b1c 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt @@ -1,11 +1,11 @@ package com.egm.stellio.search.service -import com.egm.stellio.search.config.TimescaleBasedTests import com.egm.stellio.search.model.AttributeInstance import com.egm.stellio.search.model.FullAttributeInstanceResult import com.egm.stellio.search.model.SimplifiedAttributeInstanceResult import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery +import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.util.JsonLdUtils.EGM_OBSERVED_BY import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_KW @@ -13,14 +13,18 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE import com.egm.stellio.shared.util.JsonUtils import com.egm.stellio.shared.util.matchContent import com.egm.stellio.shared.util.toUri -import io.mockk.* +import io.mockk.confirmVerified +import io.mockk.spyk +import io.mockk.verify import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.data.r2dbc.core.DatabaseClient +import org.springframework.data.r2dbc.core.R2dbcEntityTemplate +import org.springframework.data.r2dbc.core.insert +import org.springframework.r2dbc.core.DatabaseClient import org.springframework.test.context.ActiveProfiles import reactor.test.StepVerifier import java.time.Instant @@ -30,7 +34,7 @@ import kotlin.random.Random @SpringBootTest @ActiveProfiles("test") -class AttributeInstanceServiceTests : TimescaleBasedTests() { +class AttributeInstanceServiceTests : WithTimescaleContainer { @Autowired private lateinit var attributeInstanceService: AttributeInstanceService @@ -38,6 +42,9 @@ class AttributeInstanceServiceTests : TimescaleBasedTests() { @Autowired private lateinit var databaseClient: DatabaseClient + @Autowired + private lateinit var r2dbcEntityTemplate: R2dbcEntityTemplate + private val now = Instant.now().atZone(ZoneOffset.UTC) private lateinit var temporalEntityAttribute: TemporalEntityAttribute @@ -53,8 +60,7 @@ class AttributeInstanceServiceTests : TimescaleBasedTests() { attributeValueType = TemporalEntityAttribute.AttributeValueType.MEASURE ) - databaseClient.insert() - .into(TemporalEntityAttribute::class.java) + r2dbcEntityTemplate.insert() .using(temporalEntityAttribute) .then() .block() @@ -62,10 +68,8 @@ class AttributeInstanceServiceTests : TimescaleBasedTests() { @AfterEach fun clearPreviousObservations() { - databaseClient.delete() - .from("attribute_instance") - .fetch() - .rowsUpdated() + r2dbcEntityTemplate.delete(AttributeInstance::class.java) + .all() .block() } @@ -133,8 +137,7 @@ class AttributeInstanceServiceTests : TimescaleBasedTests() { attributeName = "propWithStringValue", attributeValueType = TemporalEntityAttribute.AttributeValueType.ANY ) - databaseClient.insert() - .into(TemporalEntityAttribute::class.java) + r2dbcEntityTemplate.insert() .using(temporalEntityAttribute2) .then() .block() @@ -278,8 +281,7 @@ class AttributeInstanceServiceTests : TimescaleBasedTests() { attributeValueType = TemporalEntityAttribute.AttributeValueType.MEASURE ) - databaseClient.insert() - .into(TemporalEntityAttribute::class.java) + r2dbcEntityTemplate.insert() .using(temporalEntityAttribute2) .then() .block() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/EntityServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/EntityServiceTests.kt index 7dd225dc5..6ef74abc0 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/EntityServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/EntityServiceTests.kt @@ -1,5 +1,6 @@ package com.egm.stellio.search.service +import com.egm.stellio.search.config.ApplicationProperties import com.egm.stellio.shared.util.loadSampleData import com.egm.stellio.shared.util.toUri import com.github.tomakehurst.wiremock.WireMockServer @@ -13,6 +14,8 @@ import com.github.tomakehurst.wiremock.client.WireMock.urlMatching import com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo import com.github.tomakehurst.wiremock.client.WireMock.verify import com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig +import com.ninjasquad.springmockk.MockkBean +import io.mockk.every import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test @@ -21,23 +24,23 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles import reactor.test.StepVerifier -// TODO : it should be possible to have this test not depend on the whole application -// (ie to only load the target service) -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) - +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [EntityService::class]) @ActiveProfiles("test") class EntityServiceTests { @Autowired private lateinit var entityService: EntityService + @MockkBean + private lateinit var applicationProperties: ApplicationProperties + private lateinit var wireMockServer: WireMockServer @BeforeAll fun beforeAll() { wireMockServer = WireMockServer(wireMockConfig().port(8089)) wireMockServer.start() - // If not using the default port, we need to instruct explicitely the client (quite redundant) + // If not using the default port, we need to instruct explicitly the client (quite redundant) configureFor(8089) } @@ -54,6 +57,10 @@ class EntityServiceTests { .willReturn(okJson(loadSampleData("beehive.jsonld"))) ) + every { + applicationProperties.entity + } returns ApplicationProperties.Entity("http://localhost:8089/ngsi-ld/v1".toUri(), false) + val entity = entityService.getEntityById("urn:ngsi-ld:BeeHive:TESTC".toUri(), "Bearer 1234") // verify the steps in getEntityById diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt index 25e844239..9c763cabb 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt @@ -25,7 +25,7 @@ import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.toFlux import java.time.ZonedDateTime -@SpringBootTest +@SpringBootTest(classes = [QueryService::class]) @ActiveProfiles("test") @ExperimentalCoroutinesApi class QueryServiceTests { @@ -61,7 +61,7 @@ class QueryServiceTests { val exception = assertThrows { queryService.parseAndCheckQueryParams(queryParams, APIC_COMPOUND_CONTEXT) } - Assertions.assertEquals( + assertEquals( "'timerel' and 'time' must be used in conjunction", exception.message ) @@ -76,7 +76,7 @@ class QueryServiceTests { val exception = assertThrows { queryService.parseAndCheckQueryParams(queryParams, APIC_COMPOUND_CONTEXT) } - Assertions.assertEquals( + assertEquals( "Either type or attrs need to be present in request parameters", exception.message ) @@ -95,7 +95,7 @@ class QueryServiceTests { val parsedParams = queryService.parseAndCheckQueryParams(queryParams, APIC_COMPOUND_CONTEXT) - Assertions.assertEquals( + assertEquals( parsedParams, mapOf( "ids" to setOf(entityUri, secondEntityUri), diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt index 193fce297..7a3883cd8 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt @@ -1,6 +1,9 @@ package com.egm.stellio.search.service -import com.egm.stellio.search.config.TimescaleBasedTests +import com.egm.stellio.search.model.AttributeInstance +import com.egm.stellio.search.model.EntityPayload +import com.egm.stellio.search.model.TemporalEntityAttribute +import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.util.JsonUtils.deserializeObject import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.loadSampleData @@ -16,7 +19,7 @@ import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.context.SpringBootTest -import org.springframework.data.r2dbc.core.DatabaseClient +import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.test.context.ActiveProfiles import reactor.core.publisher.Mono import reactor.test.StepVerifier @@ -24,7 +27,7 @@ import java.time.ZonedDateTime @SpringBootTest @ActiveProfiles("test") -class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { +class TemporalEntityAttributeServiceTests : WithTimescaleContainer { @Autowired @SpykBean @@ -34,7 +37,7 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { private lateinit var attributeInstanceService: AttributeInstanceService @Autowired - private lateinit var databaseClient: DatabaseClient + private lateinit var r2dbcEntityTemplate: R2dbcEntityTemplate @Value("\${application.jsonld.apic_context}") val apicContext: String? = null @@ -70,22 +73,16 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { @AfterEach fun clearPreviousTemporalEntityAttributesAndObservations() { - databaseClient.delete() - .from("entity_payload") - .fetch() - .rowsUpdated() + r2dbcEntityTemplate.delete(EntityPayload::class.java) + .all() .block() - databaseClient.delete() - .from("attribute_instance") - .fetch() - .rowsUpdated() + r2dbcEntityTemplate.delete(AttributeInstance::class.java) + .all() .block() - databaseClient.delete() - .from("temporal_entity_attribute") - .fetch() - .rowsUpdated() + r2dbcEntityTemplate.delete(TemporalEntityAttribute::class.java) + .all() .block() } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt index b3ee60edf..537d5bab4 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt @@ -21,7 +21,7 @@ import java.time.ZoneOffset import java.time.ZonedDateTime import java.util.UUID -@SpringBootTest +@SpringBootTest(classes = [TemporalEntityService::class]) @ActiveProfiles("test") class TemporalEntityServiceTests { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt new file mode 100644 index 000000000..f928ccf8f --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt @@ -0,0 +1,45 @@ +package com.egm.stellio.search.support + +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers +import org.testcontainers.utility.DockerImageName + +@Testcontainers +interface WithTimescaleContainer { + + companion object { + + private const val DB_NAME = "stellio_search" + private const val DB_USER = "stellio_search" + private const val DB_PASSWORD = "stellio_search_db_password" + + private val timescaleImage: DockerImageName = + DockerImageName.parse("stellio/stellio-timescale-postgis:1.7.2-pg11") + .asCompatibleSubstituteFor("postgres") + + @Container + val timescaleContainer = PostgreSQLContainer(timescaleImage).apply { + withEnv("POSTGRES_PASSWORD", "password") + withEnv("POSTGRES_MULTIPLE_DATABASES", "$DB_NAME,$DB_USER,$DB_PASSWORD") + } + + @JvmStatic + @DynamicPropertySource + fun properties(registry: DynamicPropertyRegistry) { + val containerAddress = "${timescaleContainer.containerIpAddress}:${timescaleContainer.firstMappedPort}" + registry.add("spring.r2dbc.url") { "r2dbc:postgresql://$containerAddress/$DB_NAME" } + registry.add("spring.r2dbc.username") { DB_USER } + registry.add("spring.r2dbc.password") { DB_PASSWORD } + registry.add("spring.flyway.url") { "jdbc:postgresql://$containerAddress/$DB_NAME" } + registry.add("spring.flyway.user") { DB_USER } + registry.add("spring.flyway.password") { DB_PASSWORD } + } + + init { + timescaleContainer.start() + } + } +} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index 54ab657df..67d7a27dc 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -5,16 +5,26 @@ import com.egm.stellio.search.model.SimplifiedAttributeInstanceResult import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.service.AttributeInstanceService -import com.egm.stellio.search.service.EntityService import com.egm.stellio.search.service.QueryService import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.ResourceNotFoundException -import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT +import com.egm.stellio.shared.util.JSON_LD_MEDIA_TYPE import com.egm.stellio.shared.util.JsonUtils.deserializeObject +import com.egm.stellio.shared.util.buildContextLinkHeader +import com.egm.stellio.shared.util.entityNotFoundMessage +import com.egm.stellio.shared.util.loadSampleData +import com.egm.stellio.shared.util.toUri import com.ninjasquad.springmockk.MockkBean -import io.mockk.* -import org.junit.jupiter.api.Assertions.* +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.confirmVerified +import io.mockk.every +import io.mockk.verify +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -60,9 +70,6 @@ class TemporalEntityHandlerTests { @MockkBean(relaxed = true) private lateinit var temporalEntityAttributeService: TemporalEntityAttributeService - @MockkBean - private lateinit var entityService: EntityService - private val entityUri = "urn:ngsi-ld:BeeHive:TESTC".toUri() @BeforeAll diff --git a/search-service/src/test/resources/application-test.properties b/search-service/src/test/resources/application-test.properties index 970ee63b1..8c2767c63 100644 --- a/search-service/src/test/resources/application-test.properties +++ b/search-service/src/test/resources/application-test.properties @@ -1,13 +1,3 @@ -logging.level.org.springframework.kafka = WARN -logging.level.org.apache.kafka = WARN -logging.level.org.apache.zookeeper = WARN -logging.level.kafka = WARN - application.jsonld.apic_context=https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld -application.entity.service-url = http://localhost:8089/ngsi-ld/v1 application.authentication.enabled = true - -# Disable Flyway here as migrations are called programmatically in test classes to perform dynamic binding to -# the running PG container -spring.flyway.enabled = false diff --git a/search-service/src/test/resources/logback-test.xml b/search-service/src/test/resources/logback-test.xml new file mode 100644 index 000000000..b70446836 --- /dev/null +++ b/search-service/src/test/resources/logback-test.xml @@ -0,0 +1,28 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} %M - %msg%n + + + + + build/tests.log + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + + diff --git a/shared/build.gradle.kts b/shared/build.gradle.kts index 60779c0bc..29cec4691 100644 --- a/shared/build.gradle.kts +++ b/shared/build.gradle.kts @@ -16,7 +16,6 @@ the().apply { } dependencies { - testFixturesImplementation("org.testcontainers:testcontainers") testFixturesImplementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") testFixturesImplementation("com.fasterxml.jackson.module:jackson-module-kotlin") testFixturesImplementation("org.springframework:spring-core") @@ -24,7 +23,6 @@ dependencies { testFixturesImplementation("org.springframework.security:spring-security-test") testFixturesImplementation("org.springframework.boot:spring-boot-starter-oauth2-resource-server") testFixturesImplementation("org.springframework.boot:spring-boot-starter-test") { - exclude(group = "org.junit.vintage", module = "junit-vintage-engine") // to ensure we are using mocks and spies from springmockk lib instead exclude(module = "mockito-core") } diff --git a/shared/config/detekt/baseline.xml b/shared/config/detekt/baseline.xml index 8c0f37e73..45883bef1 100644 --- a/shared/config/detekt/baseline.xml +++ b/shared/config/detekt/baseline.xml @@ -8,5 +8,6 @@ SpreadOperator:EntityEvent.kt$EntityEvent$( *[ JsonSubTypes.Type(value = EntityCreateEvent::class), JsonSubTypes.Type(value = EntityReplaceEvent::class), JsonSubTypes.Type(value = EntityUpdateEvent::class), JsonSubTypes.Type(value = EntityDeleteEvent::class), JsonSubTypes.Type(value = AttributeAppendEvent::class), JsonSubTypes.Type(value = AttributeReplaceEvent::class), JsonSubTypes.Type(value = AttributeUpdateEvent::class), JsonSubTypes.Type(value = AttributeDeleteEvent::class), JsonSubTypes.Type(value = AttributeDeleteAllInstancesEvent::class) ] ) TooGenericExceptionCaught:HttpUtils.kt$HttpUtils$e: Exception TooManyFunctions:JsonLdUtils.kt$JsonLdUtils$JsonLdUtils + UnusedPrivateMember:JsonLdUtils.kt$JsonLdUtils$@PostConstruct private fun loadCoreContext() diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/NgsiLdEntity.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/NgsiLdEntity.kt index e3fa33dc7..314ec78ec 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/NgsiLdEntity.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/NgsiLdEntity.kt @@ -278,25 +278,25 @@ class NgsiLdGeoPropertyInstance( // TODO this lacks sanity checks private fun extractCoordinates(geoPropertyType: GeoPropertyType, geoPropertyValue: Map>): List { - val coordinates = geoPropertyValue[NGSILD_COORDINATES_PROPERTY]!! - if (geoPropertyType == GeoPropertyType.Point) { - val longitude = (coordinates[0] as Map)["@value"]!! - val latitude = (coordinates[1] as Map)["@value"]!! - return listOf(longitude, latitude) - } else { - val res = arrayListOf>() - var count = 1 - coordinates.forEach { - if (count % 2 != 0) { - val longitude = (coordinates[count - 1] as Map)["@value"]!! - val latitude = (coordinates[count] as Map)["@value"]!! - res.add(listOf(longitude, latitude)) - } - count++ + val coordinates = geoPropertyValue[NGSILD_COORDINATES_PROPERTY]!! + if (geoPropertyType == GeoPropertyType.Point) { + val longitude = (coordinates[0] as Map)["@value"]!! + val latitude = (coordinates[1] as Map)["@value"]!! + return listOf(longitude, latitude) + } else { + val res = arrayListOf>() + var count = 1 + coordinates.forEach { + if (count % 2 != 0) { + val longitude = (coordinates[count - 1] as Map)["@value"]!! + val latitude = (coordinates[count] as Map)["@value"]!! + res.add(listOf(longitude, latitude)) } - return res + count++ } + return res } + } fun toWktFormat(geoPropertyType: GeoPropertyType, coordinates: List): String { return if (geoPropertyType == GeoPropertyType.Point) { diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index 2554870ed..bbcff11c9 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -265,7 +265,7 @@ object JsonLdUtils { val expandedType = JsonLdProcessor.expand(mapOf(type to mapOf()), jsonLdOptions) logger.debug("Expanded type $type to $expandedType") return if (expandedType.isNotEmpty()) - (expandedType[0] as Map).keys.first() + (expandedType[0] as Map).keys.first() else null } @@ -442,10 +442,10 @@ object JsonLdUtils { fun parseAndExpandAttributeFragment(attributeName: String, attributePayload: String, contexts: List): Map>>> = - expandJsonLdFragment( - serializeObject(mapOf(attributeName to deserializeAs(attributePayload))), - contexts - ) as Map>>> + expandJsonLdFragment( + serializeObject(mapOf(attributeName to deserializeAs(attributePayload))), + contexts + ) as Map>>> fun reconstructPolygonCoordinates(compactedJsonLdEntity: MutableMap) = compactedJsonLdEntity @@ -513,7 +513,7 @@ fun parseAndExpandJsonLdFragment(fragment: String, jsonLdOptions: JsonLdOptions? else JsonLdProcessor.expand(parsedFragment) } catch (e: JsonLdError) { - throw BadRequestDataException("Unexpected error while parsing payload : ${e.message}") + throw BadRequestDataException("Unexpected error while parsing payload (cause was: $e)") } if (expandedFragment.isEmpty()) throw BadRequestDataException("Unable to parse input payload") diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/UriUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/UriUtils.kt index a000de8ea..f0b63b5b6 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/UriUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/UriUtils.kt @@ -12,7 +12,7 @@ fun String.toUri(): URI = uri } catch (e: URISyntaxException) { throw BadRequestDataException( - "The supplied identifier was expected to be an URI but it is not: $this (${e.message})" + "The supplied identifier was expected to be an URI but it is not: $this (cause was: $e)" ) } diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt index 41404e84c..def19d2e8 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt @@ -157,7 +157,8 @@ class JsonLdUtilsTests { parseAndExpandJsonLdFragment(rawEntity) } assertEquals( - "Unexpected error while parsing payload : loading remote context failed: unknownContext", + "Unexpected error while parsing payload " + + "(cause was: com.github.jsonldjava.core.JsonLdError: loading remote context failed: unknownContext)", exception.message ) } diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/util/UriUtilsTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/util/UriUtilsTests.kt new file mode 100644 index 000000000..25cc3e15a --- /dev/null +++ b/shared/src/test/kotlin/com/egm/stellio/shared/util/UriUtilsTests.kt @@ -0,0 +1,37 @@ +package com.egm.stellio.shared.util + +import com.egm.stellio.shared.model.BadRequestDataException +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class UriUtilsTests { + + @Test + fun `it should throw a BadRequestData exception if input string is not an absolute URI`() { + val uri = "justAString" + + val exception = assertThrows { + uri.toUri() + } + Assertions.assertEquals( + "The supplied identifier was expected to be an URI but it is not: justAString", + exception.message + ) + } + + @Test + fun `it should throw a BadRequestData exception if input string has an invalid syntax`() { + val uri = "https://just\\AString" + + val exception = assertThrows { + uri.toUri() + } + Assertions.assertEquals( + "The supplied identifier was expected to be an URI but it is not: https://just\\AString " + + "(cause was: java.net.URISyntaxException: Illegal character in authority at index 8: " + + "https://just\\AString)", + exception.message + ) + } +} diff --git a/shared/src/testFixtures/kotlin/com/egm/stellio/shared/TestContainers.kt b/shared/src/testFixtures/kotlin/com/egm/stellio/shared/TestContainers.kt deleted file mode 100644 index 1250b5d03..000000000 --- a/shared/src/testFixtures/kotlin/com/egm/stellio/shared/TestContainers.kt +++ /dev/null @@ -1,23 +0,0 @@ -package com.egm.stellio.shared - -import org.testcontainers.containers.DockerComposeContainer -import java.io.File - -open class TestContainers( - val serviceName: String, - val servicePort: Int -) { - class KDockerComposeContainer(file: File) : DockerComposeContainer(file) - - private val DOCKER_COMPOSE_FILE = File("docker-compose.yml") - protected val instance: KDockerComposeContainer by lazy { defineDockerCompose() } - - private fun defineDockerCompose() = - KDockerComposeContainer( - DOCKER_COMPOSE_FILE - ).withLocalCompose(true).withExposedService(serviceName, servicePort) - - fun startContainers() { - instance.start() - } -} diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index 55bc2c90a..558fdd6b5 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -14,8 +14,8 @@ dependencies { // required for Flyway's direct access to the DB to apply migration scripts implementation("org.springframework:spring-jdbc") implementation("org.flywaydb:flyway-core") - implementation("com.jayway.jsonpath:json-path:2.5.0") implementation("io.r2dbc:r2dbc-postgresql") + implementation("com.jayway.jsonpath:json-path:2.5.0") implementation(project(":shared")) // firebase SDK implementation("com.google.firebase:firebase-admin:6.12.2") @@ -25,6 +25,8 @@ dependencies { runtimeOnly("org.postgresql:postgresql") testImplementation("com.github.tomakehurst:wiremock-standalone:2.25.1") + testImplementation("org.testcontainers:postgresql") + testImplementation("org.testcontainers:r2dbc") testImplementation(testFixtures(project(":shared"))) } diff --git a/subscription-service/config/detekt/baseline.xml b/subscription-service/config/detekt/baseline.xml index 5ca9bd448..8737f6423 100644 --- a/subscription-service/config/detekt/baseline.xml +++ b/subscription-service/config/detekt/baseline.xml @@ -2,17 +2,21 @@ - LargeClass:SubscriptionServiceTests.kt$SubscriptionServiceTests : TimescaleBasedTests + LargeClass:SubscriptionServiceTests.kt$SubscriptionServiceTests : WithTimescaleContainer LongParameterList:FixtureUtils.kt$( withQueryAndGeoQuery: Pair<Boolean, Boolean> = Pair(true, true), withEndpointInfo: Boolean = true, withNotifParams: Pair<FormatType, List<String>> = Pair(FormatType.NORMALIZED, emptyList()), withModifiedAt: Boolean = false, georel: String = "within", coordinates: Any = "[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]" ) MaxLineLength:SubscriptionService.kt$SubscriptionService$ AND ( string_to_array(watched_attributes, ',') && string_to_array(:updatedAttributes, ',') OR watched_attributes IS NULL ) - MaxLineLength:WebSecurityTestConfig.kt$WebSecurityTestConfig : WebSecurityConfig ReturnCount:QueryUtils.kt$QueryUtils$fun extractGeorelParams(georel: String): Triple<String, String?, String?> ReturnCount:SubscriptionHandler.kt$SubscriptionHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getSubscriptions( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap<String, String>, @RequestParam options: Optional<String> ): ResponseEntity<*> + SwallowedException:ParsingUtils.kt$ParsingUtils$catch (e: Exception) { logger.error("Error while parsing a subscription: ${e.message}", e) throw BadRequestDataException(e.message ?: "Failed to parse subscription") } + SwallowedException:QueryUtils.kt$QueryUtils$catch (e: Exception) { throw BadRequestDataException("Unmatched query since it contains an unknown attribute $it") } + SwallowedException:SubscriptionService.kt$SubscriptionService$catch (e: Exception) { false } + SwallowedException:SubscriptionService.kt$SubscriptionService$catch (e: Exception) { throw BadRequestDataException(e.message ?: "No values provided for the Geometry query attribute") } + SwallowedException:SubscriptionService.kt$SubscriptionService$catch (e: Exception) { throw BadRequestDataException(e.message ?: "No values provided for the Notification attribute") } TooGenericExceptionCaught:FCMInitializer.kt$FCMInitializer$e: Exception TooGenericExceptionCaught:FCMService.kt$FCMService$e: Exception TooGenericExceptionCaught:ParsingUtils.kt$ParsingUtils$e: Exception TooGenericExceptionCaught:QueryUtils.kt$QueryUtils$e: Exception TooGenericExceptionCaught:SubscriptionService.kt$SubscriptionService$e: Exception - TooManyFunctions:SubscriptionService.kt$SubscriptionService$SubscriptionService + TooManyFunctions:SubscriptionService.kt$SubscriptionService diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/model/GeoQuery.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/model/GeoQuery.kt index 671b32260..b0c8b8554 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/model/GeoQuery.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/model/GeoQuery.kt @@ -1,5 +1,8 @@ package com.egm.stellio.subscription.model +import org.springframework.data.relational.core.mapping.Table + +@Table(value = "geometry_query") data class GeoQuery( val georel: String, val geometry: GeometryType, diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt index 57a7c9811..e468a2a01 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt @@ -73,12 +73,13 @@ class NotificationService( request = request.contentType(MediaType.APPLICATION_JSON) return request .bodyValue(serializeObject(notification)) - .exchange() - .doOnError { e -> logger.error("Failed to send notification to $uri : ${e.message}") } - .map { - val success = it.statusCode() == HttpStatus.OK + .exchangeToMono { response -> + val success = response.statusCode() == HttpStatus.OK logger.info("The notification sent has been received with ${if (success) "success" else "failure"}") - Triple(subscription, notification, success) + if (!success) { + logger.error("Failed to send notification to $uri : ${response.statusCode()}") + } + Mono.just(Triple(subscription, notification, success)) } .doOnNext { subscriptionService.updateSubscriptionNotification(it.first, it.second, it.third).subscribe() diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/SubscriptionService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/SubscriptionService.kt index 5f799038e..2e3894cca 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/SubscriptionService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/SubscriptionService.kt @@ -24,10 +24,13 @@ import com.jayway.jsonpath.JsonPath.read import io.r2dbc.postgresql.codec.Json import io.r2dbc.spi.Row import org.slf4j.LoggerFactory -import org.springframework.data.r2dbc.core.DatabaseClient +import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.data.r2dbc.core.bind -import org.springframework.data.relational.core.query.Criteria +import org.springframework.data.relational.core.query.Criteria.where +import org.springframework.data.relational.core.query.Query.query import org.springframework.data.relational.core.query.Update +import org.springframework.r2dbc.core.DatabaseClient +import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import reactor.core.publisher.Flux @@ -41,6 +44,7 @@ import java.time.ZonedDateTime @Component class SubscriptionService( private val databaseClient: DatabaseClient, + private val r2dbcEntityTemplate: R2dbcEntityTemplate, private val subscriptionRepository: SubscriptionRepository ) { @@ -56,7 +60,7 @@ class SubscriptionService( :endpoint_uri, :endpoint_accept, :endpoint_info, :times_sent, :is_active, :sub) """.trimIndent() - return databaseClient.execute(insertStatement) + return databaseClient.sql(insertStatement) .bind("id", subscription.id) .bind("type", subscription.type) .bind("name", subscription.name) @@ -94,7 +98,7 @@ class SubscriptionService( } private fun createEntityInfo(entityInfo: EntityInfo, subscriptionId: URI): Mono = - databaseClient.execute( + databaseClient.sql( """ INSERT INTO entity_info (id, id_pattern, type, subscription_id) VALUES (:id, :id_pattern, :type, :subscription_id) @@ -116,7 +120,7 @@ class SubscriptionService( else -> geoQuery.coordinates } - databaseClient.execute( + databaseClient.sql( """ INSERT INTO geometry_query (georel, geometry, coordinates, subscription_id) VALUES (:georel, :geometry, :coordinates, :subscription_id) @@ -145,7 +149,7 @@ class SubscriptionService( WHERE subscription.id = :id """.trimIndent() - return databaseClient.execute(selectStatement) + return databaseClient.sql(selectStatement) .bind("id", id) .map(rowToSubscription) .all() @@ -162,7 +166,7 @@ class SubscriptionService( WHERE id = :id """.trimIndent() - return databaseClient.execute(selectStatement) + return databaseClient.sql(selectStatement) .bind("id", subscriptionId) .map(rowToSub) .first() @@ -224,18 +228,16 @@ class SubscriptionService( value: Any? ): Mono { val updateStatement = Update.update(columnName, value) - return databaseClient.update() - .table("subscription") - .using(updateStatement) - .matching(Criteria.where("id").`is`(subscriptionId)) - .fetch() - .rowsUpdated() + return r2dbcEntityTemplate.update(Subscription::class.java) + .matching(query(where("id").`is`(subscriptionId))) + .apply(updateStatement) .doOnError { e -> throw BadRequestDataException( e.message ?: "Could not update attribute $attributeName" ) } } + fun updateGeometryQuery(subscriptionId: URI, geoQuery: Map): Mono { try { val firstValue = geoQuery.entries.iterator().next() @@ -244,12 +246,11 @@ class SubscriptionService( updateStatement = updateStatement.set(it.key, it.value.toString()) } - return databaseClient.update() - .table("geometry_query") - .using(updateStatement) - .matching(Criteria.where("subscription_id").`is`(subscriptionId)) - .fetch() - .rowsUpdated() + return r2dbcEntityTemplate.update( + query(where("subscription_id").`is`(subscriptionId)), + updateStatement, + GeoQuery::class.java + ) .doOnError { e -> throw BadRequestDataException(e.message ?: "Could not update the Geometry query") } @@ -275,12 +276,11 @@ class SubscriptionService( } } - return databaseClient.update() - .table("subscription") - .using(updateStatement) - .matching(Criteria.where("id").`is`(subscriptionId)) - .fetch() - .rowsUpdated() + return r2dbcEntityTemplate.update( + query(where("id").`is`(subscriptionId)), + updateStatement, + Subscription::class.java + ) .doOnError { e -> throw BadRequestDataException(e.message ?: "Could not update the Notification") } @@ -338,25 +338,17 @@ class SubscriptionService( } } - fun delete(subscriptionId: URI): Mono { - val deleteStatement = - """ - DELETE FROM subscription - WHERE subscription.id = :id - """.trimIndent() - - return databaseClient.execute(deleteStatement) - .bind("id", subscriptionId) - .fetch() - .rowsUpdated() - } + fun delete(subscriptionId: URI): Mono = + r2dbcEntityTemplate.delete( + query(where("id").`is`(subscriptionId)), + Subscription::class.java + ) fun deleteEntityInfo(subscriptionId: URI): Mono = - databaseClient.delete() - .from("entity_info") - .matching(Criteria.where("subscription_id").`is`(subscriptionId)) - .fetch() - .rowsUpdated() + r2dbcEntityTemplate.delete( + query(where("subscription_id").`is`(subscriptionId)), + EntityInfo::class.java + ) fun getSubscriptions(limit: Int, offset: Int, sub: String): Flux { val selectStatement = @@ -378,7 +370,7 @@ class SubscriptionService( limit :limit offset :offset) """.trimIndent() - return databaseClient.execute(selectStatement) + return databaseClient.sql(selectStatement) .bind("limit", limit) .bind("offset", offset) .bind("sub", sub) @@ -400,7 +392,7 @@ class SubscriptionService( SELECT count(*) from subscription WHERE subscription.sub = :sub """.trimIndent() - return databaseClient.execute(selectStatement) + return databaseClient.sql(selectStatement) .bind("sub", sub) .map(rowToSubscriptionCount) .first() @@ -422,7 +414,7 @@ class SubscriptionService( AND (entity_info.id_pattern IS NULL OR :id ~ entity_info.id_pattern) ) """.trimIndent() - return databaseClient.execute(selectStatement) + return databaseClient.sql(selectStatement) .bind("id", id) .bind("type", type) .bind("updatedAttributes", updatedAttributes) @@ -471,14 +463,14 @@ class SubscriptionService( FROM geometry_query WHERE subscription_id = :sub_id """.trimIndent() - return databaseClient.execute(selectStatement) + return databaseClient.sql(selectStatement) .bind("sub_id", subscriptionId) .map(rowToGeoQuery) .first() } fun runGeoQueryStatement(geoQueryStatement: String): Mono { - return databaseClient.execute(geoQueryStatement.trimIndent()) + return databaseClient.sql(geoQueryStatement.trimIndent()) .map(matchesGeoQuery) .first() } @@ -496,12 +488,11 @@ class SubscriptionService( .set("last_notification", notification.notifiedAt) .set(lastStatusName, notification.notifiedAt) - return databaseClient.update() - .table("subscription") - .using(updateStatement) - .matching(Criteria.where("id").`is`(subscription.id)) - .fetch() - .rowsUpdated() + return r2dbcEntityTemplate.update( + query(where("id").`is`(subscription.id)), + updateStatement, + Subscription::class.java + ) } private var rowToSubscription: ((Row) -> Subscription) = { row -> diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt index 9e7d8f28d..5fcd130de 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt @@ -71,7 +71,7 @@ object ParsingUtils { fun String.toSqlColumnName(): String = this.map { - if (it.isUpperCase()) "_${it.toLowerCase()}" + if (it.isUpperCase()) "_${it.lowercase()}" else it }.joinToString("") diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/QueryUtils.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/QueryUtils.kt index de468bcdf..d3b330bc9 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/QueryUtils.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/QueryUtils.kt @@ -7,7 +7,6 @@ import com.egm.stellio.subscription.model.GeoQuery import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import org.slf4j.LoggerFactory object QueryUtils { @@ -18,8 +17,6 @@ object QueryUtils { const val PROPERTY_TYPE = "\"Property\"" const val RELATIONSHIP_TYPE = "\"Relationship\"" - private val logger = LoggerFactory.getLogger(javaClass) - /** * This method transforms a subscription query as per clause 4.9 to new query format supported by JsonPath. * The query param is subscription related query to be transformed. diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/SubscriptionServiceApplicationTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/SubscriptionServiceApplicationTests.kt deleted file mode 100644 index 568b888a8..000000000 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/SubscriptionServiceApplicationTests.kt +++ /dev/null @@ -1,13 +0,0 @@ -package com.egm.stellio.subscription - -import org.junit.jupiter.api.Test -import org.springframework.boot.test.context.SpringBootTest - -@SpringBootTest -class SubscriptionServiceApplicationTests { - - @Test - @Suppress("EmptyFunctionBlock") - fun contextLoads() { - } -} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TestContainersConfiguration.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TestContainersConfiguration.kt deleted file mode 100644 index 828d68ea8..000000000 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TestContainersConfiguration.kt +++ /dev/null @@ -1,45 +0,0 @@ -package com.egm.stellio.subscription.config - -import com.egm.stellio.shared.TestContainers -import io.r2dbc.spi.ConnectionFactories -import io.r2dbc.spi.ConnectionFactory -import io.r2dbc.spi.ConnectionFactoryOptions -import org.springframework.boot.test.context.TestConfiguration -import org.springframework.context.annotation.Bean - -@TestConfiguration -class TestContainersConfiguration { - - private val DB_NAME = "stellio_subscription" - private val DB_USER = "stellio_subscription" - private val DB_PASSWORD = "stellio_subscription_db_password" - - object SubscriptionServiceTestContainers : TestContainers("postgres", 5432) { - - fun getPostgresqlHost(): String { - return instance.getServiceHost(serviceName, servicePort) - } - - fun getPostgresqlPort(): Int { - return instance.getServicePort(serviceName, servicePort) - } - - fun getPostgresqlUri(): String { - return "jdbc:postgresql://" + getPostgresqlHost() + ":" + getPostgresqlPort() + '/' - } - } - - @Bean - fun connectionFactory(): ConnectionFactory { - val options = ConnectionFactoryOptions.builder() - .option(ConnectionFactoryOptions.DATABASE, DB_NAME) - .option(ConnectionFactoryOptions.HOST, SubscriptionServiceTestContainers.getPostgresqlHost()) - .option(ConnectionFactoryOptions.PORT, SubscriptionServiceTestContainers.getPostgresqlPort()) - .option(ConnectionFactoryOptions.USER, DB_USER) - .option(ConnectionFactoryOptions.PASSWORD, DB_PASSWORD) - .option(ConnectionFactoryOptions.DRIVER, "postgresql") - .build() - - return ConnectionFactories.get(options) - } -} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TimescaleBasedTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TimescaleBasedTests.kt deleted file mode 100644 index c8be76b71..000000000 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/TimescaleBasedTests.kt +++ /dev/null @@ -1,17 +0,0 @@ -package com.egm.stellio.subscription.config - -import org.flywaydb.core.Flyway -import org.springframework.context.annotation.Import - -@Import(TestContainersConfiguration::class) -open class TimescaleBasedTests { - - init { - val testContainers = TestContainersConfiguration.SubscriptionServiceTestContainers - testContainers.startContainers() - Flyway.configure() - .dataSource(testContainers.getPostgresqlUri(), "stellio_subscription", "stellio_subscription_db_password") - .load() - .migrate() - } -} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/WebSecurityTestConfig.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/WebSecurityTestConfig.kt index 348f97ab5..c09af11d4 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/WebSecurityTestConfig.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/config/WebSecurityTestConfig.kt @@ -7,7 +7,8 @@ import org.springframework.security.oauth2.jwt.ReactiveJwtDecoder import org.springframework.security.oauth2.jwt.ReactiveJwtDecoders /** - * This configuration class was added since handlers tests (when importing a customSecurityConfiguration) requires defining a bean of type ReactiveJwtDecoder. + * This configuration class was added since handlers tests (when importing a customSecurityConfiguration) + * requires defining a bean of type ReactiveJwtDecoder. */ @TestConfiguration class WebSecurityTestConfig : WebSecurityConfig() { diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt index 391507945..0fddbab10 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt @@ -1,7 +1,6 @@ package com.egm.stellio.subscription.service import com.egm.stellio.shared.model.* -import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.EGM_BASE_CONTEXT_URL import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CORE_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_EGM_CONTEXT @@ -42,7 +41,7 @@ import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.test.StepVerifier -@SpringBootTest +@SpringBootTest(classes = [NotificationService::class]) @ActiveProfiles("test") class NotificationServiceTests { @@ -62,13 +61,13 @@ class NotificationServiceTests { private val apiaryId = "urn:ngsi-ld:Apiary:XYZ01" - private final val contexts = listOf( + private val contexts = listOf( "$EGM_BASE_CONTEXT_URL/apic/jsonld-contexts/apic.jsonld", "$EGM_BASE_CONTEXT_URL/shared-jsonld-contexts/egm.jsonld", NGSILD_CORE_CONTEXT ) - private final val rawEntity = + private val rawEntity = """ { "id":"$apiaryId", @@ -121,7 +120,7 @@ class NotificationServiceTests { fun `it should notify the subscriber and update the subscription`() { val subscription = gimmeRawSubscription() - every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } returns Flux.just(subscription) + every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } answers { Flux.just(subscription) } every { subscriptionService.isMatchingQuery(any(), any()) } answers { true } every { subscriptionService.isMatchingGeoQuery(any(), any()) } answers { Mono.just(true) } every { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } answers { Mono.just(1) } @@ -180,7 +179,7 @@ class NotificationServiceTests { ) ) - every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } returns Flux.just(subscription) + every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } answers { Flux.just(subscription) } every { subscriptionService.isMatchingQuery(any(), any()) } answers { true } every { subscriptionService.isMatchingGeoQuery(any(), any()) } answers { Mono.just(true) } every { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } answers { Mono.just(1) } @@ -213,7 +212,7 @@ class NotificationServiceTests { fun `it should send a simplified payload when format is keyValues and include only the specified attributes`() { val subscription = gimmeRawSubscription(withNotifParams = Pair(FormatType.KEY_VALUES, listOf("location"))) - every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } returns Flux.just(subscription) + every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } answers { Flux.just(subscription) } every { subscriptionService.isMatchingQuery(any(), any()) } answers { true } every { subscriptionService.isMatchingGeoQuery(any(), any()) } answers { Mono.just(true) } every { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } answers { Mono.just(1) } @@ -245,7 +244,7 @@ class NotificationServiceTests { (read(it.operationPayload, "$.data[*]..value") as List).isEmpty() && (read(it.operationPayload, "$.data[*]..object") as List).isEmpty() && (read(it.operationPayload, "$.data[*].excludedProp") as List).isEmpty() && - it.contexts == listOf(NGSILD_EGM_CONTEXT, JsonLdUtils.NGSILD_CORE_CONTEXT) + it.contexts == listOf(NGSILD_EGM_CONTEXT, NGSILD_CORE_CONTEXT) } ) } @@ -269,10 +268,9 @@ class NotificationServiceTests { val subscription1 = gimmeRawSubscription() val subscription2 = gimmeRawSubscription() - every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } returns Flux.just( - subscription1, - subscription2 - ) + every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } answers { + Flux.just(subscription1, subscription2) + } every { subscriptionService.isMatchingQuery(any(), any()) } answers { true } every { subscriptionService.isMatchingGeoQuery(any(), any()) } answers { Mono.just(true) } every { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } answers { Mono.just(1) } @@ -313,10 +311,9 @@ class NotificationServiceTests { val subscription1 = gimmeRawSubscription() val subscription2 = gimmeRawSubscription() - every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } returns Flux.just( - subscription1, - subscription2 - ) + every { subscriptionService.getMatchingSubscriptions(any(), any(), any()) } answers { + Flux.just(subscription1, subscription2) + } every { subscriptionService.isMatchingQuery(any(), any()) } answers { true } every { subscriptionService.isMatchingGeoQuery(subscription1.id, any()) } answers { Mono.just(true) } every { subscriptionService.isMatchingGeoQuery(subscription2.id, any()) } answers { Mono.just(false) } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt index b8e78b76e..4b86af20a 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt @@ -3,11 +3,12 @@ package com.egm.stellio.subscription.service import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.NotImplementedException import com.egm.stellio.shared.model.Notification +import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.matchContent import com.egm.stellio.shared.util.toUri -import com.egm.stellio.subscription.config.TimescaleBasedTests import com.egm.stellio.subscription.model.* import com.egm.stellio.subscription.model.NotificationParams.* +import com.egm.stellio.subscription.support.WithTimescaleContainer import com.egm.stellio.subscription.utils.gimmeRawSubscription import com.ninjasquad.springmockk.SpykBean import io.mockk.verify @@ -15,10 +16,9 @@ import junit.framework.TestCase.assertEquals import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.context.SpringBootTest import org.springframework.core.io.ClassPathResource -import org.springframework.data.r2dbc.core.DatabaseClient +import org.springframework.r2dbc.core.DatabaseClient import org.springframework.test.context.ActiveProfiles import reactor.test.StepVerifier import java.net.URI @@ -27,7 +27,7 @@ import java.time.ZoneOffset @SpringBootTest @ActiveProfiles("test") -class SubscriptionServiceTests : TimescaleBasedTests() { +class SubscriptionServiceTests : WithTimescaleContainer { @Value("\${application.jsonld.apic_context}") val apicContext: String = "" @@ -39,7 +39,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { @SpykBean private lateinit var databaseClient: DatabaseClient - private val MOCK_USER_SUB = "mock-user-sub" + private val mockUserSub = "mock-user-sub" private lateinit var subscription1Id: URI private lateinit var subscription2Id: URI @@ -62,13 +62,13 @@ class SubscriptionServiceTests : TimescaleBasedTests() { } private fun createSubscription(subscription: Subscription): URI { - subscriptionService.create(subscription, MOCK_USER_SUB).block() + subscriptionService.create(subscription, mockUserSub).block() return subscription.id } private fun createSubscription1() { val subscription = gimmeRawSubscription( - withQueryAndGeoQuery = Pair(true, false), + withQueryAndGeoQuery = Pair(first = true, second = false), withEndpointInfo = false, withNotifParams = Pair(FormatType.NORMALIZED, listOf("incoming")) ).copy( @@ -163,7 +163,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { ) ) - val creationResult = subscriptionService.create(subscription, MOCK_USER_SUB) + val creationResult = subscriptionService.create(subscription, mockUserSub) StepVerifier.create(creationResult) .expectNext(2) @@ -172,14 +172,14 @@ class SubscriptionServiceTests : TimescaleBasedTests() { // TODO this is not totally satisfying but well it's a first check that our inserts are triggered verify { - databaseClient.execute( + databaseClient.sql( match { it.startsWith("INSERT INTO subscription") } ) } verify(atLeast = 2) { - databaseClient.execute( + databaseClient.sql( match { """ INSERT INTO entity_info (id, id_pattern, type, subscription_id) @@ -189,7 +189,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { ) } verify(atLeast = 1) { - databaseClient.execute( + databaseClient.sql( match { """ INSERT INTO geometry_query (georel, geometry, coordinates, subscription_id) @@ -217,12 +217,16 @@ class SubscriptionServiceTests : TimescaleBasedTests() { null ) && it.entities.size == 2 && - it.entities.any { - it.type == "Beekeeper" && - it.id == null && - it.idPattern == "urn:ngsi-ld:Beekeeper:1234*" + it.entities.any { entityInfo -> + entityInfo.type == "Beekeeper" && + entityInfo.id == null && + entityInfo.idPattern == "urn:ngsi-ld:Beekeeper:1234*" + } && + it.entities.any { entityInfo -> + entityInfo.type == "Beehive" && + entityInfo.id == null && + entityInfo.idPattern == null } && - it.entities.any { it.type == "Beehive" && it.id == null && it.idPattern == null } && it.geoQ == null } .verifyComplete() @@ -355,7 +359,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { ) val notifiedAt = Instant.now().atZone(ZoneOffset.UTC) - subscriptionService.create(subscription, MOCK_USER_SUB).block() + subscriptionService.create(subscription, mockUserSub).block() subscriptionService.updateSubscriptionNotification( subscription, Notification(subscriptionId = subscription.id, notifiedAt = notifiedAt, data = emptyList()), @@ -377,19 +381,18 @@ class SubscriptionServiceTests : TimescaleBasedTests() { fun `it should delete an existing subscription`() { val subscription = gimmeRawSubscription() - subscriptionService.create(subscription, MOCK_USER_SUB).block() - - val deletionResult = subscriptionService.delete(subscription.id).block() + subscriptionService.create(subscription, mockUserSub).block() - assertEquals(deletionResult, 1) + StepVerifier.create(subscriptionService.delete(subscription.id)) + .expectNext(1) + .verifyComplete() } @Test fun `it should not delete an unknown subscription`() { - val deletionResult = subscriptionService.delete("urn:ngsi-ld:Subscription:UnknownSubscription".toUri()) - .block() - - assertEquals(deletionResult, 0) + StepVerifier.create(subscriptionService.delete("urn:ngsi-ld:Subscription:UnknownSubscription".toUri())) + .expectNext(0) + .verifyComplete() } @Test @@ -525,7 +528,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { watchedAttributes = null ) - subscriptionService.create(subscription, MOCK_USER_SUB).block() + subscriptionService.create(subscription, mockUserSub).block() val persistedSubscription = subscriptionService.getMatchingSubscriptions( @@ -551,7 +554,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { watchedAttributes = listOf("incoming", "outgoing", "temperature") ) - subscriptionService.create(subscription, MOCK_USER_SUB).block() + subscriptionService.create(subscription, mockUserSub).block() val persistedSubscription = subscriptionService.getMatchingSubscriptions( @@ -577,7 +580,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { watchedAttributes = listOf("outgoing", "temperature") ) - subscriptionService.create(subscription, MOCK_USER_SUB).block() + subscriptionService.create(subscription, mockUserSub).block() val persistedSubscription = subscriptionService.getMatchingSubscriptions( @@ -602,7 +605,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { "q" to "foodQuantity>=150", "geoQ" to mapOf("georel" to "equals", "geometry" to "Point", "coordinates" to "[100.0, 0.0]") ), - listOf(apicContext) + listOf(APIC_COMPOUND_CONTEXT) ) subscriptionService.update(subscription4Id, parsedInput).block() @@ -634,7 +637,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { ) ) - subscriptionService.updateNotification(subscription4Id, parsedInput, listOf(apicContext)).block() + subscriptionService.updateNotification(subscription4Id, parsedInput, listOf(APIC_COMPOUND_CONTEXT)).block() val updateResult = subscriptionService.getById(subscription4Id) StepVerifier.create(updateResult) @@ -664,7 +667,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { ) ) - subscriptionService.updateEntities(subscription4Id, parsedInput, listOf(apicContext)).doOnNext { + subscriptionService.updateEntities(subscription4Id, parsedInput, listOf(APIC_COMPOUND_CONTEXT)).doOnNext { val updateResult = subscriptionService.getById(subscription4Id) StepVerifier.create(updateResult) @@ -691,7 +694,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { @Test fun `it should activate a subscription`() { - val parsedInput = Pair(mapOf("isActive" to true), listOf(apicContext)) + val parsedInput = Pair(mapOf("isActive" to true), listOf(APIC_COMPOUND_CONTEXT)) subscriptionService.update(subscription3Id, parsedInput).block() val updateResult = subscriptionService.getById(subscription3Id) @@ -704,7 +707,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { @Test fun `it should deactivate a subscription`() { - val parsedInput = Pair(mapOf("isActive" to false), listOf(apicContext)) + val parsedInput = Pair(mapOf("isActive" to false), listOf(APIC_COMPOUND_CONTEXT)) subscriptionService.update(subscription1Id, parsedInput).block() val updateResult = subscriptionService.getById(subscription1Id) @@ -719,7 +722,7 @@ class SubscriptionServiceTests : TimescaleBasedTests() { @Test fun `it should update a subscription watched attributes`() { val parsedInput = - Pair(mapOf("watchedAttributes" to arrayListOf("incoming", "temperature")), listOf(apicContext)) + Pair(mapOf("watchedAttributes" to arrayListOf("incoming", "temperature")), listOf(APIC_COMPOUND_CONTEXT)) subscriptionService.update(subscription5Id, parsedInput).block() val updateResult = subscriptionService.getById(subscription5Id) @@ -734,22 +737,22 @@ class SubscriptionServiceTests : TimescaleBasedTests() { @Test fun `it should throw a BadRequestData exception if the subscription has an unknown attribute`() { - val parsedInput = Pair(mapOf("unknownAttribute" to "unknownValue"), listOf(apicContext)) + val parsedInput = Pair(mapOf("unknownAttribute" to "unknownValue"), listOf(APIC_COMPOUND_CONTEXT)) StepVerifier.create(subscriptionService.update(subscription5Id, parsedInput)) .expectErrorMatches { throwable -> - throwable.cause is BadRequestDataException + throwable is BadRequestDataException } .verify() } @Test fun `it should throw a NotImplemented exception if the subscription has an unsupported attribute`() { - val parsedInput = Pair(mapOf("throttling" to "someValue"), listOf(apicContext)) + val parsedInput = Pair(mapOf("throttling" to "someValue"), listOf(APIC_COMPOUND_CONTEXT)) StepVerifier.create(subscriptionService.update(subscription5Id, parsedInput)) .expectErrorMatches { throwable -> - throwable.cause is NotImplementedException + throwable is NotImplementedException } .verify() } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt new file mode 100644 index 000000000..7c6530cfc --- /dev/null +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt @@ -0,0 +1,45 @@ +package com.egm.stellio.subscription.support + +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers +import org.testcontainers.utility.DockerImageName + +@Testcontainers +interface WithTimescaleContainer { + + companion object { + + private const val DB_NAME = "stellio_subscription" + private const val DB_USER = "stellio_subscription" + private const val DB_PASSWORD = "stellio_subscription_db_password" + + private val timescaleImage: DockerImageName = + DockerImageName.parse("stellio/stellio-timescale-postgis:1.7.2-pg11") + .asCompatibleSubstituteFor("postgres") + + @Container + val timescaleContainer = PostgreSQLContainer(timescaleImage).apply { + withEnv("POSTGRES_PASSWORD", "password") + withEnv("POSTGRES_MULTIPLE_DATABASES", "$DB_NAME,$DB_USER,$DB_PASSWORD") + } + + @JvmStatic + @DynamicPropertySource + fun properties(registry: DynamicPropertyRegistry) { + val containerAddress = "${timescaleContainer.containerIpAddress}:${timescaleContainer.firstMappedPort}" + registry.add("spring.r2dbc.url") { "r2dbc:postgresql://$containerAddress/$DB_NAME" } + registry.add("spring.r2dbc.username") { DB_USER } + registry.add("spring.r2dbc.password") { DB_PASSWORD } + registry.add("spring.flyway.url") { "jdbc:postgresql://$containerAddress/$DB_NAME" } + registry.add("spring.flyway.user") { DB_USER } + registry.add("spring.flyway.password") { DB_PASSWORD } + } + + init { + timescaleContainer.start() + } + } +} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt index 0194de666..8f7df1aa4 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt @@ -2,6 +2,7 @@ package com.egm.stellio.subscription.web import com.egm.stellio.shared.WithMockCustomUser import com.egm.stellio.shared.model.* +import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.JSON_LD_MEDIA_TYPE import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonUtils.serializeObject @@ -18,7 +19,6 @@ import org.hamcrest.core.Is import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.autoconfigure.web.reactive.AutoConfigureWebTestClient import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest import org.springframework.context.annotation.Import diff --git a/subscription-service/src/test/resources/application-test.properties b/subscription-service/src/test/resources/application-test.properties index bf93653f1..8c2767c63 100644 --- a/subscription-service/src/test/resources/application-test.properties +++ b/subscription-service/src/test/resources/application-test.properties @@ -1,7 +1,3 @@ application.jsonld.apic_context=https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld -# Disable Flyway here as migrations are called programmatically in test classes to perform dynamic binding to -# the running PG container -spring.flyway.enabled = false - application.authentication.enabled = true From c92271bfe4c5eef2c484b0dd5f9332f78ea9359a Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Wed, 14 Jul 2021 09:35:44 +0200 Subject: [PATCH 05/56] chore(common): use common APIC context from testFixtures (instead of redefining in tests config) --- .../TemporalEntityAttributeServiceTests.kt | 45 ++++++++++--------- .../service/TemporalEntityServiceTests.kt | 9 ++-- .../search/web/TemporalEntityHandlerTests.kt | 28 +++++------- .../TemporalEntityOperationsHandlerTests.kt | 10 ++--- .../resources/application-test.properties | 2 - .../service/SubscriptionServiceTests.kt | 3 -- .../web/SubscriptionHandlerTests.kt | 9 ++-- .../resources/application-test.properties | 2 - 8 files changed, 45 insertions(+), 63 deletions(-) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt index 7a3883cd8..b36d7bcb2 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt @@ -4,6 +4,7 @@ import com.egm.stellio.search.model.AttributeInstance import com.egm.stellio.search.model.EntityPayload import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.support.WithTimescaleContainer +import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.JsonUtils.deserializeObject import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.loadSampleData @@ -17,7 +18,6 @@ import io.mockk.verify import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.test.context.ActiveProfiles @@ -39,9 +39,6 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { @Autowired private lateinit var r2dbcEntityTemplate: R2dbcEntityTemplate - @Value("\${application.jsonld.apic_context}") - val apicContext: String? = null - val incomingAttrExpandedName = "https://ontology.eglobalmark.com/apic#incoming" val outgoingAttrExpandedName = "https://ontology.eglobalmark.com/apic#outgoing" @@ -92,7 +89,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() val temporalEntityAttributes = temporalEntityAttributeService.getForEntity( @@ -137,7 +134,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val temporalReferencesResults = temporalEntityAttributeService.createEntityTemporalReferences( rawEntity, - listOf(apicContext!!) + listOf(APIC_COMPOUND_CONTEXT) ) StepVerifier.create(temporalReferencesResults) @@ -172,7 +169,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val temporalReferencesResults = temporalEntityAttributeService.createEntityTemporalReferences( rawEntity, - listOf(apicContext!!) + listOf(APIC_COMPOUND_CONTEXT) ) StepVerifier.create(temporalReferencesResults) @@ -201,7 +198,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val temporalReferencesResults = temporalEntityAttributeService.createEntityTemporalReferences( rawEntity, - listOf(apicContext!!) + listOf(APIC_COMPOUND_CONTEXT) ) StepVerifier.create(temporalReferencesResults) @@ -220,7 +217,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { temporalEntityAttributeService.createEntityTemporalReferences( rawEntity, - listOf(apicContext!!) + listOf(APIC_COMPOUND_CONTEXT) ).block() verify { @@ -250,7 +247,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() val temporalEntityAttributeId = temporalEntityAttributeService.getForEntityAndAttribute( "urn:ngsi-ld:BeeHive:TESTC".toUri(), incomingAttrExpandedName @@ -268,7 +265,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() val temporalEntityAttributeId = temporalEntityAttributeService.getForEntityAndAttribute( "urn:ngsi-ld:BeeHive:TESTC".toUri(), @@ -287,7 +284,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() val temporalEntityAttributeId = temporalEntityAttributeService.getForEntityAndAttribute( "urn:ngsi-ld:BeeHive:TESTC".toUri(), @@ -307,8 +304,10 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(apicContext!!)).block() - temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() + temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( @@ -339,8 +338,10 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(apicContext!!)).block() - temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() + temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( @@ -365,8 +366,10 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(apicContext!!)).block() - temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() + temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( @@ -401,7 +404,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() val deletedRecords = temporalEntityAttributeService.deleteTemporalAttributesOfEntity(entityId).block() @@ -424,7 +427,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() every { attributeInstanceService.deleteAttributeInstancesOfTemporalAttribute(any(), any(), any()) @@ -455,7 +458,7 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { every { attributeInstanceService.create(any()) } returns Mono.just(1) - temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(apicContext!!)).block() + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)).block() every { attributeInstanceService.deleteAllAttributeInstancesOfTemporalAttribute(any(), any()) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt index 537d5bab4..887f7976b 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityServiceTests.kt @@ -1,6 +1,7 @@ package com.egm.stellio.search.service import com.egm.stellio.search.model.* +import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CORE_CONTEXT import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.assertJsonPayloadsAreEqual @@ -12,7 +13,6 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles import java.net.URI @@ -28,9 +28,6 @@ class TemporalEntityServiceTests { @Autowired private lateinit var temporalEntityService: TemporalEntityService - @Value("\${application.jsonld.apic_context}") - val apicContext: String? = null - @Test fun `it should return a temporal entity with an empty array of instances if it has no temporal history`() { val temporalEntityAttribute = TemporalEntityAttribute( @@ -68,7 +65,7 @@ class TemporalEntityServiceTests { "urn:ngsi-ld:BeeHive:TESTC".toUri(), attributeAndResultsMap, TemporalQuery(), - listOf(apicContext!!), + listOf(APIC_COMPOUND_CONTEXT), withTemporalValues ) assertJsonPayloadsAreEqual(expectation, serializeObject(temporalEntity)) @@ -125,7 +122,7 @@ class TemporalEntityServiceTests { val temporalEntity = temporalEntityService.buildTemporalEntities( queryResult, TemporalQuery(), - listOf(apicContext!!), + listOf(APIC_COMPOUND_CONTEXT), withTemporalValues ) assertJsonPayloadsAreEqual(expectation, serializeObject(temporalEntity)) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index 67d7a27dc..75b162152 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -28,7 +28,6 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest import org.springframework.context.annotation.Import import org.springframework.http.MediaType @@ -53,9 +52,6 @@ class TemporalEntityHandlerTests { private val incomingAttrExpandedName = "https://ontology.eglobalmark.com/apic#incoming" private val outgoingAttrExpandedName = "https://ontology.eglobalmark.com/apic#outgoing" - @Value("\${application.jsonld.apic_context}") - val apicContext: String? = null - private lateinit var apicHeaderLink: String @Autowired @@ -74,7 +70,7 @@ class TemporalEntityHandlerTests { @BeforeAll fun configureWebClientDefaults() { - apicHeaderLink = buildContextLinkHeader(apicContext!!) + apicHeaderLink = buildContextLinkHeader(APIC_COMPOUND_CONTEXT) webClient = webClient.mutate() .defaultHeaders { @@ -97,7 +93,7 @@ class TemporalEntityHandlerTests { webClient.post() .uri("/ngsi-ld/v1/temporal/entities/$entityUri/attrs") - .header("Link", buildContextLinkHeader(apicContext!!)) + .header("Link", buildContextLinkHeader(APIC_COMPOUND_CONTEXT)) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue(jsonLdObservation)) .exchange() @@ -127,7 +123,7 @@ class TemporalEntityHandlerTests { fun `it should return a 400 if temporal entity fragment is badly formed`() { webClient.post() .uri("/ngsi-ld/v1/temporal/entities/entityId/attrs") - .header("Link", buildContextLinkHeader(apicContext!!)) + .header("Link", buildContextLinkHeader(APIC_COMPOUND_CONTEXT)) .contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromValue("{ \"id\": \"bad\" }")) .exchange() @@ -509,7 +505,7 @@ class TemporalEntityHandlerTests { it.getFirst("endTime") == "2019-10-18T07:31:39Z" && it.getFirst("type") == "BeeHive" }, - apicContext!! + APIC_COMPOUND_CONTEXT ) } coVerify { @@ -518,7 +514,7 @@ class TemporalEntityHandlerTests { setOf("BeeHive"), temporalQuery, false, - apicContext!! + APIC_COMPOUND_CONTEXT ) } @@ -607,7 +603,7 @@ class TemporalEntityHandlerTests { queryParams.add("time", "2019-10-17T07:31:39Z") queryParams.add("attrs", "outgoing") - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertTrue(temporalQuery.expandedAttrs.size == 1) assertTrue(temporalQuery.expandedAttrs.contains(outgoingAttrExpandedName)) @@ -620,7 +616,7 @@ class TemporalEntityHandlerTests { queryParams.add("time", "2019-10-17T07:31:39Z") queryParams.add("attrs", "incoming,outgoing") - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertTrue(temporalQuery.expandedAttrs.size == 2) assertTrue( @@ -639,7 +635,7 @@ class TemporalEntityHandlerTests { queryParams.add("timerel", "after") queryParams.add("time", "2019-10-17T07:31:39Z") - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertTrue(temporalQuery.expandedAttrs.isEmpty()) } @@ -651,7 +647,7 @@ class TemporalEntityHandlerTests { queryParams.add("time", "2019-10-17T07:31:39Z") queryParams.add("lastN", "2") - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertTrue(temporalQuery.lastN == 2) } @@ -663,7 +659,7 @@ class TemporalEntityHandlerTests { queryParams.add("time", "2019-10-17T07:31:39Z") queryParams.add("lastN", "A") - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertNull(temporalQuery.lastN) } @@ -675,7 +671,7 @@ class TemporalEntityHandlerTests { queryParams.add("time", "2019-10-17T07:31:39Z") queryParams.add("lastN", "-2") - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertNull(temporalQuery.lastN) } @@ -684,7 +680,7 @@ class TemporalEntityHandlerTests { fun `it should treat time and timerel properties as optional`() { val queryParams = LinkedMultiValueMap() - val temporalQuery = buildTemporalQuery(queryParams, apicContext!!) + val temporalQuery = buildTemporalQuery(queryParams, APIC_COMPOUND_CONTEXT) assertEquals(null, temporalQuery.time) assertEquals(null, temporalQuery.timerel) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt index 8da095b35..4816294fc 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt @@ -10,7 +10,6 @@ import io.mockk.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest import org.springframework.context.annotation.Import import org.springframework.security.test.context.support.WithAnonymousUser @@ -28,9 +27,6 @@ import java.time.ZonedDateTime @WithMockUser class TemporalEntityOperationsHandlerTests { - @Value("\${application.jsonld.apic_context}") - val apicContext: String? = null - private lateinit var apicHeaderLink: String @Autowired @@ -41,7 +37,7 @@ class TemporalEntityOperationsHandlerTests { @BeforeAll fun configureWebClientDefaults() { - apicHeaderLink = buildContextLinkHeader(apicContext!!) + apicHeaderLink = buildContextLinkHeader(APIC_COMPOUND_CONTEXT) webClient = webClient.mutate() .defaultHeaders { @@ -87,7 +83,7 @@ class TemporalEntityOperationsHandlerTests { verify { queryService.parseAndCheckQueryParams( queryParams, - apicContext!! + APIC_COMPOUND_CONTEXT ) } coVerify { @@ -96,7 +92,7 @@ class TemporalEntityOperationsHandlerTests { setOf("BeeHive", "Apiary"), temporalQuery, true, - apicContext!! + APIC_COMPOUND_CONTEXT ) } diff --git a/search-service/src/test/resources/application-test.properties b/search-service/src/test/resources/application-test.properties index 8c2767c63..0755369df 100644 --- a/search-service/src/test/resources/application-test.properties +++ b/search-service/src/test/resources/application-test.properties @@ -1,3 +1 @@ -application.jsonld.apic_context=https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld - application.authentication.enabled = true diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt index 4b86af20a..53563882a 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt @@ -29,9 +29,6 @@ import java.time.ZoneOffset @ActiveProfiles("test") class SubscriptionServiceTests : WithTimescaleContainer { - @Value("\${application.jsonld.apic_context}") - val apicContext: String = "" - @Autowired private lateinit var subscriptionService: SubscriptionService diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt index 8f7df1aa4..a6cbb69c8 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt @@ -38,9 +38,6 @@ import reactor.core.publisher.Mono @WithMockCustomUser(name = "Mock User", username = "mock-user") class SubscriptionHandlerTests { - @Value("\${application.jsonld.apic_context}") - val apicContext: String? = null - @Autowired private lateinit var webClient: WebTestClient @@ -168,7 +165,7 @@ class SubscriptionHandlerTests { it.entityId == "urn:ngsi-ld:Subscription:1".toUri() && it.operationPayload.removeNoise() == expectedOperationPayload.inputStream.readBytes() .toString(Charsets.UTF_8).removeNoise() && - it.contexts == listOf(apicContext!!) + it.contexts == listOf(APIC_COMPOUND_CONTEXT) } ) } @@ -410,7 +407,7 @@ class SubscriptionHandlerTests { val subscriptionId = subscriptionId val parsedSubscription = parseSubscriptionUpdate( jsonLdFile.inputStream.readBytes().toString(Charsets.UTF_8), - listOf(apicContext!!) + listOf(APIC_COMPOUND_CONTEXT) ) val updatedSubscription = gimmeRawSubscription() every { subscriptionService.exists(any()) } returns Mono.just(true) @@ -450,7 +447,7 @@ class SubscriptionHandlerTests { val subscriptionId = subscriptionId val parsedSubscription = parseSubscriptionUpdate( jsonLdFile.inputStream.readBytes().toString(Charsets.UTF_8), - listOf(apicContext!!) + listOf(APIC_COMPOUND_CONTEXT) ) every { subscriptionService.exists(any()) } returns Mono.just(true) diff --git a/subscription-service/src/test/resources/application-test.properties b/subscription-service/src/test/resources/application-test.properties index 8c2767c63..0755369df 100644 --- a/subscription-service/src/test/resources/application-test.properties +++ b/subscription-service/src/test/resources/application-test.properties @@ -1,3 +1 @@ -application.jsonld.apic_context=https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld - application.authentication.enabled = true From 80f056d5b8ca312c0b4b0d7245ac1f39f4f1833d Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 15 Jul 2021 07:34:03 +0200 Subject: [PATCH 06/56] fix(common): do not swallow the root exception when returning a 500 --- .../stellio/entity/web/EntityHandlerTests.kt | 20 +++++++++---- .../stellio/shared/web/ExceptionHandler.kt | 2 +- .../web/SubscriptionHandlerTests.kt | 30 +++++++++++++------ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt index 3171e41fc..ec2a7bdc8 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt @@ -163,9 +163,13 @@ class EntityHandlerTests { .exchange() .expectStatus().isEqualTo(500) .expectBody().json( - "{\"type\":\"https://uri.etsi.org/ngsi-ld/errors/InternalError\"," + - "\"title\":\"There has been an error during the operation execution\"," + - "\"detail\":\"Internal Server Exception\"}" + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/InternalError", + "title":"There has been an error during the operation execution", + "detail":"InternalErrorException(message=Internal Server Exception)" + } + """ ) } @@ -1808,9 +1812,13 @@ class EntityHandlerTests { .exchange() .expectStatus().isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR) .expectBody().json( - "{\"type\":\"https://uri.etsi.org/ngsi-ld/errors/InternalError\"," + - "\"title\":\"There has been an error during the operation execution\"," + - "\"detail\":\"Unexpected server error\"}" + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/InternalError", + "title":"There has been an error during the operation execution", + "detail":"java.lang.RuntimeException: Unexpected server error" + } + """ ) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt b/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt index 884cffa88..694e1b9f7 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt @@ -50,7 +50,7 @@ class ExceptionHandler { ) else -> generateErrorResponse( HttpStatus.INTERNAL_SERVER_ERROR, - InternalErrorResponse(cause.message ?: "There has been an error during the operation execution") + InternalErrorResponse("$cause") ) } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt index a6cbb69c8..e31195cd8 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt @@ -204,9 +204,13 @@ class SubscriptionHandlerTests { .exchange() .expectStatus().isEqualTo(500) .expectBody().json( - "{\"type\":\"https://uri.etsi.org/ngsi-ld/errors/InternalError\"," + - "\"title\":\"There has been an error during the operation execution\"," + - "\"detail\":\"Internal Server Exception\"}" + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/InternalError", + "title":"There has been an error during the operation execution", + "detail":"InternalErrorException(message=Internal Server Exception)" + } + """ ) } @@ -460,9 +464,13 @@ class SubscriptionHandlerTests { .exchange() .expectStatus().is5xxServerError .expectBody().json( - "{\"type\":\"https://uri.etsi.org/ngsi-ld/errors/InternalError\"," + - "\"title\":\"There has been an error during the operation execution\"," + - "\"detail\":\"Update failed\"}" + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/InternalError", + "title":"There has been an error during the operation execution", + "detail":"java.lang.RuntimeException: Update failed" + } + """ ) verify { subscriptionService.exists(eq(subscriptionId)) } @@ -617,9 +625,13 @@ class SubscriptionHandlerTests { .exchange() .expectStatus().isEqualTo(HttpStatus.INTERNAL_SERVER_ERROR) .expectBody().json( - "{\"type\":\"https://uri.etsi.org/ngsi-ld/errors/InternalError\"," + - "\"title\":\"There has been an error during the operation execution\"," + - "\"detail\":\"Unexpected server error\"}" + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/InternalError", + "title":"There has been an error during the operation execution", + "detail":"java.lang.RuntimeException: Unexpected server error" + } + """ ) verify { subscriptionService.exists(subscriptionId) } From 4ba1ce62f6a44053d7f95d1b152fccc0ba49c051 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 15 Jul 2021 07:36:53 +0200 Subject: [PATCH 07/56] conf(common): cleanup and align default Logback configuration for the 3 main services - default loggers to see incoming HTTP requests / responses - default loggers to see Cypher queries - default loggers to see PG queries --- entity-service/src/main/resources/logback-spring.xml | 7 +++---- search-service/src/main/resources/logback-spring.xml | 3 +-- subscription-service/src/main/resources/logback-spring.xml | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/entity-service/src/main/resources/logback-spring.xml b/entity-service/src/main/resources/logback-spring.xml index 8c1b996ac..376679a28 100644 --- a/entity-service/src/main/resources/logback-spring.xml +++ b/entity-service/src/main/resources/logback-spring.xml @@ -30,11 +30,10 @@ - + + - - - + diff --git a/search-service/src/main/resources/logback-spring.xml b/search-service/src/main/resources/logback-spring.xml index cf44bbbd9..010d7013e 100644 --- a/search-service/src/main/resources/logback-spring.xml +++ b/search-service/src/main/resources/logback-spring.xml @@ -33,8 +33,7 @@ - - + diff --git a/subscription-service/src/main/resources/logback-spring.xml b/subscription-service/src/main/resources/logback-spring.xml index 09b461960..b3857fe59 100644 --- a/subscription-service/src/main/resources/logback-spring.xml +++ b/subscription-service/src/main/resources/logback-spring.xml @@ -33,8 +33,7 @@ - - + From 97558934623faaf28d70591c43d9144bea1de9d4 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 15 Jul 2021 16:52:49 +0200 Subject: [PATCH 08/56] feat(entity): refactor process used to retrieve and serialize an NGSI-LD entity --- .../entity/repository/EntityRepository.kt | 17 -- .../entity/repository/Neo4jRepository.kt | 17 ++ .../stellio/entity/service/EntityService.kt | 155 ++++++------------ .../entity/service/EntityServiceTests.kt | 2 - .../egm/stellio/shared/util/JsonLdUtils.kt | 7 - 5 files changed, 63 insertions(+), 135 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt index 74558ccb5..6d6e9edb9 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/EntityRepository.kt @@ -16,23 +16,6 @@ interface EntityRepository : Neo4jRepository { ) fun getEntityCoreById(id: String): Entity? - @Query( - "MATCH (entity:Entity { id: \$id })-[:HAS_VALUE]->(property:Property)" + - "OPTIONAL MATCH (property)-[:HAS_VALUE]->(propValue:Property)" + - "OPTIONAL MATCH (property)-[:HAS_OBJECT]->(relOfProp:Relationship)-[rel]->(relOfPropObject)" + - "RETURN property, propValue, type(rel) as relType, relOfProp, relOfPropObject.id as relOfPropObjectId" - ) - fun getEntitySpecificProperties(id: String): List> - - @Query( - "MATCH (entity:Entity { id: \$id })-[:HAS_OBJECT]->(rel:Relationship)-[r]->(relObject)" + - "OPTIONAL MATCH (rel)-[:HAS_VALUE]->(propValue:Property)" + - "OPTIONAL MATCH (rel)-[:HAS_OBJECT]->(relOfRel:Relationship)-[or]->(relOfRelObject)" + - "RETURN rel, propValue, type(r) as relType, relObject.id as relObjectId, " + - " relOfRel, type(or) as relOfRelType, relOfRelObject.id as relOfRelObjectId" - ) - fun getEntityRelationships(id: String): List> - @Query( "MATCH ({ id: \$subjectId })-[:HAS_OBJECT]->(r:Relationship { datasetId: \$datasetId }) " + "-[:`:#{literal(#relationshipType)}`]->(e:Entity)" + diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt index d5384f9f6..4b158759c 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt @@ -177,6 +177,23 @@ class Neo4jRepository( return neo4jClient.query(query).bind(entityId.toString()).to("entityId").run().counters().propertiesSet() } + fun getTargetObjectIdOfRelationship(relationshipId: URI, relationshipType: String): URI { + val query = + """ + MATCH (r:Relationship { id: ${'$'}relationshipId })-[:$relationshipType]->(e) + RETURN e.id as entityId + """.trimIndent() + + val result = neo4jClient.query(query) + .bind(relationshipId.toString()).to("relationshipId") + .fetch() + .one() + + return result.map { + (it["entityId"] as String).toUri() + }.orElse(null) + } + fun hasRelationshipOfType(subjectNodeInfo: SubjectNodeInfo, relationshipType: String): Boolean { val query = """ diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 3bbf70127..0f876da35 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -9,10 +9,8 @@ import com.egm.stellio.entity.repository.Neo4jRepository import com.egm.stellio.entity.repository.PartialEntityRepository import com.egm.stellio.shared.model.* import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_HAS_OBJECT -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_TYPE -import com.egm.stellio.shared.util.JsonLdUtils.expandRelationshipType +import com.egm.stellio.shared.util.entityNotFoundMessage import com.egm.stellio.shared.util.extractShortTypeFromExpanded import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -169,128 +167,67 @@ class EntityService( * @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity */ fun getFullEntityById(entityId: URI, includeSysAttrs: Boolean = false): JsonLdEntity? { - val entity = entityRepository.getEntityCoreById(entityId.toString()) ?: return null + val entity = entityRepository.findById(entityId) + .orElseThrow { ResourceNotFoundException(entityNotFoundMessage(entityId.toString())) } val resultEntity = entity.serializeCoreProperties(includeSysAttrs) - entityRepository.getEntitySpecificProperties(entityId.toString()) - .groupBy { - (it["property"] as Property).id - } - .values - .map { buildPropertyFragment(it, entity.contexts, includeSysAttrs) } - .groupBy { it.first } - .mapValues { propertyInstances -> - propertyInstances.value.map { instanceFragment -> - instanceFragment.second - } - } - .forEach { property -> - resultEntity[property.key] = property.value - } + entity.properties + .map { property -> + val serializedProperty = property.serializeCoreProperties(includeSysAttrs) - entityRepository.getEntityRelationships(entityId.toString()) - .groupBy { - (it["rel"] as Relationship).id - }.values - .map { - buildRelationshipFragment(it, entity.contexts, includeSysAttrs) - } - .groupBy { it.first } - .mapValues { relationshipInstances -> - relationshipInstances.value.map { instanceFragment -> - instanceFragment.second + property.properties.forEach { innerProperty -> + val serializedSubProperty = innerProperty.serializeCoreProperties(includeSysAttrs) + serializedProperty[innerProperty.name] = serializedSubProperty } - } - .forEach { relationship -> - resultEntity[relationship.key] = relationship.value - } - - return JsonLdEntity(resultEntity, entity.contexts) - } - fun getEntityCoreProperties(entityId: URI) = entityRepository.getEntityCoreById(entityId.toString())!! + property.relationships.forEach { innerRelationship -> + val serializedSubRelationship = serializeRelationship(innerRelationship, includeSysAttrs) + serializedProperty[innerRelationship.relationshipType()] = serializedSubRelationship + } - private fun buildPropertyFragment( - rawProperty: List>, - contexts: List, - includeSysAttrs: Boolean - ): Pair> { - val property = rawProperty[0]["property"] as Property - val propertyKey = property.name - val propertyValues = property.serializeCoreProperties(includeSysAttrs) - - rawProperty.filter { relEntry -> relEntry["propValue"] != null } - .forEach { - val propertyOfProperty = it["propValue"] as Property - propertyValues[propertyOfProperty.name] = propertyOfProperty.serializeCoreProperties(includeSysAttrs) + Pair(property.name, serializedProperty) } - - rawProperty.filter { relEntry -> relEntry["relOfProp"] != null } - .forEach { - val relationship = it["relOfProp"] as Relationship - val targetEntityId = it["relOfPropObjectId"] as String - val relationshipKey = (it["relType"] as String) - logger.debug("Adding relOfProp to $targetEntityId with type $relationshipKey") - - val relationshipValue = mapOf( - JSONLD_TYPE to NGSILD_RELATIONSHIP_TYPE.uri, - NGSILD_RELATIONSHIP_HAS_OBJECT to mapOf(JSONLD_ID to targetEntityId) - ) - val relationshipValues = relationship.serializeCoreProperties(includeSysAttrs) - relationshipValues.putAll(relationshipValue) - val expandedRelationshipKey = - expandRelationshipType(mapOf(relationshipKey to relationshipValue), contexts) - propertyValues[expandedRelationshipKey] = relationshipValues + .groupBy({ it.first }, { it.second }) + .forEach { (propertyName, propertyValues) -> + resultEntity[propertyName] = propertyValues } - return Pair(propertyKey, propertyValues) - } + entity.relationships + .map { relationship -> + val serializedRelationship = serializeRelationship(relationship, includeSysAttrs) - private fun buildRelationshipFragment( - rawRelationship: List>, - contexts: List, - includeSysAttrs: Boolean - ): Pair> { - val relationship = rawRelationship[0]["rel"] as Relationship - val primaryRelType = relationship.type[0] - val primaryRelation = - rawRelationship.find { relEntry -> relEntry["relType"] == primaryRelType.toRelationshipTypeName() }!! - val relationshipTargetId = primaryRelation["relObjectId"] as String - val relationshipValue = mapOf( - JSONLD_TYPE to NGSILD_RELATIONSHIP_TYPE.uri, - NGSILD_RELATIONSHIP_HAS_OBJECT to mapOf(JSONLD_ID to relationshipTargetId) - ) + relationship.properties.forEach { innerProperty -> + val serializedSubProperty = innerProperty.serializeCoreProperties(includeSysAttrs) + serializedRelationship[innerProperty.name] = serializedSubProperty + } - val relationshipValues = relationship.serializeCoreProperties(includeSysAttrs) - relationshipValues.putAll(relationshipValue) + relationship.relationships.forEach { innerRelationship -> + val serializedSubRelationship = serializeRelationship(innerRelationship, includeSysAttrs) + serializedRelationship[innerRelationship.relationshipType()] = serializedSubRelationship + } - rawRelationship.filter { relEntry -> relEntry["propValue"] != null } - .forEach { - val propertyOfProperty = it["propValue"] as Property - relationshipValues[propertyOfProperty.name] = - propertyOfProperty.serializeCoreProperties(includeSysAttrs) + Pair(relationship.relationshipType(), serializedRelationship) + } + .groupBy({ it.first }, { it.second }) + .forEach { (relationshipName, relationshipValues) -> + resultEntity[relationshipName] = relationshipValues } - rawRelationship.filter { relEntry -> relEntry["relOfRel"] != null } - .forEach { - val relationship = it["relOfRel"] as Relationship - val innerRelType = (it["relOfRelType"] as String) - val innerTargetEntityId = it["relOfRelObjectId"] as String - - val innerRelationship = mapOf( - JSONLD_TYPE to NGSILD_RELATIONSHIP_TYPE.uri, - NGSILD_RELATIONSHIP_HAS_OBJECT to mapOf(JSONLD_ID to innerTargetEntityId) - ) - - val innerRelationshipValues = relationship.serializeCoreProperties(includeSysAttrs) - innerRelationshipValues.putAll(innerRelationship) - val expandedInnerRelationshipType = - expandRelationshipType(mapOf(innerRelType to relationshipValue), contexts) + return JsonLdEntity(resultEntity, entity.contexts) + } - relationshipValues[expandedInnerRelationshipType] = innerRelationshipValues - } + fun getEntityCoreProperties(entityId: URI) = entityRepository.getEntityCoreById(entityId.toString())!! - return Pair(primaryRelType, relationshipValues) + private fun serializeRelationship(relationship: Relationship, includeSysAttrs: Boolean): MutableMap { + val serializedRelationship = relationship.serializeCoreProperties(includeSysAttrs) + // TODO not perfect as it makes one more DB request per relationship to get the target entity id + val targetEntityId = + neo4jRepository.getTargetObjectIdOfRelationship( + relationship.id, + relationship.relationshipType().toRelationshipTypeName() + ) + serializedRelationship[NGSILD_RELATIONSHIP_HAS_OBJECT] = mapOf(JSONLD_ID to targetEntityId.toString()) + return serializedRelationship } /** @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt index 1affb1fa0..2a8c1b573 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityServiceTests.kt @@ -66,8 +66,6 @@ class EntityServiceTests { "@id" to mortalityRemovalServiceUri.toString(), "@type" to listOf("MortalityRemovalService") ) - every { entityRepository.getEntitySpecificProperties(any()) } returns listOf() - every { entityRepository.getEntityRelationships(any()) } returns listOf() every { mockedBreedingService.contexts } returns sampleDataWithContext.contexts entityService.createEntity(sampleDataWithContext) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index bbcff11c9..073a51ec2 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -248,13 +248,6 @@ object JsonLdUtils { } } - fun expandRelationshipType(relationship: Map>, contexts: List): String { - val jsonLdOptions = JsonLdOptions() - jsonLdOptions.expandContext = mapOf(JSONLD_CONTEXT to contexts) - val expKey = JsonLdProcessor.expand(relationship, jsonLdOptions) - return (expKey[0] as Map).keys.first() - } - fun expandJsonLdKey(type: String, context: String): String? = expandJsonLdKey(type, listOf(context)) From e03ab097fb337b11a727cdcf4023cfda3a6d0a27 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 17 Jul 2021 07:46:10 +0200 Subject: [PATCH 09/56] quality(common): add Jacoco code coverage reporting --- build.gradle.kts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 6f47db286..ef9bca40c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -21,6 +21,7 @@ plugins { kotlin("kapt") version "1.5.20" apply false id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false id("org.sonarqube") version "3.1.1" + jacoco } subprojects { @@ -35,6 +36,7 @@ subprojects { apply(plugin = "org.jlleitschuh.gradle.ktlint") apply(plugin = "kotlin-kapt") apply(plugin = "io.gitlab.arturbosch.detekt") + apply(plugin = "jacoco") java.sourceCompatibility = JavaVersion.VERSION_11 @@ -125,6 +127,21 @@ subprojects { } } + // see https://docs.gradle.org/current/userguide/jacoco_plugin.html for configuration instructions + jacoco { + toolVersion = "0.8.7" + } + tasks.test { + finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run + } + tasks.withType { + dependsOn(tasks.test) // tests are required to run before generating the report + reports { + xml.isEnabled = true + html.isEnabled = true + } + } + project.ext.set("jibFromImage", "gcr.io/distroless/java:11") project.ext.set("jibContainerJvmFlags", listOf("-Xms256m", "-Xmx768m")) project.ext.set("jibContainerCreationTime", "USE_CURRENT_TIMESTAMP") From 345c3c914670f9842312ae93e781ca8cd251af87 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 17 Jul 2021 08:21:39 +0200 Subject: [PATCH 10/56] quality(common): add SonarQube as a GitHub action --- .github/workflows/sonarqube.yml | 37 +++++++++++++++++++++++++++++++++ Jenkinsfile | 7 ------- build.gradle.kts | 10 ++++++++- gradle.properties | 3 +-- 4 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/sonarqube.yml diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml new file mode 100644 index 000000000..89f8052c9 --- /dev/null +++ b/.github/workflows/sonarqube.yml @@ -0,0 +1,37 @@ +name: Build +on: + push: + branches: + - master + - develop + pull_request: + types: [opened, synchronize, reopened] +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache SonarCloud packages + uses: actions/cache@v1 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Cache Gradle packages + uses: actions/cache@v1 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: ${{ runner.os }}-gradle + - name: Build and analyze + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: ./gradlew build sonarqube --info diff --git a/Jenkinsfile b/Jenkinsfile index 4b180a6da..7601873b7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -66,13 +66,6 @@ pipeline { sh './gradlew build -p subscription-service' } } - stage('Perform SonarCloud analysis') { - steps { - withSonarQubeEnv('SonarCloud for Stellio') { - sh './gradlew sonarqube' - } - } - } /* Jib only allows to add tags and always set the "latest" tag on the Docker images created. It's unavoidable to create separate stages for Dockerizing dev services and specify the full to.image path */ stage('Dockerize Dev Api Gateway') { diff --git a/build.gradle.kts b/build.gradle.kts index ef9bca40c..56aa97c12 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,7 +20,7 @@ plugins { id("com.google.cloud.tools.jib") version "2.5.0" apply false kotlin("kapt") version "1.5.20" apply false id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false - id("org.sonarqube") version "3.1.1" + id("org.sonarqube") version "3.3" jacoco } @@ -173,4 +173,12 @@ allprojects { mavenCentral() maven { url = uri("https://repo.spring.io/milestone") } } + + sonarqube { + properties { + property("sonar.projectKey", "stellio-hub_stellio-context-broker") + property("sonar.organization", "stellio-hub") + property("sonar.host.url", "https://sonarcloud.io") + } + } } diff --git a/gradle.properties b/gradle.properties index a183ab728..41c5beede 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,2 @@ kotlin.code.style=official - -# org.gradle.parallel=true +org.gradle.jvmargs=-XX:MaxMetaspaceSize=512m From 07d2afb47739d3e05c02959e04a732abd270a294 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 18 Jul 2021 07:09:08 +0200 Subject: [PATCH 11/56] quality(common): add GitHub action to publish unit tests results --- .github/workflows/sonarqube.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index 89f8052c9..2c6d4efdb 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -35,3 +35,13 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} run: ./gradlew build sonarqube --info + - name: Publish unit test results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: '**/test-results/**/TEST-*.xml' + - name: Perform SonarQube analysis + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: ./gradlew sonarqube --info From 120453017208951f2ddc1b4a3baa8c9d6eb34337 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 18 Jul 2021 07:31:31 +0200 Subject: [PATCH 12/56] conf: update configuration of CLA GitHub Action --- .github/workflows/cla.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 5939f74ef..516b06805 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -11,7 +11,7 @@ jobs: steps: - name: "CLA Assistant" if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - # Alpha Release + # Beta Release uses: cla-assistant/github-action@v2.1.3-beta env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -19,7 +19,7 @@ jobs: PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} with: path-to-signatures: 'docs/cla/cla.json' - path-to-cla-document: 'https://fiware.github.io/contribution-requirements/individual-cla.pdf' # e.g. a CLA or a DCO document + path-to-document: 'https://fiware.github.io/contribution-requirements/individual-cla.pdf' # e.g. a CLA or a DCO document # branch should not be protected branch: 'master' allowlist: bobeal,franckLG,HoucemKacem,dependabot,bot* From 9633056933c125485c818f78546e31f94f585407 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 18 Jul 2021 07:37:57 +0200 Subject: [PATCH 13/56] chore: rename GitHub action used to build and check the project --- .github/workflows/{sonarqube.yml => build.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{sonarqube.yml => build.yml} (94%) diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/build.yml similarity index 94% rename from .github/workflows/sonarqube.yml rename to .github/workflows/build.yml index 2c6d4efdb..6e528a7c0 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/build.yml @@ -30,11 +30,11 @@ jobs: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} restore-keys: ${{ runner.os }}-gradle - - name: Build and analyze + - name: Build env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: ./gradlew build sonarqube --info + run: ./gradlew build - name: Publish unit test results uses: EnricoMi/publish-unit-test-result-action@v1 if: always() From 25390b07b72f8c312eb79ea86b9bf7b704516c98 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 18 Jul 2021 10:37:14 +0200 Subject: [PATCH 14/56] conf: add badge for build status --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f152c067e..31569d717 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![Docker badge](https://img.shields.io/docker/pulls/stellio/stellio-entity-service.svg)](https://hub.docker.com/r/stellio) [![License: Apache-2.0](https://img.shields.io/github/license/stellio-hub/stellio-context-broker.svg)](https://spdx.org/licenses/Apache-2.0.html)
-![Release Drafter](https://github.com/stellio-hub/stellio-context-broker/workflows/Release%20Drafter/badge.svg) +![Build](https://github.com/stellio-hub/stellio-context-broker/workflows/Build/badge.svg) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=stellio-hub_stellio-context-broker&metric=alert_status)](https://sonarcloud.io/dashboard?id=stellio-hub_stellio-context-broker) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4527/badge)](https://bestpractices.coreinfrastructure.org/projects/4527) From 13237ed0c286fe7f303398ce97e91cfb65b01079 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 18 Jul 2021 11:20:34 +0200 Subject: [PATCH 15/56] doc: add a more detailed introduction of FIWARE --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 31569d717..f4c5bbe3e 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,13 @@ Stellio is an NGSI-LD compliant context broker developed by EGM. NGSI-LD is an Open API and Datamodel specification for context management [published by ETSI](https://www.etsi.org/deliver/etsi_gs/CIM/001_099/009/01.02.02_60/gs_CIM009v010202p.pdf). -This project is part of [FIWARE](https://www.fiware.org/). For more information check the FIWARE Catalogue entry for -[Core Context](https://github.com/Fiware/catalogue/tree/master/core). +Stellio is a [FIWARE](https://www.fiware.org/) Generic Enabler. Therefore, it can be integrated as part of any platform “Powered by FIWARE”. +FIWARE is a curated framework of open source platform components which can be assembled together with other third-party +platform components to accelerate the development of Smart Solutions. For more information check the FIWARE Catalogue entry for +[Core Context](https://github.com/Fiware/catalogue/tree/master/core). The roadmap of this FIWARE GE is described [here](./docs/roadmap.md). -The roadmap of this FIWARE GE is described [here](./docs/roadmap.md). +You can find more info at the [FIWARE developers](https://developers.fiware.org/) website and the [FIWARE](https://fiware.org/) website. +The complete list of FIWARE GEs and Incubated FIWARE GEs can be found in the [FIWARE Catalogue](https://catalogue.fiware.org/). | :books: [Documentation](https://stellio.rtfd.io/) | :whale: [Docker Hub](https://hub.docker.com/orgs/stellio/repositories) | :dart: [Roadmap](./docs/roadmap.md) | | ------------------------------------------------- | ---------------------------------------------------------------------- | ----------------------------------- | From 2795d7236d3a3d74c49de1acb585ba50129917ae Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 18 Jul 2021 11:20:59 +0200 Subject: [PATCH 16/56] chore(common): upgrade to Kotlin 1.5.21 --- build.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 56aa97c12..e9f092a1e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,11 +14,11 @@ plugins { // and the shared lib is obviously not one id("org.springframework.boot") version "2.5.2" apply false id("io.spring.dependency-management") version "1.0.11.RELEASE" apply false - kotlin("jvm") version "1.5.20" apply false - kotlin("plugin.spring") version "1.5.20" apply false + kotlin("jvm") version "1.5.21" apply false + kotlin("plugin.spring") version "1.5.21" apply false id("org.jlleitschuh.gradle.ktlint") version "10.1.0" id("com.google.cloud.tools.jib") version "2.5.0" apply false - kotlin("kapt") version "1.5.20" apply false + kotlin("kapt") version "1.5.21" apply false id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false id("org.sonarqube") version "3.3" jacoco From bb0e281b4b4e6d7019a5e6372181be9fbfa1bb44 Mon Sep 17 00:00:00 2001 From: Ahmed Abid <49984082+AHABID@users.noreply.github.com> Date: Mon, 19 Jul 2021 15:28:03 +0200 Subject: [PATCH 17/56] feat: delete API_Quick_Start, used samples and postman collection (#455) --- API_Quick_Start.md | 213 --- README.md | 2 +- .../API_Quick_Start.postman_collection.json | 1181 ----------------- samples/apiary.jsonld | 21 - samples/apiculture_entities.jsonld | 101 -- samples/beehive.jsonld | 45 - samples/beehive_addName.json | 6 - samples/beehive_secondTemperatureUpdate.json | 12 - samples/beehive_updateHumidity.json | 12 - samples/beehive_updateTemperature.json | 12 - samples/beekeeper.jsonld | 11 - samples/sensor_humidity.jsonld | 11 - samples/sensor_temperature.jsonld | 11 - samples/subscription_newEndpoint.jsonld | 19 - samples/subscription_newQuery.jsonld | 8 - samples/subscription_to_beehive.jsonld | 27 - 16 files changed, 1 insertion(+), 1691 deletions(-) delete mode 100644 API_Quick_Start.md delete mode 100644 samples/API_Quick_Start.postman_collection.json delete mode 100644 samples/apiary.jsonld delete mode 100644 samples/apiculture_entities.jsonld delete mode 100644 samples/beehive.jsonld delete mode 100644 samples/beehive_addName.json delete mode 100644 samples/beehive_secondTemperatureUpdate.json delete mode 100644 samples/beehive_updateHumidity.json delete mode 100644 samples/beehive_updateTemperature.json delete mode 100644 samples/beekeeper.jsonld delete mode 100644 samples/sensor_humidity.jsonld delete mode 100644 samples/sensor_temperature.jsonld delete mode 100644 samples/subscription_newEndpoint.jsonld delete mode 100644 samples/subscription_newQuery.jsonld delete mode 100644 samples/subscription_to_beehive.jsonld diff --git a/API_Quick_Start.md b/API_Quick_Start.md deleted file mode 100644 index 012af4015..000000000 --- a/API_Quick_Start.md +++ /dev/null @@ -1,213 +0,0 @@ -# Quickstart - -This quickstart guide shows a real use case scenario of interaction with the API in an Apiculture context. - -## Prepare your environment - -The provided examples make use of the HTTPie command line tool (installation instructions: https://httpie.org/docs#installation) - -All requests are grouped in a Postman collection that can be found [here](samples/API_Quick_Start.postman_collection.json). -For more details about how to import a Postman collection see https://learning.postman.com/docs/getting-started/importing-and-exporting-data/. - -Start a Stellio instance. You can use the provided Docker compose configuration in this directory: - -```shell -docker-compose -f docker-compose.yml up -d && docker-compose -f docker-compose.yml logs -f -``` - -Export the link to the JSON-LD context used in this use case in an environment variable for easier referencing in -the requests: - -````shell -export CONTEXT_LINK="; rel=http://www.w3.org/ns/json-ld#context; type=application/ld+json" -```` - -## Case study - -This case study is written for anyone who wants to get familiar with the API, we use a real example to make it more concrete. - -We will create the following entities: -- A beekeeper -- An apiary -- A beehive -- Sensors linked to the beehive which measures two metrics (temperature and humidity) - -## Queries - -* We start by creating the beekeeper, the apiary, the beehive and sensors - -```shell -http POST http://localhost:8080/ngsi-ld/v1/entities Content-Type:application/ld+json < samples/beekeeper.jsonld -http POST http://localhost:8080/ngsi-ld/v1/entities Content-Type:application/ld+json < samples/apiary.jsonld -http POST http://localhost:8080/ngsi-ld/v1/entities Content-Type:application/ld+json < samples/sensor_temperature.jsonld -http POST http://localhost:8080/ngsi-ld/v1/entities Content-Type:application/ld+json < samples/sensor_humidity.jsonld -http POST http://localhost:8080/ngsi-ld/v1/entities Content-Type:application/ld+json < samples/beehive.jsonld -``` - -* We can delete the created entities and recreate them in batch (optional): - -```shell -http DELETE http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:Beekeeper:01 -http DELETE http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:Apiary:01 -http DELETE http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:Sensor:01 -http DELETE http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:Sensor:02 -http DELETE http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01 - -http POST http://localhost:8080/ngsi-ld/v1/entityOperations/create Content-Type:application/ld+json < samples/apiculture_entities.jsonld -``` - -* The created beehive can be retrieved by id: - -```shell -http http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01 Link:$CONTEXT_LINK -``` - -* Or by querying all entities of type BeeHive: - -```shell -http http://localhost:8080/ngsi-ld/v1/entities type==BeeHive Link:$CONTEXT_LINK -``` - -* Let's add a name to the created beehive: - -```shell -http POST http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs \ - Link:$CONTEXT_LINK < samples/beehive_addName.json -``` - -* The recently added name property can be deleted: - -```shell -http DELETE http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs/name Link:$CONTEXT_LINK -``` - -* Let's create a subscription to the beehive that sends a notification when the temperature exceeds 40 - -```shell -http POST http://localhost:8080/ngsi-ld/v1/subscriptions Content-Type:application/ld+json < samples/subscription_to_beehive.jsonld -``` - -* The created subscription can be retrieved by id - -```shell -http http://localhost:8080/ngsi-ld/v1/subscriptions/urn:ngsi-ld:Subscription:01 Link:$CONTEXT_LINK -``` - -* By doing this, increasing the beehive temperature to 42 will raise a notification - (the notification is sent via a POST request to the provided URI when creating the subscription, - please consider providing working `endpoint` params in order to receive it) - -```shell -http PATCH http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs \ - Link:$CONTEXT_LINK < samples/beehive_updateTemperature.json -``` - -* We can also update the beehive humidity - -```shell -http PATCH http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs \ - Link:$CONTEXT_LINK < samples/beehive_updateHumidity.json -``` - -* Since we updated both temperature and humidity, we can get the temporal evolution of those properties - -```shell -http http://localhost:8080/ngsi-ld/v1/temporal/entities/urn:ngsi-ld:BeeHive:01 \ - timerel==between \ - time==2019-10-25T12:00:00Z \ - endTime==2020-10-27T12:00:00Z \ - Link:$CONTEXT_LINK -``` - -Sample payload returned showing the temporal evolution of temperature and humidity properties: - -```json -{ - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ], - "humidity": [ - { - "instanceId": "urn:ngsi-ld:Instance:a768ffb8-79e0-488f-9a4c-e7217ba2dff4", - "observedAt": "2019-10-26T21:35:52.986010Z", - "type": "Property", - "value": 58.0 - }, - { - "instanceId": "urn:ngsi-ld:Instance:28e44b9e-86f5-4bbb-a363-718d849d1782", - "observedAt": "2019-10-26T21:32:52.986010Z", - "type": "Property", - "value": 60.0 - } - ], - "id": "urn:ngsi-ld:BeeHive:01", - "temperature": [ - { - "instanceId": "urn:ngsi-ld:Instance:33642ff7-9b66-42c4-bcdf-9ca640cba782", - "observedAt": "2019-10-26T22:35:52.986010Z", - "type": "Property", - "value": 42.0 - }, - { - "instanceId": "urn:ngsi-ld:Instance:73226f04-1c47-49a6-ab88-976cb7493bea", - "observedAt": "2019-10-26T21:32:52.986010Z", - "type": "Property", - "value": 22.2 - } - ], - "type": "BeeHive" -} -``` - -* Let's update again the temperature property of the beehive - -```shell -http PATCH http://localhost:8080/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs \ - Link:$CONTEXT_LINK < samples/beehive_secondTemperatureUpdate.json -``` - -* We can get the simplified temporal representation of the temperature property of the beehive - -```shell -http http://localhost:8080/ngsi-ld/v1/temporal/entities/urn:ngsi-ld:BeeHive:01?options=temporalValues \ - attrs==temperature \ - timerel==between \ - time==2019-10-25T12:00:00Z \ - endTime==2020-10-27T12:00:00Z \ - Link:$CONTEXT_LINK -``` - -The sample payload returned showing the simplified temporal evolution of temperature and humidity properties: - -```json -{ - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ], - "temperature": { - "createdAt": "2020-06-15T11:37:25.803985Z", - "observedBy": { - "createdAt": "2020-06-15T11:37:25.830823Z", - "object": "urn:ngsi-ld:Sensor:01", - "type": "Relationship" - }, - "type": "Property", - "unitCode": "CEL", - "values": [ - [ - 42.0, - "2019-10-26T22:35:52.986010Z" - ], - [ - 22.2, - "2019-10-26T21:32:52.986010Z" - ], - [ - 100.0, - "2020-05-10T10:20:30.98601Z" - ] - ] - }, - "type": "BeeHive" -} -``` diff --git a/README.md b/README.md index f4c5bbe3e..bd3da8244 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ docker run stellio/stellio-entity-service:latest ## Usage -To start using Stellio, you can follow the [API quick start](API_Quick_Start.md). +To start using Stellio, you can follow the [API quick start](https://github.com/stellio-hub/stellio-docs/blob/master/docs/quick_start_guide.md). ## License diff --git a/samples/API_Quick_Start.postman_collection.json b/samples/API_Quick_Start.postman_collection.json deleted file mode 100644 index f893d555c..000000000 --- a/samples/API_Quick_Start.postman_collection.json +++ /dev/null @@ -1,1181 +0,0 @@ -{ - "info": { - "_postman_id": "17426e30-c429-45fa-a443-a2ee54702111", - "name": "API_Quick_Start", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" - }, - "item": [ - { - "name": "Create entities", - "item": [ - { - "name": "Create entity", - "item": [ - { - "name": "Create Beekeeper", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/ld+json", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\":\"urn:ngsi-ld:Beekeeper:01\",\n \"type\":\"Beekeeper\",\n \"name\":{\n \"type\":\"Property\",\n \"value\":\"Scalpa\"\n },\n \"@context\":[\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities" - ] - } - }, - "response": [] - }, - { - "name": "Create Apiary", - "protocolProfileBehavior": { - "disabledSystemHeaders": { - "content-type": true - } - }, - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/ld+json", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\":\"urn:ngsi-ld:Apiary:01\",\n \"type\":\"Apiary\",\n \"name\":{\n \"type\":\"Property\",\n \"value\":\"ApiarySophia\"\n },\n \"location\": {\n \"type\": \"GeoProperty\",\n \"value\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 24.30623,\n 60.07966\n ]\n }\n },\n \"@context\":[\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities" - ] - } - }, - "response": [] - }, - { - "name": "Create Temperature Sensor", - "protocolProfileBehavior": { - "disabledSystemHeaders": { - "content-type": true - } - }, - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/ld+json", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"urn:ngsi-ld:Sensor:02\",\n \"type\": \"Sensor\",\n \"deviceParameter\":{\n \"type\":\"Property\",\n \"value\":\"temperature\"\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities" - ] - } - }, - "response": [] - }, - { - "name": "Create Humidity Sensor", - "protocolProfileBehavior": { - "disabledSystemHeaders": { - "content-type": true - } - }, - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/ld+json", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"urn:ngsi-ld:Sensor:01\",\n \"type\": \"Sensor\",\n \"deviceParameter\":{\n \"type\":\"Property\",\n \"value\":\"humidity\"\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities" - ] - } - }, - "response": [] - }, - { - "name": "Create Beehive", - "protocolProfileBehavior": { - "disabledSystemHeaders": { - "content-type": true - } - }, - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/ld+json", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"urn:ngsi-ld:BeeHive:01\",\n \"type\": \"BeeHive\",\n \"belongs\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Apiary:01\"\n },\n \"managedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Beekeeper:01\"\n },\n \"location\": {\n \"type\": \"GeoProperty\",\n \"value\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 24.30623,\n 60.07966\n ]\n }\n },\n \"temperature\": {\n \"type\": \"Property\",\n \"value\": 22.2,\n \"unitCode\": \"CEL\",\n \"observedAt\": \"2019-10-26T21:32:52.98601Z\",\n \"observedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Sensor:01\"\n }\n },\n \"humidity\": {\n \"type\": \"Property\",\n \"value\": 60,\n \"unitCode\": \"P1\",\n \"observedAt\": \"2019-10-26T21:32:52.98601Z\",\n \"observedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Sensor:02\"\n }\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Create batch of entities", - "item": [ - { - "name": "Create Apiculture Entities", - "request": { - "method": "POST", - "header": [ - { - "key": "", - "value": "", - "type": "text", - "disabled": true - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "[\n {\n \"id\":\"urn:ngsi-ld:Beekeeper:01\",\n \"type\":\"Beekeeper\",\n \"name\":{\n \"type\":\"Property\",\n \"value\":\"Scalpa\"\n },\n \"@context\":[\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n },\n {\n \"id\":\"urn:ngsi-ld:Apiary:01\",\n \"type\":\"Apiary\",\n \"name\":{\n \"type\":\"Property\",\n \"value\":\"ApiarySophia\"\n },\n \"location\": {\n \"type\": \"GeoProperty\",\n \"value\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 24.30623,\n 60.07966\n ]\n }\n },\n \"@context\":[\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n },\n {\n \"id\": \"urn:ngsi-ld:BeeHive:01\",\n \"type\": \"BeeHive\",\n \"belongs\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Apiary:01\"\n },\n \"managedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Beekeeper:01\"\n },\n \"location\": {\n \"type\": \"GeoProperty\",\n \"value\": {\n \"type\": \"Point\",\n \"coordinates\": [\n 24.30623,\n 60.07966\n ]\n }\n },\n \"temperature\": {\n \"type\": \"Property\",\n \"value\": 22.2,\n \"unitCode\": \"CEL\",\n \"observedAt\": \"2019-10-26T21:32:52.98601Z\",\n \"observedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Sensor:01\"\n }\n },\n \"humidity\": {\n \"type\": \"Property\",\n \"value\": 60,\n \"unitCode\": \"P1\",\n \"observedAt\": \"2019-10-26T21:32:52.98601Z\",\n \"observedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Sensor:02\"\n }\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n },\n {\n \"id\": \"urn:ngsi-ld:Sensor:01\",\n \"type\": \"Sensor\",\n \"deviceParameter\":{\n \"type\":\"Property\",\n \"value\":\"humidity\"\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n },\n {\n \"id\": \"urn:ngsi-ld:Sensor:02\",\n \"type\": \"Sensor\",\n \"deviceParameter\":{\n \"type\":\"Property\",\n \"value\":\"temperature\"\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entityOperations/create", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entityOperations", - "create" - ] - } - }, - "response": [] - } - ] - } - ] - }, - { - "name": "Delete Entities", - "item": [ - { - "name": "Delete Beekeeper", - "request": { - "method": "DELETE", - "header": [ - { - "key": "", - "value": "", - "type": "text", - "disabled": true - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Beekeeper:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Beekeeper:01" - ] - } - }, - "response": [] - }, - { - "name": "Delete Apiary", - "request": { - "method": "DELETE", - "header": [ - { - "key": "", - "type": "text", - "value": "", - "disabled": true - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Apiary:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Apiary:01" - ] - } - }, - "response": [] - }, - { - "name": "Delete Temperature Sensor", - "request": { - "method": "DELETE", - "header": [ - { - "key": "", - "type": "text", - "value": "", - "disabled": true - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Sensor:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Sensor:01" - ] - } - }, - "response": [] - }, - { - "name": "Delete Humidity Sensor", - "request": { - "method": "DELETE", - "header": [ - { - "key": "", - "type": "text", - "value": "", - "disabled": true - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Sensor:02", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Sensor:02" - ] - } - }, - "response": [] - }, - { - "name": "Delete Beehive", - "request": { - "method": "DELETE", - "header": [ - { - "key": "", - "type": "text", - "value": "", - "disabled": true - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Update Attributes", - "item": [ - { - "name": "Update Beehive temperature", - "request": { - "method": "PATCH", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"temperature\": {\n \"type\": \"Property\",\n \"value\": 43,\n \"unitCode\": \"CEL\",\n \"observedAt\": \"2020-10-26T22:35:52.98601Z\",\n \"observedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Sensor:01\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01", - "attrs" - ] - } - }, - "response": [] - }, - { - "name": "Update Beehive humidity", - "request": { - "method": "PATCH", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"humidity\": {\n \"type\": \"Property\",\n \"value\": 58,\n \"unitCode\": \"P1\",\n \"observedAt\": \"2019-10-26T21:35:52.98601Z\",\n \"observedBy\": {\n \"type\": \"Relationship\",\n \"object\": \"urn:ngsi-ld:Sensor:02\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01", - "attrs" - ] - } - }, - "response": [] - }, - { - "name": "Patch update Beehive temperature", - "request": { - "method": "PATCH", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"value\": 2,\n \"observedAt\": \"2021-01-08T10:20:30.98601Z\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs/temperature", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01", - "attrs", - "temperature" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Retrieve Entities", - "item": [ - { - "name": "Retrieve Beekeeper", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "value": "{{apicContextLink}}", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Beekeeper:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Beekeeper:01" - ] - } - }, - "response": [] - }, - { - "name": "Retrieve Apiary", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Apiary:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Apiary:01" - ] - } - }, - "response": [] - }, - { - "name": "Retrieve Temperature Sensor", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Sensor:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Sensor:01" - ] - } - }, - "response": [] - }, - { - "name": "Retrieve Humidity Sensor", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:Sensor:02", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:Sensor:02" - ] - } - }, - "response": [] - }, - { - "name": "Retrieve Beehive", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01" - ] - } - }, - "response": [] - }, - { - "name": "Retrieve Beehive - JSON representation", - "protocolProfileBehavior": { - "disabledSystemHeaders": { - "accept": true - } - }, - "request": { - "method": "GET", - "header": [ - { - "key": "Accept", - "type": "text", - "value": "application/json" - }, - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Query Entities", - "item": [ - { - "name": "Search Beehive", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "value": "{{apicContextLink}}", - "type": "text" - }, - { - "key": "", - "value": "", - "type": "text", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities?type=BeeHive", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities" - ], - "query": [ - { - "key": "type", - "value": "BeeHive" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "Append Attributes", - "item": [ - { - "name": "Add name to Beehive", - "request": { - "method": "POST", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"name\":{\n \"type\":\"Property\",\n \"value\":\"BeeHiveSophia\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01", - "attrs" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Delete Attributes", - "item": [ - { - "name": "Remove name from Beehive", - "request": { - "method": "DELETE", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/entities/urn:ngsi-ld:BeeHive:01/attrs/name", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "entities", - "urn:ngsi-ld:BeeHive:01", - "attrs", - "name" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Create Subscription", - "item": [ - { - "name": "Create subscription to Beehive", - "protocolProfileBehavior": { - "disabledSystemHeaders": { - "content-type": true - } - }, - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "type": "text", - "value": "application/ld+json" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\":\"urn:ngsi-ld:Subscription:01\",\n \"type\":\"Subscription\",\n \"entities\": [\n {\n \"type\": \"BeeHive\"\n }\n ],\n \"q\": \"temperature>40\",\n \"notification\": {\n \"attributes\": [\"temperature\"],\n \"format\": \"normalized\",\n \"endpoint\": {\n \"uri\": \"http://my-domain-name\",\n \"accept\": \"application/json\",\n \"info\": [\n {\n \"key\": \"Authorization-token\",\n \"value\": \"Authorization-token-value\"\n }\n ]\n }\n },\n \"@context\": [\n \"https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld\",\n \"http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/subscriptions", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "subscriptions" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Retrieve Subscription", - "item": [ - { - "name": "Retrieve subscription to Beehive", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/subscriptions/urn:ngsi-ld:Subscription:01", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "subscriptions", - "urn:ngsi-ld:Subscription:01" - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "Get Temporal Evolution of Entities", - "item": [ - { - "name": "Temporal evolution of Beehive properties", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/temporal/entities/urn:ngsi-ld:BeeHive:01?timerel=between&time=2019-10-25T12:00:00Z&endTime=2021-01-10T12:00:00Z", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "temporal", - "entities", - "urn:ngsi-ld:BeeHive:01" - ], - "query": [ - { - "key": "timerel", - "value": "between" - }, - { - "key": "time", - "value": "2019-10-25T12:00:00Z" - }, - { - "key": "endTime", - "value": "2021-01-10T12:00:00Z" - } - ] - } - }, - "response": [] - }, - { - "name": "Simplified temporal evolution of Beehive properties Copy", - "request": { - "method": "GET", - "header": [ - { - "key": "Link", - "type": "text", - "value": "{{apicContextLink}}" - }, - { - "key": "", - "type": "text", - "value": "", - "disabled": true - } - ], - "url": { - "raw": "{{gatewayServer}}/ngsi-ld/v1/temporal/entities/urn:ngsi-ld:BeeHive:01?options=temporalValues&attrs=temperature&timerel=between&time=2019-10-25T12:00:00Z&endTime=2021-01-10T12:00:00Z", - "host": [ - "{{gatewayServer}}" - ], - "path": [ - "ngsi-ld", - "v1", - "temporal", - "entities", - "urn:ngsi-ld:BeeHive:01" - ], - "query": [ - { - "key": "options", - "value": "temporalValues" - }, - { - "key": "attrs", - "value": "temperature" - }, - { - "key": "timerel", - "value": "between" - }, - { - "key": "time", - "value": "2019-10-25T12:00:00Z" - }, - { - "key": "endTime", - "value": "2021-01-10T12:00:00Z" - } - ] - } - }, - "response": [] - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ], - "variable": [ - { - "key": "apicContextLink", - "value": "; rel=\"http://www.w3.org/ns/json-ld#context\"; type=\"application/ld+json\"" - }, - { - "key": "gatewayServer", - "value": "http://localhost:8080" - } - ] -} \ No newline at end of file diff --git a/samples/apiary.jsonld b/samples/apiary.jsonld deleted file mode 100644 index 46623affe..000000000 --- a/samples/apiary.jsonld +++ /dev/null @@ -1,21 +0,0 @@ -{ - "id":"urn:ngsi-ld:Apiary:01", - "type":"Apiary", - "name":{ - "type":"Property", - "value":"ApiarySophia" - }, - "location": { - "type": "GeoProperty", - "value": { - "type": "Point", - "coordinates": [ - 24.30623, - 60.07966 - ] - } - }, - "@context":[ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/apiculture_entities.jsonld b/samples/apiculture_entities.jsonld deleted file mode 100644 index 67f6bf39b..000000000 --- a/samples/apiculture_entities.jsonld +++ /dev/null @@ -1,101 +0,0 @@ -[ - { - "id":"urn:ngsi-ld:Beekeeper:01", - "type":"Beekeeper", - "name":{ - "type":"Property", - "value":"Scalpa" - }, - "@context":[ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] - }, - { - "id":"urn:ngsi-ld:Apiary:01", - "type":"Apiary", - "name":{ - "type":"Property", - "value":"ApiarySophia" - }, - "location": { - "type": "GeoProperty", - "value": { - "type": "Point", - "coordinates": [ - 24.30623, - 60.07966 - ] - } - }, - "@context":[ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] - }, - { - "id": "urn:ngsi-ld:BeeHive:01", - "type": "BeeHive", - "belongs": { - "type": "Relationship", - "object": "urn:ngsi-ld:Apiary:01" - }, - "managedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Beekeeper:01" - }, - "location": { - "type": "GeoProperty", - "value": { - "type": "Point", - "coordinates": [ - 24.30623, - 60.07966 - ] - } - }, - "temperature": { - "type": "Property", - "value": 22.2, - "unitCode": "CEL", - "observedAt": "2019-10-26T21:32:52.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:01" - } - }, - "humidity": { - "type": "Property", - "value": 60, - "unitCode": "P1", - "observedAt": "2019-10-26T21:32:52.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:02" - } - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] - }, - { - "id": "urn:ngsi-ld:Sensor:01", - "type": "Sensor", - "deviceParameter":{ - "type":"Property", - "value":"humidity" - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] - }, - { - "id": "urn:ngsi-ld:Sensor:02", - "type": "Sensor", - "deviceParameter":{ - "type":"Property", - "value":"temperature" - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] - } -] diff --git a/samples/beehive.jsonld b/samples/beehive.jsonld deleted file mode 100644 index 163d62fa0..000000000 --- a/samples/beehive.jsonld +++ /dev/null @@ -1,45 +0,0 @@ -{ - "id": "urn:ngsi-ld:BeeHive:01", - "type": "BeeHive", - "belongs": { - "type": "Relationship", - "object": "urn:ngsi-ld:Apiary:01" - }, - "managedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Beekeeper:01" - }, - "location": { - "type": "GeoProperty", - "value": { - "type": "Point", - "coordinates": [ - 24.30623, - 60.07966 - ] - } - }, - "temperature": { - "type": "Property", - "value": 22.2, - "unitCode": "CEL", - "observedAt": "2019-10-26T21:32:52.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:01" - } - }, - "humidity": { - "type": "Property", - "value": 60, - "unitCode": "P1", - "observedAt": "2019-10-26T21:32:52.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:02" - } - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/beehive_addName.json b/samples/beehive_addName.json deleted file mode 100644 index c8014af78..000000000 --- a/samples/beehive_addName.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name":{ - "type":"Property", - "value":"BeeHiveSophia" - } -} \ No newline at end of file diff --git a/samples/beehive_secondTemperatureUpdate.json b/samples/beehive_secondTemperatureUpdate.json deleted file mode 100644 index 30272af6a..000000000 --- a/samples/beehive_secondTemperatureUpdate.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "temperature": { - "type": "Property", - "value": 100, - "unitCode": "CEL", - "observedAt": "2020-05-10T10:20:30.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:01" - } - } -} \ No newline at end of file diff --git a/samples/beehive_updateHumidity.json b/samples/beehive_updateHumidity.json deleted file mode 100644 index 6a86e9e33..000000000 --- a/samples/beehive_updateHumidity.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "humidity": { - "type": "Property", - "value": 58, - "unitCode": "P1", - "observedAt": "2019-10-26T21:35:52.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:02" - } - } -} \ No newline at end of file diff --git a/samples/beehive_updateTemperature.json b/samples/beehive_updateTemperature.json deleted file mode 100644 index 5e917f5ef..000000000 --- a/samples/beehive_updateTemperature.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "temperature": { - "type": "Property", - "value": 42, - "unitCode": "CEL", - "observedAt": "2019-10-26T22:35:52.98601Z", - "observedBy": { - "type": "Relationship", - "object": "urn:ngsi-ld:Sensor:01" - } - } -} \ No newline at end of file diff --git a/samples/beekeeper.jsonld b/samples/beekeeper.jsonld deleted file mode 100644 index 3d68de106..000000000 --- a/samples/beekeeper.jsonld +++ /dev/null @@ -1,11 +0,0 @@ -{ - "id":"urn:ngsi-ld:Beekeeper:01", - "type":"Beekeeper", - "name":{ - "type":"Property", - "value":"Scalpa" - }, - "@context":[ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/sensor_humidity.jsonld b/samples/sensor_humidity.jsonld deleted file mode 100644 index 51094fba7..000000000 --- a/samples/sensor_humidity.jsonld +++ /dev/null @@ -1,11 +0,0 @@ -{ - "id": "urn:ngsi-ld:Sensor:02", - "type": "Sensor", - "deviceParameter":{ - "type":"Property", - "value":"temperature" - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/sensor_temperature.jsonld b/samples/sensor_temperature.jsonld deleted file mode 100644 index f9b7e5e96..000000000 --- a/samples/sensor_temperature.jsonld +++ /dev/null @@ -1,11 +0,0 @@ -{ - "id": "urn:ngsi-ld:Sensor:01", - "type": "Sensor", - "deviceParameter":{ - "type":"Property", - "value":"humidity" - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/subscription_newEndpoint.jsonld b/samples/subscription_newEndpoint.jsonld deleted file mode 100644 index acdd4b7a9..000000000 --- a/samples/subscription_newEndpoint.jsonld +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id":"urn:ngsi-ld:Subscription:01", - "type":"Subscription", - "notification": { - "endpoint": { - "uri": "http://my-domain-name", - "accept": "application/json", - "info": [ - { - "key": "Authorization-token", - "value": "New-Authorization-token-value" - } - ] - } - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/subscription_newQuery.jsonld b/samples/subscription_newQuery.jsonld deleted file mode 100644 index 231c7882f..000000000 --- a/samples/subscription_newQuery.jsonld +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id":"urn:ngsi-ld:Subscription:01", - "type":"Subscription", - "q": "temperature>27", - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/samples/subscription_to_beehive.jsonld b/samples/subscription_to_beehive.jsonld deleted file mode 100644 index f1351abed..000000000 --- a/samples/subscription_to_beehive.jsonld +++ /dev/null @@ -1,27 +0,0 @@ -{ - "id":"urn:ngsi-ld:Subscription:01", - "type":"Subscription", - "entities": [ - { - "type": "BeeHive" - } - ], - "q": "temperature>40", - "notification": { - "attributes": ["temperature"], - "format": "normalized", - "endpoint": { - "uri": "http://my-domain-name", - "accept": "application/json", - "info": [ - { - "key": "Authorization-token", - "value": "Authorization-token-value" - } - ] - } - }, - "@context": [ - "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/apic/jsonld-contexts/apic-compound.jsonld" - ] -} From 956aec0a5724b87f42c7798e85cc60038f7ae0b4 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Tue, 20 Jul 2021 08:37:03 +0200 Subject: [PATCH 18/56] chore: upgrade cache GitHub Action to v2 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e528a7c0..501e4604b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,13 +19,13 @@ jobs: with: java-version: 11 - name: Cache SonarCloud packages - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - name: Cache Gradle packages - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} From e41ede8dee8d9db9b25d7ab55cdadfde3a8fa4be Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Tue, 20 Jul 2021 08:44:39 +0200 Subject: [PATCH 19/56] chore: upgrade setup-java GitHub Action to v2 (using Adopt distribution) --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 501e4604b..4f9aa017c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,8 +15,9 @@ jobs: with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: + distribution: adopt java-version: 11 - name: Cache SonarCloud packages uses: actions/cache@v2 From 33c1963ee0f4451e5fb2f2d89becaeac2f7d1d24 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Wed, 21 Jul 2021 10:37:48 +0200 Subject: [PATCH 20/56] feat(entity): implement attrs filtering on entities #452 (#460) --- .../stellio/entity/repository/QueryUtils.kt | 131 +++++++++-------- .../egm/stellio/entity/web/EntityHandler.kt | 5 +- .../repository/Neo4jSearchRepositoryTests.kt | 56 +++++--- .../StandaloneNeo4jSearchRepositoryTests.kt | 132 ++++++++++++++---- .../src/test/resources/logback-test.xml | 6 +- .../egm/stellio/shared/model/QueryParams.kt | 3 +- 6 files changed, 214 insertions(+), 119 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt index fd8400d98..1ca18894e 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt @@ -14,16 +14,17 @@ import java.util.regex.Pattern object QueryUtils { + private val qPattern = Pattern.compile("([^();|]+)") + fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, page: Int, limit: Int, contexts: List ): String { - val (id, expandedType, idPattern, q) = queryParams - val formattedIds = id?.map { "'$it'" } - val pattern = Pattern.compile("([^();|]+)") - val innerQuery = q?.let { buildInnerQuery(q, pattern, contexts) } ?: "" + val (id, expandedType, idPattern, q, expandedAttrs) = queryParams + val qClause = q?.let { buildInnerQuery(it, contexts) } ?: "" + val attrsClause = buildInnerAttrsFilterQuery(expandedAttrs) val matchUserClause = """ @@ -39,33 +40,20 @@ object QueryUtils { with userEntity """.trimIndent() - val idClause = - if (id != null) "AND entity.id in $formattedIds" - else "" - - val idPatternClause = - when { - idPattern != null -> """ - AND entity.id =~ '$idPattern' - ${if (innerQuery.isNotEmpty()) " AND " else ""} - """ - innerQuery.isNotEmpty() -> " AND " - else -> "" - } + val matchEntityClause = buildMatchEntityClause(expandedType, prefix = "") + val idClause = buildIdClause(id) + val idPatternClause = buildIdPatternClause(idPattern) - val matchEntityClause = - if (expandedType == null) - "(entity:Entity)" - else - "(entity:Entity:`$expandedType`)" + val finalFilterClause = setOf(idClause, idPatternClause, qClause, attrsClause) + .filter { it.isNotEmpty() } + .joinToString(separator = " AND ", prefix = " AND ") + .takeIf { it.trim() != "AND" } ?: "" val matchEntitiesClause = """ MATCH (userEntity)-[:HAS_OBJECT]->(right:Attribute:Relationship)-[]->$matchEntityClause WHERE any(r IN labels(right) WHERE r IN ${AuthorizationService.READ_RIGHT.map { "'$it'" }}) - $idClause - $idPatternClause - $innerQuery + $finalFilterClause return entity.id as entityId """.trimIndent() @@ -74,9 +62,7 @@ object QueryUtils { MATCH (userEntity)-[:HAS_OBJECT]->(:Attribute:Relationship) -[:isMemberOf]->(:Entity)-[:HAS_OBJECT]-(grpRight:Attribute:Relationship)-[]->$matchEntityClause WHERE any(r IN labels(grpRight) WHERE r IN ${AuthorizationService.READ_RIGHT.map { "'$it'" }}) - $idClause - $idPatternClause - $innerQuery + $finalFilterClause return entity.id as entityId """.trimIndent() @@ -87,9 +73,7 @@ object QueryUtils { '${AuthorizationService.SpecificAccessPolicy.AUTH_WRITE.name}', '${AuthorizationService.SpecificAccessPolicy.AUTH_READ.name}' ] - $idClause - $idPatternClause - $innerQuery + $finalFilterClause return entity.id as entityId """.trimIndent() @@ -126,36 +110,18 @@ object QueryUtils { limit: Int, contexts: List ): String { - val (id, expandedType, idPattern, q) = queryParams - val formattedIds = id?.map { "'$it'" } - val pattern = Pattern.compile("([^();|]+)") - val innerQuery = q?.let { buildInnerQuery(q, pattern, contexts) } ?: "" - - val matchClause = - if (expandedType == null) - "MATCH (entity:Entity)" - else - "MATCH (entity:`$expandedType`)" - - val whereClause = - if (innerQuery.isNotEmpty() || id != null || idPattern != null) " WHERE " - else "" - - val idClause = - if (id != null) - """ - entity.id in $formattedIds - ${if (idPattern != null || innerQuery.isNotEmpty()) " AND " else ""} - """ - else "" + val (id, expandedType, idPattern, q, expandedAttrs) = queryParams + val qClause = q?.let { buildInnerQuery(it, contexts) } ?: "" + val attrsClause = buildInnerAttrsFilterQuery(expandedAttrs) - val idPatternClause = - if (idPattern != null) - """ - entity.id =~ '$idPattern' - ${if (innerQuery.isNotEmpty()) " AND " else ""} - """ - else "" + val matchEntityClause = buildMatchEntityClause(expandedType) + val idClause = buildIdClause(id) + val idPatternClause = buildIdPatternClause(idPattern) + + val finalFilterClause = setOf(idClause, idPatternClause, qClause, attrsClause) + .filter { it.isNotEmpty() } + .joinToString(separator = " AND ", prefix = " WHERE ") + .takeIf { it.trim() != "WHERE" } ?: "" val pagingClause = if (limit == 0) """ @@ -171,18 +137,33 @@ object QueryUtils { """.trimIndent() return """ - $matchClause - $whereClause - $idClause - $idPatternClause - $innerQuery + $matchEntityClause + $finalFilterClause $pagingClause """ } - private fun buildInnerQuery(rawQuery: String, pattern: Pattern, contexts: List): String = + private fun buildMatchEntityClause(expandedType: String?, prefix: String = "MATCH"): String = + if (expandedType == null) + "$prefix (entity:Entity)" + else + "$prefix (entity:`$expandedType`)" + + private fun buildIdClause(id: List?): String { + return if (id != null) { + val formattedIds = id.map { "'$it'" } + " entity.id in $formattedIds " + } else "" + } + + private fun buildIdPatternClause(idPattern: String?): String = + if (idPattern != null) + " entity.id =~ '$idPattern' " + else "" + + private fun buildInnerQuery(rawQuery: String, contexts: List): String = - rawQuery.replace(pattern.toRegex()) { matchResult -> + rawQuery.replace(qPattern.toRegex()) { matchResult -> val parsedQueryTerm = extractComparisonParametersFromQuery(matchResult.value) if (parsedQueryTerm.third.isRelationshipTarget()) { """ @@ -220,4 +201,20 @@ object QueryUtils { } .replace(";", " AND ") .replace("|", " OR ") + + private fun buildInnerAttrsFilterQuery(expandedAttrs: Set): String = + expandedAttrs.joinToString( + separator = " AND " + ) { expandedAttr -> + """ + EXISTS { + MATCH (entity) + WHERE ( + (entity)-[:HAS_VALUE]->(:Property { name: '$expandedAttr' }) + OR + (entity)-[:HAS_OBJECT]-(:Relationship:`$expandedAttr`) + ) + } + """.trimIndent() + } } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt index 439bedcfe..2af393468 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt @@ -130,12 +130,14 @@ class EntityHandler( ) ) + val expandedAttrs = parseAndExpandRequestParameter(attrs, contextLink) + /** * Decoding query parameters is not supported by default so a call to a decode function was added query * with the right parameters values */ val countAndEntities = entityService.searchEntities( - QueryParams(ids, type?.let { expandJsonLdKey(type, contextLink) }, idPattern, q?.decode()), + QueryParams(ids, type?.let { expandJsonLdKey(type, contextLink) }, idPattern, q?.decode(), expandedAttrs), userId, page, limit, @@ -152,7 +154,6 @@ class EntityHandler( mediaType, contextLink ) - val expandedAttrs = parseAndExpandRequestParameter(attrs, contextLink) val filteredEntities = countAndEntities.second.filter { it.containsAnyOf(expandedAttrs) } .map { diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt index 90f926909..2f6b59515 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt @@ -52,6 +52,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { private val userUri = "urn:ngsi-ld:User:01".toUri() private val clientUri = "urn:ngsi-ld:Client:01".toUri() private val serviceAccountUri = "urn:ngsi-ld:User:01".toUri() + private val expandedNameProperty = expandJsonLdKey("name", DEFAULT_CONTEXTS)!! private val sub = "01" private val page = 1 private val limit = 20 @@ -67,17 +68,17 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val thirdEntity = createEntity( "urn:ngsi-ld:Beekeeper:1232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_WRITE, firstEntity.id) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_ADMIN, secondEntity.id) @@ -102,12 +103,12 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) createRelationship(EntitySubjectNode(groupEntity.id), R_CAN_WRITE, firstEntity.id) createRelationship(EntitySubjectNode(groupEntity.id), R_CAN_WRITE, secondEntity.id) @@ -129,7 +130,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), @@ -157,18 +158,29 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) createRelationship(EntitySubjectNode(clientEntity.id), R_CAN_READ, firstEntity.id) createRelationship(EntitySubjectNode(clientEntity.id), R_CAN_READ, secondEntity.id) - val entities = searchRepository.getEntities( - QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), + val queryParams = QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\"") + var entities = searchRepository.getEntities( + queryParams, + sub, + page, + limit, + DEFAULT_CONTEXTS + ).second + + assertTrue(entities.containsAll(listOf(firstEntity.id, secondEntity.id))) + + entities = searchRepository.getEntities( + queryParams.copy(expandedAttrs = setOf(expandedNameProperty)), sub, page, limit, @@ -193,12 +205,12 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) every { neo4jAuthorizationService.userIsAdmin(any()) } returns true @@ -221,7 +233,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { beekeeperUri, listOf("Beekeeper"), mutableListOf( - Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa"), + Property(name = expandedNameProperty, value = "Scalpa"), Property( name = JsonLdUtils.EGM_SPECIFIC_ACCESS_POLICY, value = AuthorizationService.SpecificAccessPolicy.AUTH_READ.name @@ -231,7 +243,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val entities = searchRepository.getEntities( @@ -252,17 +264,17 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) val thirdEntity = createEntity( "urn:ngsi-ld:Beekeeper:03432".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_WRITE, firstEntity.id) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_WRITE, secondEntity.id) @@ -285,12 +297,12 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_WRITE, firstEntity.id) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_WRITE, secondEntity.id) @@ -319,17 +331,17 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) val thirdEntity = createEntity( "urn:ngsi-ld:Beekeeper:03432".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_READ, firstEntity.id) createRelationship(EntitySubjectNode(userEntity.id), R_CAN_READ, secondEntity.id) diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt index 2d7ff17a3..39c607bf5 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt @@ -43,6 +43,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { private val beekeeperUri = "urn:ngsi-ld:Beekeeper:1230".toUri() private val deadFishUri = "urn:ngsi-ld:DeadFishes:019BN".toUri() private val partialTargetEntityUri = "urn:ngsi-ld:Entity:4567".toUri() + private val expandedNameProperty = expandJsonLdKey("name", DEFAULT_CONTEXTS)!! private val userId = "" private val page = 1 private val limit = 20 @@ -57,7 +58,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val entities = searchRepository.getEntities( @@ -76,7 +77,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val entities = searchRepository.getEntities( @@ -207,7 +208,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "ScalpaXYZ")) + mutableListOf(Property(name = expandedNameProperty, value = "ScalpaXYZ")) ) val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "name!=\"ScalpaXYZ\""), @@ -225,7 +226,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name!=\"ScalpaXYZ\""), @@ -366,7 +367,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { name = expandJsonLdKey("testedAt", DEFAULT_CONTEXTS)!!, value = LocalTime.parse("12:00:00") ), - Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "beekeeper") + Property(name = expandedNameProperty, value = "beekeeper") ) ) val entities = searchRepository.getEntities( @@ -390,7 +391,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { name = expandJsonLdKey("testedAt", DEFAULT_CONTEXTS)!!, value = LocalTime.parse("12:00:00") ), - Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "beekeeper") + Property(name = expandedNameProperty, value = "beekeeper") ) ) @@ -424,7 +425,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { name = expandJsonLdKey("testedAt", DEFAULT_CONTEXTS)!!, value = LocalTime.parse("12:00:00") ), - Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "beekeeper") + Property(name = expandedNameProperty, value = "beekeeper") ) ) createRelationship(EntitySubjectNode(entity.id), "observedBy", partialTargetEntityUri) @@ -481,12 +482,12 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) val entities = searchRepository.getEntities( QueryParams(id = listOf("urn:ngsi-ld:Beekeeper:1231"), expandedType = "Beekeeper"), @@ -579,17 +580,17 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( beekeeperUri, listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) val thirdEntity = createEntity( "urn:ngsi-ld:Beekeeper:1232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) val entities = searchRepository.getEntities( QueryParams( @@ -612,17 +613,17 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val firstEntity = createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) val thirdEntity = createEntity( "urn:ngsi-ld:Beekeeper:11232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:Beekeeper:0.*2$"), @@ -642,17 +643,17 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) createEntity( "urn:ngsi-ld:Beekeeper:11232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:BeeHive:*"), @@ -665,22 +666,105 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { assertTrue(entities.isEmpty()) } + @Test + fun `it should return an entity matching attrs on a property`() { + createEntity( + "urn:ngsi-ld:Beekeeper:01231".toUri(), + listOf("Beekeeper"), + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) + ) + createEntity( + "urn:ngsi-ld:Beekeeper:01232".toUri(), + listOf("Beekeeper"), + mutableListOf(Property(name = expandJsonLdKey("description", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + ) + val entities = searchRepository.getEntities( + QueryParams(expandedAttrs = setOf(expandedNameProperty)), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).second + + assertEquals(1, entities.size) + } + + @Test + fun `it should return an entity matching attrs on a relationship`() { + val entity = createEntity( + "urn:ngsi-ld:Beekeeper:01231".toUri(), + listOf("Beekeeper"), + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) + ) + createRelationship(EntitySubjectNode(entity.id), "observedBy", partialTargetEntityUri) + + val entities = searchRepository.getEntities( + QueryParams(expandedAttrs = setOf("observedBy")), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).second + + assertEquals(1, entities.size) + } + + @Test + fun `it should return an entity matching attrs on a relationship and a property value`() { + val entity = createEntity( + "urn:ngsi-ld:Beekeeper:01231".toUri(), + listOf("Beekeeper"), + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) + ) + createRelationship(EntitySubjectNode(entity.id), "observedBy", partialTargetEntityUri) + + val entities = searchRepository.getEntities( + QueryParams(q = "name==\"Scalpa\"", expandedAttrs = setOf("observedBy")), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).second + + assertEquals(1, entities.size) + } + + @Test + fun `it should return an entity matching attrs on a relationship and an entity type`() { + val entity = createEntity( + "urn:ngsi-ld:Beekeeper:01231".toUri(), + listOf("Beekeeper"), + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) + ) + createRelationship(EntitySubjectNode(entity.id), "observedBy", partialTargetEntityUri) + + val entities = searchRepository.getEntities( + QueryParams(expandedType = "Beekeeper", expandedAttrs = setOf("observedBy")), + userId, + page, + limit, + DEFAULT_CONTEXTS + ).second + + assertEquals(1, entities.size) + } + @Test fun `it should return matching entities count`() { createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) createEntity( "urn:ngsi-ld:Beekeeper:03432".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) val entitiesCount = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:Beekeeper:0.*2$"), @@ -704,17 +788,17 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { createEntity( "urn:ngsi-ld:Beekeeper:01231".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa")) ) createEntity( "urn:ngsi-ld:Beekeeper:01232".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa2")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa2")) ) createEntity( "urn:ngsi-ld:Beekeeper:03432".toUri(), listOf("Beekeeper"), - mutableListOf(Property(name = expandJsonLdKey("name", DEFAULT_CONTEXTS)!!, value = "Scalpa3")) + mutableListOf(Property(name = expandedNameProperty, value = "Scalpa3")) ) val entities = searchRepository.getEntities( diff --git a/entity-service/src/test/resources/logback-test.xml b/entity-service/src/test/resources/logback-test.xml index e6a9d7154..b7d020ade 100644 --- a/entity-service/src/test/resources/logback-test.xml +++ b/entity-service/src/test/resources/logback-test.xml @@ -14,9 +14,9 @@ - - - + + + diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/QueryParams.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/QueryParams.kt index 2e3847a0a..ad0d5a9e5 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/QueryParams.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/QueryParams.kt @@ -4,5 +4,6 @@ data class QueryParams( val id: List? = null, val expandedType: String? = null, val idPattern: String? = null, - val q: String? = null + val q: String? = null, + val expandedAttrs: Set = emptySet() ) From 06ed33b10b8979613a25996897535feae57d267b Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 22 Jul 2021 07:40:46 +0200 Subject: [PATCH 21/56] chore: upgrade Jib to 2.7.1 to fix Java 8 time error (#461) - see https://github.com/GoogleContainerTools/jib/issues/2966 for details --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index e9f092a1e..8e0b3a56b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,7 +17,7 @@ plugins { kotlin("jvm") version "1.5.21" apply false kotlin("plugin.spring") version "1.5.21" apply false id("org.jlleitschuh.gradle.ktlint") version "10.1.0" - id("com.google.cloud.tools.jib") version "2.5.0" apply false + id("com.google.cloud.tools.jib") version "2.7.1" apply false kotlin("kapt") version "1.5.21" apply false id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false id("org.sonarqube") version "3.3" From e68d3694cebe9aaf3f92d3c152273b04cfc8551c Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 23 Jul 2021 06:29:51 +0200 Subject: [PATCH 22/56] fix(search): return empty arrays for attributes that do not have any instance #462 (#463) --- .../stellio/search/service/QueryService.kt | 83 ++++++++++++------- .../service/AttributeInstanceServiceTests.kt | 19 +++++ .../search/service/QueryServiceTests.kt | 48 +++++++++++ 3 files changed, 118 insertions(+), 32 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt index 3ff055a0d..65acdea7c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt @@ -1,5 +1,7 @@ package com.egm.stellio.search.service +import com.egm.stellio.search.model.AttributeInstanceResult +import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.web.buildTemporalQuery import com.egm.stellio.shared.model.BadRequestDataException @@ -56,26 +58,15 @@ class QueryService( ) } - // split the group according to attribute type (measure or any) as this currently triggers 2 different queries - // then do one search for each type of attribute - val allAttributesInstancesPerAttribute = temporalEntityAttributes - .groupBy { - it.attributeValueType - }.mapValues { - attributeInstanceService.search(temporalQuery, it.value, withTemporalValues).awaitFirst() - } - .values - .flatten() - .groupBy { attributeInstanceResult -> - // group them by temporal entity attribute - temporalEntityAttributes.find { tea -> - tea.id == attributeInstanceResult.temporalEntityAttribute - }!! - } + val temporalEntityAttributesWithMatchingInstances = + searchInstancesForTemporalEntityAttributes(temporalEntityAttributes, temporalQuery, withTemporalValues) + + val temporalEntityAttributesWithInstances = + fillWithTEAWithoutInstances(temporalEntityAttributes, temporalEntityAttributesWithMatchingInstances) return temporalEntityService.buildTemporalEntity( entityId, - allAttributesInstancesPerAttribute, + temporalEntityAttributesWithInstances, temporalQuery, listOf(contextLink), withTemporalValues @@ -95,23 +86,14 @@ class QueryService( temporalQuery.expandedAttrs ).awaitFirstOrDefault(emptyList()) - // split the group according to attribute type (measure or any) as this currently triggers 2 different queries - // then do one search for each type of attribute - val allAttributesInstances = - temporalEntityAttributes.groupBy { - it.attributeValueType - }.mapValues { - attributeInstanceService.search(temporalQuery, it.value, withTemporalValues).awaitFirst() - }.values.flatten() + val temporalEntityAttributesWithMatchingInstances = + searchInstancesForTemporalEntityAttributes(temporalEntityAttributes, temporalQuery, withTemporalValues) + + val temporalEntityAttributesWithInstances = + fillWithTEAWithoutInstances(temporalEntityAttributes, temporalEntityAttributesWithMatchingInstances) val attributeInstancesPerEntityAndAttribute = - allAttributesInstances - .groupBy { - // first, group them by temporal entity attribute - temporalEntityAttributes.find { tea -> - tea.id == it.temporalEntityAttribute - }!! - } + temporalEntityAttributesWithInstances .toList() .groupBy { // then, group them by entity @@ -129,4 +111,41 @@ class QueryService( withTemporalValues ) } + + private suspend fun searchInstancesForTemporalEntityAttributes( + temporalEntityAttributes: List, + temporalQuery: TemporalQuery, + withTemporalValues: Boolean + ): Map> = + // split the group according to attribute type (measure or any) as this currently triggers 2 different queries + // then do one search for each type of attribute (less queries for improved performance) + temporalEntityAttributes + .groupBy { + it.attributeValueType + }.mapValues { + attributeInstanceService.search(temporalQuery, it.value, withTemporalValues).awaitFirst() + } + .values + .flatten() + .groupBy { attributeInstanceResult -> + // group them by temporal entity attribute + temporalEntityAttributes.find { tea -> + tea.id == attributeInstanceResult.temporalEntityAttribute + }!! + } + + private fun fillWithTEAWithoutInstances( + temporalEntityAttributes: List, + temporalEntityAttributesWithInstances: Map> + ): Map> { + // filter the temporal entity attributes for which there are no attribute instances + val temporalEntityAttributesWithoutInstances = + temporalEntityAttributes.filter { + !temporalEntityAttributesWithInstances.keys.contains(it) + } + // add them in the result set accompanied with an empty list + return temporalEntityAttributesWithInstances.plus( + temporalEntityAttributesWithoutInstances.map { it to emptyList() } + ) + } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt index e13920b1c..b7fdbe367 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt @@ -329,6 +329,25 @@ class AttributeInstanceServiceTests : WithTimescaleContainer { .verify() } + @Test + fun `it should not retrieve any instance if there is no value in the time interval`() { + (1..10).forEach { _ -> attributeInstanceService.create(gimmeAttributeInstance()).block() } + + val temporalQuery = TemporalQuery( + timerel = TemporalQuery.Timerel.AFTER, + time = now.plusHours(1) + ) + val enrichedEntity = + attributeInstanceService.search(temporalQuery, temporalEntityAttribute, false) + + StepVerifier.create(enrichedEntity) + .expectNextMatches { + it.isEmpty() + } + .expectComplete() + .verify() + } + @Test fun `it should not allow to create two attribute instances with same observation date`() { val attributeInstance = gimmeAttributeInstance() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt index 9c763cabb..97c6de7b6 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt @@ -16,6 +16,7 @@ import kotlinx.coroutines.test.runBlockingTest import org.junit.Rule import org.junit.jupiter.api.* import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles @@ -266,4 +267,51 @@ class QueryServiceTests { ) } } + + @Test + fun `it should return an empty list for a temporal entity attribute if it has no temporal values`() = + coroutinesTestRule.testDispatcher.runBlockingTest { + val temporalEntityAttribute = TemporalEntityAttribute( + entityId = entityUri, + type = "BeeHive", + attributeName = "incoming", + attributeValueType = TemporalEntityAttribute.AttributeValueType.MEASURE + ) + every { temporalEntityAttributeService.getForEntities(any(), any(), any()) } returns Mono.just( + listOf(temporalEntityAttribute) + ) + every { + attributeInstanceService.search(any(), any>(), any()) + } returns Mono.just(emptyList()) + + every { temporalEntityService.buildTemporalEntities(any(), any(), any(), any()) } returns emptyList() + + val entitiesList = queryService.queryTemporalEntities( + emptySet(), + setOf(beehiveType, apiaryType), + TemporalQuery( + expandedAttrs = emptySet(), + timerel = TemporalQuery.Timerel.BEFORE, + time = ZonedDateTime.parse("2019-10-17T07:31:39Z") + ), + false, + APIC_COMPOUND_CONTEXT + ) + + assertTrue(entitiesList.isEmpty()) + + io.mockk.verify { + temporalEntityService.buildTemporalEntities( + match { + it.size == 1 && + it.first().first == entityUri && + it.first().second.size == 1 && + it.first().second.values.first().isEmpty() + }, + any(), + listOf(APIC_COMPOUND_CONTEXT), + false + ) + } + } } From e8be87a100a5ddfae0ccdb18acf41904fcbbf49e Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 23 Jul 2021 09:33:20 +0200 Subject: [PATCH 23/56] chore(search): log incoming messages from entity topics --- .../com/egm/stellio/search/service/EntityEventListenerService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt index b86513b95..66383ff88 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/EntityEventListenerService.kt @@ -41,6 +41,7 @@ class EntityEventListenerService( @KafkaListener(topicPattern = "cim.entity.*", groupId = "context_search") fun processMessage(content: String) { + logger.debug("Processing message: $content") when (val entityEvent = deserializeAs(content)) { is EntityCreateEvent -> handleEntityCreateEvent(entityEvent) is EntityDeleteEvent -> handleEntityDeleteEvent(entityEvent) From 8ae2605b55cf132e26969b222b81f719a3dc1ab0 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 23 Jul 2021 13:52:45 +0200 Subject: [PATCH 24/56] fix(entity): when deleting an entity, use stored contexts in the propagated event (not required in request) --- .../main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt | 6 ++---- .../kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt index 2af393468..83d21de0f 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt @@ -240,10 +240,8 @@ class EntityHandler( */ @DeleteMapping("/{entityId}") suspend fun delete( - @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: String ): ResponseEntity<*> { - val contexts = listOf(getContextFromLinkHeaderOrDefault(httpHeaders)) val userId = extractSubjectOrEmpty().awaitFirst() if (!entityService.exists(entityId.toUri())) @@ -251,13 +249,13 @@ class EntityHandler( if (!authorizationService.userIsAdminOfEntity(entityId.toUri(), userId)) throw AccessDeniedException("User forbidden admin access to entity $entityId") - // Is there a way to avoid loading the entity to get its type... for the later event + // Is there a way to avoid loading the entity to get its type and contexts (for the event to be published)? val entity = entityService.getEntityCoreProperties(entityId.toUri()) entityService.deleteEntity(entityId.toUri()) entityEventService.publishEntityEvent( - EntityDeleteEvent(entityId.toUri(), contexts), entity.type[0] + EntityDeleteEvent(entityId.toUri(), entity.contexts), entity.type[0] ) return ResponseEntity.status(HttpStatus.NO_CONTENT).build() diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt index ec2a7bdc8..c7382f978 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt @@ -1757,7 +1757,6 @@ class EntityHandlerTests { webClient.delete() .uri("/ngsi-ld/v1/entities/$entityId") - .header(HttpHeaders.LINK, aquacHeaderLink) .exchange() .expectStatus().isNoContent .expectBody().isEmpty From 41fe819794ed2a1280ad5ca2b4e165cd246ac608 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 24 Jul 2021 10:41:30 +0200 Subject: [PATCH 25/56] chore(common): upgrade to Jib 3.1.2 #464 (#465) --- api-gateway/build.gradle.kts | 2 +- build.gradle.kts | 4 ++-- entity-service/build.gradle.kts | 2 +- search-service/build.gradle.kts | 2 +- subscription-service/build.gradle.kts | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api-gateway/build.gradle.kts b/api-gateway/build.gradle.kts index b1dcced3b..bd91dc13e 100644 --- a/api-gateway/build.gradle.kts +++ b/api-gateway/build.gradle.kts @@ -13,4 +13,4 @@ jib.to.image = "stellio/stellio-api-gateway" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8080") jib.container.creationTime = project.ext["jibContainerCreationTime"].toString() -jib.container.labels = project.ext["jibContainerLabels"] as Map +jib.container.labels.putAll(project.ext["jibContainerLabels"] as Map) diff --git a/build.gradle.kts b/build.gradle.kts index 8e0b3a56b..dde1eae74 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,7 +17,7 @@ plugins { kotlin("jvm") version "1.5.21" apply false kotlin("plugin.spring") version "1.5.21" apply false id("org.jlleitschuh.gradle.ktlint") version "10.1.0" - id("com.google.cloud.tools.jib") version "2.7.1" apply false + id("com.google.cloud.tools.jib") version "3.1.2" apply false kotlin("kapt") version "1.5.21" apply false id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false id("org.sonarqube") version "3.3" @@ -142,7 +142,7 @@ subprojects { } } - project.ext.set("jibFromImage", "gcr.io/distroless/java:11") + project.ext.set("jibFromImage", "adoptopenjdk:11-jre") project.ext.set("jibContainerJvmFlags", listOf("-Xms256m", "-Xmx768m")) project.ext.set("jibContainerCreationTime", "USE_CURRENT_TIMESTAMP") project.ext.set( diff --git a/entity-service/build.gradle.kts b/entity-service/build.gradle.kts index 12bdb7a1f..a09a3ffa7 100644 --- a/entity-service/build.gradle.kts +++ b/entity-service/build.gradle.kts @@ -42,5 +42,5 @@ jib.container.entrypoint = listOf( jib.container.environment = mapOf("NEO4J_WAIT_TIMEOUT" to "100") jib.container.ports = listOf("8082") jib.container.creationTime = project.ext["jibContainerCreationTime"].toString() -jib.container.labels = project.ext["jibContainerLabels"] as Map +jib.container.labels.putAll(project.ext["jibContainerLabels"] as Map) jib.extraDirectories.permissions = mapOf("/database/wait-for-neo4j.sh" to "775") diff --git a/search-service/build.gradle.kts b/search-service/build.gradle.kts index c09c1c35a..c087efc63 100644 --- a/search-service/build.gradle.kts +++ b/search-service/build.gradle.kts @@ -40,4 +40,4 @@ jib.to.image = "stellio/stellio-search-service" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8083") jib.container.creationTime = project.ext["jibContainerCreationTime"].toString() -jib.container.labels = project.ext["jibContainerLabels"] as Map +jib.container.labels.putAll(project.ext["jibContainerLabels"] as Map) diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index 558fdd6b5..4a7b9bd6f 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -41,4 +41,4 @@ jib.to.image = "stellio/stellio-subscription-service" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8084") jib.container.creationTime = project.ext["jibContainerCreationTime"].toString() -jib.container.labels = project.ext["jibContainerLabels"] as Map +jib.container.labels.putAll(project.ext["jibContainerLabels"] as Map) From d6977ba83ac95831283225fb55e80bda34e5d9db Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Tue, 27 Jul 2021 12:06:22 +0200 Subject: [PATCH 26/56] feat(entity): support for JSON object as value #158 (#469) --- .../config/Neo4jValuePropertyConverter.kt | 16 +++ .../com/egm/stellio/entity/model/Property.kt | 11 +- .../entity/repository/Neo4jRepositoryTests.kt | 117 ++++++++++++++++-- .../egm/stellio/shared/util/JsonLdUtils.kt | 37 ++++-- .../stellio/shared/model/NgsiLdEntityTests.kt | 61 +++++++++ .../service/EntityEventListenerService.kt | 4 +- 6 files changed, 220 insertions(+), 26 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt index 125aea2ee..267738d46 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/config/Neo4jValuePropertyConverter.kt @@ -1,5 +1,7 @@ package com.egm.stellio.entity.config +import com.egm.stellio.shared.util.JsonUtils.deserializeAs +import com.egm.stellio.shared.util.JsonUtils.serializeObject import org.neo4j.driver.Value import org.neo4j.driver.Values import org.neo4j.driver.internal.value.ListValue @@ -7,6 +9,8 @@ import org.neo4j.driver.internal.value.StringValue import org.springframework.data.neo4j.core.convert.Neo4jPersistentPropertyConverter import java.net.URI +const val JSON_OBJECT_PREFIX = "jsonObject@" + class Neo4jValuePropertyConverter : Neo4jPersistentPropertyConverter { @Suppress("SpreadOperator") @@ -14,6 +18,12 @@ class Neo4jValuePropertyConverter : Neo4jPersistentPropertyConverter { return when (source) { is List<*> -> ListValue(*source.map { Values.value(it) }.toTypedArray()) is URI -> StringValue(source.toString()) + is Map<*, *> -> { + // there is no neo4j support for JSON object + // store the serialized map prefixed with 'jsonObject@' to know how to deserialize it later + val value = JSON_OBJECT_PREFIX + serializeObject(source) + StringValue(value) + } else -> Values.value(source) } } @@ -21,6 +31,12 @@ class Neo4jValuePropertyConverter : Neo4jPersistentPropertyConverter { override fun read(source: Value): Any { return when (source) { is ListValue -> source.asList() + is StringValue -> { + if (source.asString().startsWith(JSON_OBJECT_PREFIX)) { + val sourceString = source.asString().removePrefix(JSON_OBJECT_PREFIX) + deserializeAs>(sourceString) + } else Values.ofObject().apply(source) + } else -> Values.ofObject().apply(source) } } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt index 04164ed05..f46edae16 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Property.kt @@ -1,5 +1,6 @@ package com.egm.stellio.entity.model +import com.egm.stellio.entity.config.JSON_OBJECT_PREFIX import com.egm.stellio.entity.config.Neo4jUriPropertyConverter import com.egm.stellio.entity.config.Neo4jValuePropertyConverter import com.egm.stellio.shared.model.NgsiLdPropertyInstance @@ -14,10 +15,12 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_UNIT_CODE_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMap import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsDateTime import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsString +import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.toNgsiLdFormat import com.egm.stellio.shared.util.toUri import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonRawValue +import org.neo4j.driver.internal.value.StringValue import org.springframework.data.annotation.LastModifiedDate import org.springframework.data.neo4j.core.convert.ConvertWith import org.springframework.data.neo4j.core.schema.Id @@ -133,7 +136,13 @@ data class Property( } nodeProperties["name"] = name - nodeProperties["value"] = value + nodeProperties["value"] = + // there is no neo4j support for JSON object + // store the serialized map prefixed with 'jsonObject@' to know how to deserialize it later + if (value is Map<*, *>) + StringValue(JSON_OBJECT_PREFIX + serializeObject(value)) + else + value unitCode?.run { nodeProperties["unitCode"] = this diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt index 7ac7c02e1..a43f3a9dd 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt @@ -49,6 +49,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { private val beekeeperUri = "urn:ngsi-ld:Beekeeper:1230".toUri() private val partialTargetEntityUri = "urn:ngsi-ld:Entity:4567".toUri() + private val sizeExpandedName = "https://uri.etsi.org/ngsi-ld/size" @AfterEach fun cleanData() { @@ -86,15 +87,111 @@ class Neo4jRepositoryTests : WithNeo4jContainer { assertEquals(EGM_OBSERVED_BY, relsOfProp[0].type[0]) } + @Test + fun `it should create an entity with a property whose name contains a colon`() { + val entity = createEntity( + beekeeperUri, + listOf("Beekeeper"), + mutableListOf( + Property(name = "prefix:name", value = "value") + ) + ) + + val persistedEntity = entityRepository.findById(entity.id) + assertTrue(persistedEntity.isPresent) + val persistedProperty = persistedEntity.get().properties[0] + assertEquals("prefix:name", persistedProperty.name) + } + + @Test + fun `it should create an entity with a property whose value is a JSON object`() { + val entity = createEntity( + beekeeperUri, + listOf("Beekeeper"), + mutableListOf( + Property( + name = sizeExpandedName, + value = mapOf("length" to 2, "height" to 12, "comment" to "none") + ) + ) + ) + + val persistedEntity = entityRepository.findById(entity.id) + assertTrue(persistedEntity.isPresent) + val persistedProperty = persistedEntity.get().properties[0] + assertEquals(mapOf("length" to 2, "height" to 12, "comment" to "none"), persistedProperty.value) + } + + @Test + fun `it should create a property for a subject`() { + val entity = createEntity( + beekeeperUri, + listOf("Beekeeper") + ) + + val property = Property(name = sizeExpandedName, value = 100L) + + val created = neo4jRepository.createPropertyOfSubject(EntitySubjectNode(entity.id), property) + + assertTrue(created) + + val propertyFromDb = propertyRepository.getPropertyOfSubject(entity.id, sizeExpandedName) + assertEquals(sizeExpandedName, propertyFromDb.name) + assertEquals(100L, propertyFromDb.value) + assertNotNull(propertyFromDb.createdAt) + assertNull(propertyFromDb.datasetId) + assertNull(propertyFromDb.observedAt) + assertEquals(0, propertyFromDb.properties.size) + assertEquals(0, propertyFromDb.relationships.size) + } + + @Test + fun `it should create a property with a JSON object value for a sujbect`() { + val entity = createEntity( + beekeeperUri, + listOf("Beekeeper") + ) + + val propertyValue = mapOf("key1" to "value1", "key2" to 12, "key3" to listOf("v1", "v2")) + val property = Property(name = sizeExpandedName, value = propertyValue) + + val created = neo4jRepository.createPropertyOfSubject(EntitySubjectNode(entity.id), property) + + assertTrue(created) + + val propertyFromDb = propertyRepository.getPropertyOfSubject(entity.id, sizeExpandedName) + assertEquals(sizeExpandedName, propertyFromDb.name) + assertEquals(propertyValue, propertyFromDb.value) + } + + @Test + fun `it should create a property with a list value for a subject`() { + val entity = createEntity( + beekeeperUri, + listOf("Beekeeper") + ) + + val propertyValue = listOf("v1", "v2") + val property = Property(name = sizeExpandedName, value = propertyValue) + + val created = neo4jRepository.createPropertyOfSubject(EntitySubjectNode(entity.id), property) + + assertTrue(created) + + val propertyFromDb = propertyRepository.getPropertyOfSubject(entity.id, sizeExpandedName) + assertEquals(sizeExpandedName, propertyFromDb.name) + assertEquals(propertyValue, propertyFromDb.value) + } + @Test fun `it should update the default property instance`() { val entity = createEntity( beekeeperUri, listOf("Beekeeper"), mutableListOf( - Property(name = "https://uri.etsi.org/ngsi-ld/size", value = 100L), + Property(name = sizeExpandedName, value = 100L), Property( - name = "https://uri.etsi.org/ngsi-ld/size", + name = sizeExpandedName, value = 200L, datasetId = "urn:ngsi-ld:Dataset:size:1".toUri() ) @@ -116,18 +213,18 @@ class Neo4jRepositoryTests : WithNeo4jContainer { neo4jRepository.updateEntityPropertyInstance( EntitySubjectNode(entity.id), - "https://uri.etsi.org/ngsi-ld/size", + sizeExpandedName, newProperty.instances[0] ) assertEquals( 300L, - propertyRepository.getPropertyOfSubject(entity.id, "https://uri.etsi.org/ngsi-ld/size").value + propertyRepository.getPropertyOfSubject(entity.id, sizeExpandedName).value ) assertEquals( 200L, propertyRepository.getPropertyOfSubject( - entity.id, "https://uri.etsi.org/ngsi-ld/size", "urn:ngsi-ld:Dataset:size:1".toUri() + entity.id, sizeExpandedName, "urn:ngsi-ld:Dataset:size:1".toUri() ).value ) } @@ -138,9 +235,9 @@ class Neo4jRepositoryTests : WithNeo4jContainer { beekeeperUri, listOf("Beekeeper"), mutableListOf( - Property(name = "https://uri.etsi.org/ngsi-ld/size", value = 100L), + Property(name = sizeExpandedName, value = 100L), Property( - name = "https://uri.etsi.org/ngsi-ld/size", + name = sizeExpandedName, value = 200L, datasetId = "urn:ngsi-ld:Dataset:size:1".toUri() ) @@ -163,18 +260,18 @@ class Neo4jRepositoryTests : WithNeo4jContainer { neo4jRepository.updateEntityPropertyInstance( EntitySubjectNode(entity.id), - "https://uri.etsi.org/ngsi-ld/size", + sizeExpandedName, newProperty.instances[0] ) assertEquals( 100L, - propertyRepository.getPropertyOfSubject(entity.id, "https://uri.etsi.org/ngsi-ld/size").value + propertyRepository.getPropertyOfSubject(entity.id, sizeExpandedName).value ) assertEquals( 300L, propertyRepository.getPropertyOfSubject( - entity.id, "https://uri.etsi.org/ngsi-ld/size", "urn:ngsi-ld:Dataset:size:1".toUri() + entity.id, sizeExpandedName, "urn:ngsi-ld:Dataset:size:1".toUri() ).value ) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index 073a51ec2..e77cb81d4 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -171,8 +171,9 @@ object JsonLdUtils { fun expandValueAsListOfMap(value: Any): List>> = value as List>> + /** - * Extract the actual value (@value) of a given property from the properties map of an expanded property. + * Extract the actual value (@value) of a property from the properties map of an expanded property. * * @param value a map similar to: * { @@ -196,19 +197,29 @@ object JsonLdUtils { if (intermediateList.size == 1) { val firstListEntry = intermediateList[0] val finalValueType = firstListEntry[JSONLD_TYPE] - if (finalValueType != null) { - val finalValue = String::class.safeCast(firstListEntry[JSONLD_VALUE_KW]) - when (finalValueType) { - NGSILD_DATE_TIME_TYPE -> ZonedDateTime.parse(finalValue) - NGSILD_DATE_TYPE -> LocalDate.parse(finalValue) - NGSILD_TIME_TYPE -> LocalTime.parse(finalValue) - else -> firstListEntry[JSONLD_VALUE_KW] + when { + finalValueType != null -> { + val finalValue = String::class.safeCast(firstListEntry[JSONLD_VALUE_KW]) + when (finalValueType) { + NGSILD_DATE_TIME_TYPE -> ZonedDateTime.parse(finalValue) + NGSILD_DATE_TYPE -> LocalDate.parse(finalValue) + NGSILD_TIME_TYPE -> LocalTime.parse(finalValue) + else -> firstListEntry[JSONLD_VALUE_KW] + } + } + firstListEntry[JSONLD_VALUE_KW] != null -> { + firstListEntry[JSONLD_VALUE_KW] + } + firstListEntry[JSONLD_ID] != null -> { + // Used to get the value of datasetId property, + // since it is mapped to "@id" key rather than "@value" + firstListEntry[JSONLD_ID] + } + else -> { + // it is a map / JSON object, keep it as is + // {https://uri.etsi.org/ngsi-ld/default-context/key=[{@value=value}], ...} + firstListEntry } - } else if (firstListEntry[JSONLD_VALUE_KW] != null) { - firstListEntry[JSONLD_VALUE_KW] - } else { - // Used to get the value of datasetId property, since it is mapped to "@id" key rather than "@value" - firstListEntry[JSONLD_ID] } } else { intermediateList.map { diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/model/NgsiLdEntityTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/model/NgsiLdEntityTests.kt index 797cd22b6..c5af7c22a 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/model/NgsiLdEntityTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/model/NgsiLdEntityTests.kt @@ -1,6 +1,7 @@ package com.egm.stellio.shared.model import com.egm.stellio.shared.util.DEFAULT_CONTEXTS +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_KW import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdFragment import com.egm.stellio.shared.util.toUri @@ -158,6 +159,66 @@ class NgsiLdEntityTests { assertEquals("Open", ngsiLdPropertyInstance.value) } + @Test + fun `it should parse an entity with a property whose name contains a colon`() { + val rawEntity = + """ + { + "id": "urn:ngsi-ld:Device:01234", + "type": "Device", + "prefix:name": { + "type": "Property", + "value": "Open" + } + } + """.trimIndent() + + val ngsiLdEntity = expandJsonLdEntity(rawEntity, DEFAULT_CONTEXTS).toNgsiLdEntity() + + val ngsiLdProperty = ngsiLdEntity.properties[0] + assertEquals("prefix:name", ngsiLdProperty.name) + } + + @Test + fun `it should parse an entity with a property having a JSON object value`() { + val rawEntity = + """ + { + "id": "urn:ngsi-ld:Device:01234", + "type": "Device", + "deviceState": { + "type": "Property", + "value": { + "state1": "open", + "state2": "closed" + } + } + } + """.trimIndent() + + val ngsiLdEntity = expandJsonLdEntity(rawEntity, DEFAULT_CONTEXTS).toNgsiLdEntity() + + assertEquals(1, ngsiLdEntity.properties.size) + val ngsiLdProperty = ngsiLdEntity.properties[0] + assertEquals("https://uri.fiware.org/ns/data-models#deviceState", ngsiLdProperty.name) + assertEquals(1, ngsiLdProperty.instances.size) + val ngsiLdPropertyInstance = ngsiLdProperty.instances[0] + assertTrue(ngsiLdPropertyInstance.value is Map<*, *>) + val valueMap = ngsiLdPropertyInstance.value as Map + assertEquals(2, valueMap.size) + assertEquals( + setOf( + "https://uri.etsi.org/ngsi-ld/default-context/state1", + "https://uri.etsi.org/ngsi-ld/default-context/state2" + ), + valueMap.keys + ) + assertEquals( + listOf(mapOf(JSONLD_VALUE_KW to "open")), + valueMap["https://uri.etsi.org/ngsi-ld/default-context/state1"] + ) + } + @Test fun `it should parse an entity with a property having all core fields`() { val rawEntity = diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/EntityEventListenerService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/EntityEventListenerService.kt index d4fc94857..088040c41 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/EntityEventListenerService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/EntityEventListenerService.kt @@ -24,7 +24,7 @@ class EntityEventListenerService( entityEvent.getEntity(), entityEvent.contexts ) - is EntityDeleteEvent -> logger.info("Entity delete operation is not yet implemented") + is EntityDeleteEvent -> logger.debug("Nothing to do on entity delete operation") is AttributeAppendEvent -> logger.info("Attribute append operation is not yet implemented") is AttributeReplaceEvent -> handleEntityEvent( deserializeObject(entityEvent.operationPayload).keys, @@ -36,7 +36,7 @@ class EntityEventListenerService( entityEvent.getEntity(), entityEvent.contexts ) - is AttributeDeleteEvent -> logger.info("Attribute delete operation is not yet implemented") + is AttributeDeleteEvent -> logger.info("Nothing to do on attribute delete operation") } } From 1b89ca2c3fc7544708ae2f01d311f44f4277aa05 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Tue, 27 Jul 2021 12:23:43 +0200 Subject: [PATCH 27/56] chore(subscription): do not create a new jackson mapper in each method --- .../com/egm/stellio/subscription/utils/ParsingUtils.kt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt index 5fcd130de..df19e15ea 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/utils/ParsingUtils.kt @@ -5,6 +5,7 @@ import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.subscription.model.EndpointInfo import com.egm.stellio.subscription.model.EntityInfo import com.egm.stellio.subscription.model.Subscription +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import org.slf4j.LoggerFactory @@ -13,8 +14,9 @@ object ParsingUtils { private val logger = LoggerFactory.getLogger(javaClass) + private val mapper: ObjectMapper = jacksonObjectMapper() + fun parseSubscription(input: String, context: List): Subscription { - val mapper = jacksonObjectMapper() val rawParsedData = mapper.readTree(input) as ObjectNode if (rawParsedData.get("@context") != null) rawParsedData.remove("@context") @@ -30,7 +32,6 @@ object ParsingUtils { } fun parseSubscriptionUpdate(input: String, context: List): Pair, List> { - val mapper = jacksonObjectMapper() val parsedSubscription: Map> = mapper.readValue( input, mapper.typeFactory.constructMapLikeType( @@ -42,7 +43,6 @@ object ParsingUtils { } fun parseEntityInfo(input: Map, contexts: List?): EntityInfo { - val mapper = jacksonObjectMapper() val entityInfo = mapper.convertValue(input, EntityInfo::class.java) entityInfo.type = JsonLdUtils.expandJsonLdKey(entityInfo.type, contexts!!)!! return entityInfo @@ -50,7 +50,6 @@ object ParsingUtils { fun parseEndpointInfo(input: String?): List? { input?.let { - val mapper = jacksonObjectMapper() return mapper.readValue( input, mapper.typeFactory.constructCollectionType(List::class.java, EndpointInfo::class.java) @@ -60,12 +59,10 @@ object ParsingUtils { } fun endpointInfoToString(input: List?): String { - val mapper = jacksonObjectMapper() return mapper.writeValueAsString(input) } fun endpointInfoMapToString(input: List>?): String { - val mapper = jacksonObjectMapper() return mapper.writeValueAsString(input) } From 77461fadc603d10ffaefda2759c9482dab4bae4f Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Wed, 28 Jul 2021 11:08:47 +0200 Subject: [PATCH 28/56] fix(subscription): decode q parameter in subscriptions #470 (#471) --- .../main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt | 4 ---- .../main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt | 1 - .../kotlin/com/egm/stellio/entity/web/EntityTypeHandler.kt | 1 - .../src/main/kotlin/com/egm/stellio/shared/util/HttpUtils.kt | 4 ++++ .../egm/stellio/subscription/service/NotificationService.kt | 3 ++- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt index 468fb1f6c..8e91697bb 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/util/ParsingUtils.kt @@ -1,7 +1,6 @@ package com.egm.stellio.entity.util import com.egm.stellio.shared.model.OperationNotSupportedException -import java.net.URLDecoder import java.time.LocalDate import java.time.LocalTime import java.time.ZonedDateTime @@ -37,9 +36,6 @@ fun extractComparisonParametersFromQuery(queryTerm: String): Triple Date: Wed, 4 Aug 2021 18:04:06 +0200 Subject: [PATCH 29/56] feat: upgrade TimescaleDB to 2.3.0 with PostgreSQL 13 (#475) * for running instances, procedure is documented on https://stellio.readthedocs.io/en/latest/admin/upgrading_to_1.0.0.html#upgrade-to-timescale-23-postgresql-13 --- docker-compose.yml | 2 +- search-service/docker-compose.yml | 2 +- .../com/egm/stellio/search/support/WithTimescaleContainer.kt | 2 +- subscription-service/docker-compose.yml | 2 +- .../egm/stellio/subscription/support/WithTimescaleContainer.kt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 286dfae64..50e20692a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,7 @@ services: - 7474:7474 - 7687:7687 postgres: - image: stellio/stellio-timescale-postgis:1.7.2-pg11 + image: stellio/stellio-timescale-postgis:2.3.0-pg13 container_name: stellio-postgres environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} diff --git a/search-service/docker-compose.yml b/search-service/docker-compose.yml index 33bb082b2..1b225ba39 100644 --- a/search-service/docker-compose.yml +++ b/search-service/docker-compose.yml @@ -23,7 +23,7 @@ services: depends_on: - zookeeper postgres: - image: stellio/stellio-timescale-postgis:1.7.2-pg11 + image: stellio/stellio-timescale-postgis:2.3.0-pg13 environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - "POSTGRES_MULTIPLE_DATABASES=${STELLIO_SEARCH_DB_DATABASE},${STELLIO_SEARCH_DB_USER},${STELLIO_SEARCH_DB_PASSWORD}" diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt index f928ccf8f..60ca5b0e7 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithTimescaleContainer.kt @@ -17,7 +17,7 @@ interface WithTimescaleContainer { private const val DB_PASSWORD = "stellio_search_db_password" private val timescaleImage: DockerImageName = - DockerImageName.parse("stellio/stellio-timescale-postgis:1.7.2-pg11") + DockerImageName.parse("stellio/stellio-timescale-postgis:2.3.0-pg13") .asCompatibleSubstituteFor("postgres") @Container diff --git a/subscription-service/docker-compose.yml b/subscription-service/docker-compose.yml index 0d1085933..1dbfd87c3 100644 --- a/subscription-service/docker-compose.yml +++ b/subscription-service/docker-compose.yml @@ -23,7 +23,7 @@ services: depends_on: - zookeeper postgres: - image: stellio/stellio-timescale-postgis:1.7.2-pg11 + image: stellio/stellio-timescale-postgis:2.3.0-pg13 environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - "POSTGRES_MULTIPLE_DATABASES=${STELLIO_SUBSCRIPTION_DB_DATABASE},${STELLIO_SUBSCRIPTION_DB_USER},${STELLIO_SUBSCRIPTION_DB_PASSWORD}" diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt index 7c6530cfc..918b7e067 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithTimescaleContainer.kt @@ -17,7 +17,7 @@ interface WithTimescaleContainer { private const val DB_PASSWORD = "stellio_subscription_db_password" private val timescaleImage: DockerImageName = - DockerImageName.parse("stellio/stellio-timescale-postgis:1.7.2-pg11") + DockerImageName.parse("stellio/stellio-timescale-postgis:2.3.0-pg13") .asCompatibleSubstituteFor("postgres") @Container From ae73c0e84cb202514981e1dd7a592860ee1e7835 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Aug 2021 06:35:51 +0200 Subject: [PATCH 30/56] chore(deps): bump json-path from 2.5.0 to 2.6.0 (#454) Bumps [json-path](https://github.com/jayway/JsonPath) from 2.5.0 to 2.6.0. - [Release notes](https://github.com/jayway/JsonPath/releases) - [Changelog](https://github.com/json-path/JsonPath/blob/master/changelog.md) - [Commits](https://github.com/jayway/JsonPath/compare/json-path-2.5.0...json-path-2.6.0) --- updated-dependencies: - dependency-name: com.jayway.jsonpath:json-path dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- subscription-service/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index 4a7b9bd6f..f8ec47abb 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -15,7 +15,7 @@ dependencies { implementation("org.springframework:spring-jdbc") implementation("org.flywaydb:flyway-core") implementation("io.r2dbc:r2dbc-postgresql") - implementation("com.jayway.jsonpath:json-path:2.5.0") + implementation("com.jayway.jsonpath:json-path:2.6.0") implementation(project(":shared")) // firebase SDK implementation("com.google.firebase:firebase-admin:6.12.2") From 6b9e155ec6041e18cd64d366b55cbd6e6fd711e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Aug 2021 08:04:27 +0200 Subject: [PATCH 31/56] chore(deps): bump jsonld-java from 0.13.2 to 0.13.3 (#476) Bumps [jsonld-java](https://github.com/jsonld-java/jsonld-java) from 0.13.2 to 0.13.3. - [Release notes](https://github.com/jsonld-java/jsonld-java/releases) - [Commits](https://github.com/jsonld-java/jsonld-java/compare/v0.13.2...v0.13.3) --- updated-dependencies: - dependency-name: com.github.jsonld-java:jsonld-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index dde1eae74..2700970ff 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -63,7 +63,7 @@ subprojects { implementation("org.springframework.kafka:spring-kafka") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") - implementation("com.github.jsonld-java:jsonld-java:0.13.2") + implementation("com.github.jsonld-java:jsonld-java:0.13.3") implementation("io.arrow-kt:arrow-fx:0.10.4") implementation("io.arrow-kt:arrow-syntax:0.10.4") From aadcae0bb0a351c98016f52b08759c4dcd166732 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 5 Aug 2021 16:23:05 +0200 Subject: [PATCH 32/56] feat(search): reimplement add instances to temporal entity #71 (#478) --- search-service/config/detekt/baseline.xml | 1 + .../service/AttributeInstanceService.kt | 24 ++- .../search/web/TemporalEntityHandler.kt | 43 ++--- .../service/AttributeInstanceServiceTests.kt | 66 ++++---- .../search/web/TemporalEntityHandlerTests.kt | 151 ++++++++++++++++-- ...ment_many_attributes_many_instances.jsonld | 30 ++++ ...agment_many_attributes_one_instance.jsonld | 18 +++ ...agment_one_attribute_many_instances.jsonld | 16 ++ ...fragment_one_attribute_one_instance.jsonld | 10 ++ .../test/resources/ngsild/observation.jsonld | 8 - .../egm/stellio/shared/util/JsonLdUtils.kt | 3 + 11 files changed, 283 insertions(+), 87 deletions(-) create mode 100644 search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_many_instances.jsonld create mode 100644 search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_one_instance.jsonld create mode 100644 search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_many_instances.jsonld create mode 100644 search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_one_instance.jsonld delete mode 100644 search-service/src/test/resources/ngsild/observation.jsonld diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml index cc39b71f3..fef8ed57e 100644 --- a/search-service/config/detekt/baseline.xml +++ b/search-service/config/detekt/baseline.xml @@ -3,6 +3,7 @@ LargeClass:EntityEventListenerServiceTest.kt$EntityEventListenerServiceTest + LargeClass:TemporalEntityHandlerTests.kt$TemporalEntityHandlerTests LongMethod:ParameterizedTests.kt$ParameterizedTests.Companion$@JvmStatic fun rawResultsProvider(): Stream<Arguments> LongParameterList:AttributeInstance.kt$AttributeInstance.Companion$( temporalEntityAttribute: UUID, instanceId: URI? = null, observedAt: ZonedDateTime, value: String? = null, measuredValue: Double? = null, payload: Map<String, Any> ) LongParameterList:EntityEventListenerService.kt$EntityEventListenerService$( entityId: URI, expandedAttributeName: String, datasetId: URI?, attributeValuesNode: JsonNode, updatedEntity: String, contexts: List<String> ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index 69539e55c..0721cfcb0 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -4,11 +4,12 @@ import com.egm.stellio.search.model.* import com.egm.stellio.search.util.valueToDoubleOrNull import com.egm.stellio.search.util.valueToStringOrNull import com.egm.stellio.shared.model.BadRequestDataException -import com.egm.stellio.shared.util.JsonLdUtils.EGM_OBSERVED_BY +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_CONTEXT +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE +import com.egm.stellio.shared.util.JsonLdUtils.compactFragment import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMap import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsDateTime -import com.egm.stellio.shared.util.extractAttributeInstanceFromCompactedEntity import io.r2dbc.postgresql.codec.Json import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind @@ -44,27 +45,24 @@ class AttributeInstanceService( .rowsUpdated() .onErrorReturn(-1) - // TODO not totally compatible with the specification - // it should accept an array of attribute instances - fun addAttributeInstances( + @Transactional + fun addAttributeInstance( temporalEntityAttributeUuid: UUID, attributeKey: String, attributeValues: Map>, - parsedPayload: Map + contexts: List ): Mono { val attributeValue = getPropertyValueFromMap(attributeValues, NGSILD_PROPERTY_VALUE) - ?: throw BadRequestDataException("Value cannot be null") + ?: throw BadRequestDataException("Attribute $attributeKey has an instance without a value") + val observedAt = getPropertyValueFromMapAsDateTime(attributeValues, NGSILD_OBSERVED_AT_PROPERTY) + ?: throw BadRequestDataException("Attribute $attributeKey has an instance without an observed date") val attributeInstance = AttributeInstance( temporalEntityAttribute = temporalEntityAttributeUuid, - observedAt = getPropertyValueFromMapAsDateTime(attributeValues, EGM_OBSERVED_BY)!!, + observedAt = observedAt, value = valueToStringOrNull(attributeValue), measuredValue = valueToDoubleOrNull(attributeValue), - payload = extractAttributeInstanceFromCompactedEntity( - parsedPayload, - attributeKey, - null - ) + payload = compactFragment(attributeValues, contexts).minus(JSONLD_CONTEXT) ) return create(attributeInstance) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt index 797936d66..71f363605 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt @@ -11,12 +11,12 @@ import com.egm.stellio.search.service.QueryService import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.BadRequestDataResponse +import com.egm.stellio.shared.model.getDatasetId import com.egm.stellio.shared.util.JSON_LD_CONTENT_TYPE import com.egm.stellio.shared.util.JsonLdUtils.addContextsToEntity import com.egm.stellio.shared.util.JsonLdUtils.compactTerm import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdFragment -import com.egm.stellio.shared.util.JsonLdUtils.expandValueAsMap -import com.egm.stellio.shared.util.JsonUtils +import com.egm.stellio.shared.util.JsonLdUtils.expandValueAsListOfMap import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.OptionsParamValue import com.egm.stellio.shared.util.buildGetSuccessResponse @@ -55,36 +55,39 @@ class TemporalEntityHandler( ) { /** - * Mirror of what we receive from Kafka. - * - * Implements 6.20.3.1 + * Implements 6.20.3.1 - Add attributes to Temporal Representation of Entity */ @PostMapping("/{entityId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) - suspend fun addAttrs( + suspend fun addAttributes( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: String, @RequestBody requestBody: Mono ): ResponseEntity<*> { val body = requestBody.awaitFirst() - val parsedBody = JsonUtils.deserializeObject(body) val contexts = checkAndGetContext(httpHeaders, body) val jsonLdAttributes = expandJsonLdFragment(body, contexts) jsonLdAttributes - .forEach { - val compactedAttributeName = compactTerm(it.key, contexts) - val temporalEntityAttributeUuid = temporalEntityAttributeService.getForEntityAndAttribute( - entityId.toUri(), - it.key - ).awaitFirst() - - attributeInstanceService.addAttributeInstances( - temporalEntityAttributeUuid, - compactedAttributeName, - expandValueAsMap(it.value), - parsedBody - ).awaitFirst() + .forEach { attributeEntry -> + val attributeInstances = expandValueAsListOfMap(attributeEntry.value) + attributeInstances.forEach { attributeInstance -> + val datasetId = attributeInstance.getDatasetId() + val temporalEntityAttributeUuid = temporalEntityAttributeService.getForEntityAndAttribute( + entityId.toUri(), + attributeEntry.key, + datasetId + ).awaitFirst() + + val compactedAttributeName = compactTerm(attributeEntry.key, contexts) + attributeInstanceService.addAttributeInstance( + temporalEntityAttributeUuid, + compactedAttributeName, + attributeInstance, + contexts + ).awaitFirst() + } } + return ResponseEntity.status(HttpStatus.NO_CONTENT).build() } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt index b7fdbe367..23a9a1f58 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt @@ -7,16 +7,19 @@ import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.model.BadRequestDataException -import com.egm.stellio.shared.util.JsonLdUtils.EGM_OBSERVED_BY +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_KW +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CORE_CONTEXT +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATE_TIME_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE -import com.egm.stellio.shared.util.JsonUtils import com.egm.stellio.shared.util.matchContent import com.egm.stellio.shared.util.toUri import io.mockk.confirmVerified import io.mockk.spyk import io.mockk.verify import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -363,22 +366,13 @@ class AttributeInstanceServiceTests : WithTimescaleContainer { } @Test - fun `it should create an attribute instance if it has a non null value or measuredValue`() { + fun `it should create an attribute instance if it has a non null value`() { val attributeInstanceService = spyk(AttributeInstanceService(databaseClient), recordPrivateCalls = true) - val observationPayload = - """ - { - "outgoing": { - "type": "Property", - "value": 550.0 - } - } - """.trimIndent() - val parsedObservationPayload = JsonUtils.deserializeObject(observationPayload) val attributeValues = mapOf( - EGM_OBSERVED_BY to listOf( + NGSILD_OBSERVED_AT_PROPERTY to listOf( mapOf( - "@value" to Instant.parse("2015-10-18T11:20:30.000001Z").atZone(ZoneOffset.UTC) + JSONLD_VALUE_KW to "2015-10-18T11:20:30.000001Z", + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE ) ), NGSILD_PROPERTY_VALUE to listOf( @@ -388,19 +382,27 @@ class AttributeInstanceServiceTests : WithTimescaleContainer { ) ) - attributeInstanceService.addAttributeInstances( + attributeInstanceService.addAttributeInstance( temporalEntityAttribute.id, "outgoing", attributeValues, - parsedObservationPayload + listOf(NGSILD_CORE_CONTEXT) ).block() verify { attributeInstanceService["create"]( match { - it.measuredValue == 550.0 && + it.observedAt.toString() == "2015-10-18T11:20:30.000001Z" && + it.value == null && + it.measuredValue == 550.0 && it.payload.matchContent( - """{"type": "Property","value": 550.0, "instanceId": "${it.instanceId}"}""" + """ + { + "value": 550.0, + "observedAt": "2015-10-18T11:20:30.000001Z", + "instanceId": "${it.instanceId}" + } + """.trimIndent() ) } ) @@ -411,33 +413,25 @@ class AttributeInstanceServiceTests : WithTimescaleContainer { @Test fun `it should not create an attribute instance if it has a null value and null measuredValue`() { - val observationPayload = - """ - { - "outgoing": { - "type": "Property", - "observedBy": "2015-10-18T11:20:30.000001Z" - } - } - """.trimIndent() - val parsedObservationPayload = JsonUtils.deserializeObject(observationPayload) - + val attributeInstanceService = spyk(AttributeInstanceService(databaseClient), recordPrivateCalls = true) val attributeValues = mapOf( - EGM_OBSERVED_BY to listOf( + NGSILD_OBSERVED_AT_PROPERTY to listOf( mapOf( - "@value" to Instant.parse("2015-10-18T11:20:30.000001Z").atZone(ZoneOffset.UTC) + JSONLD_VALUE_KW to "2015-10-18T11:20:30.000001Z", + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE ) - ) + ), ) - assertThrows("Value cannot be null") { - attributeInstanceService.addAttributeInstances( + val exception = assertThrows("It should have thrown a BadRequestDataException") { + attributeInstanceService.addAttributeInstance( temporalEntityAttribute.id, "outgoing", attributeValues, - parsedObservationPayload + listOf(NGSILD_CORE_CONTEXT) ) } + assertEquals("Attribute outgoing has an instance without a value", exception.message) } @Test diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index 75b162152..751af6a2c 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -81,21 +81,23 @@ class TemporalEntityHandlerTests { } @Test - fun `it should return a 204 if temporal entity fragment is valid`() { - val jsonLdObservation = loadSampleData("observation.jsonld") - val parsedJsonLdObservation = deserializeObject(jsonLdObservation) + fun `it should correctly handle an attribute with one instance`() { + val entityTemporalFragment = + loadSampleData("fragments/temporal_entity_fragment_one_attribute_one_instance.jsonld") val temporalEntityAttributeUuid = UUID.randomUUID() - every { temporalEntityAttributeService.getForEntityAndAttribute(any(), any()) } returns Mono.just( - temporalEntityAttributeUuid - ) - every { attributeInstanceService.addAttributeInstances(any(), any(), any(), any()) } returns Mono.just(1) + every { temporalEntityAttributeService.getForEntityAndAttribute(any(), any()) } answers { + Mono.just(temporalEntityAttributeUuid) + } + every { attributeInstanceService.addAttributeInstance(any(), any(), any(), any()) } answers { + Mono.just(1) + } webClient.post() .uri("/ngsi-ld/v1/temporal/entities/$entityUri/attrs") .header("Link", buildContextLinkHeader(APIC_COMPOUND_CONTEXT)) .contentType(MediaType.APPLICATION_JSON) - .body(BodyInserters.fromValue(jsonLdObservation)) + .body(BodyInserters.fromValue(entityTemporalFragment)) .exchange() .expectStatus().isNoContent @@ -106,13 +108,142 @@ class TemporalEntityHandlerTests { ) } verify { - attributeInstanceService.addAttributeInstances( + attributeInstanceService.addAttributeInstance( eq(temporalEntityAttributeUuid), eq("incoming"), match { it.size == 4 }, - parsedJsonLdObservation + listOf(APIC_COMPOUND_CONTEXT) + ) + } + confirmVerified(temporalEntityAttributeService) + confirmVerified(attributeInstanceService) + } + + @Test + fun `it should correctly handle an attribute with many instances`() { + val entityTemporalFragment = + loadSampleData("fragments/temporal_entity_fragment_one_attribute_many_instances.jsonld") + val temporalEntityAttributeUuid = UUID.randomUUID() + + every { temporalEntityAttributeService.getForEntityAndAttribute(any(), any()) } answers { + Mono.just(temporalEntityAttributeUuid) + } + every { attributeInstanceService.addAttributeInstance(any(), any(), any(), any()) } answers { + Mono.just(1) + } + + webClient.post() + .uri("/ngsi-ld/v1/temporal/entities/$entityUri/attrs") + .header("Link", buildContextLinkHeader(APIC_COMPOUND_CONTEXT)) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromValue(entityTemporalFragment)) + .exchange() + .expectStatus().isNoContent + + verify { + temporalEntityAttributeService.getForEntityAndAttribute( + eq(entityUri), + eq("https://ontology.eglobalmark.com/apic#incoming") + ) + } + verify(exactly = 2) { + attributeInstanceService.addAttributeInstance( + eq(temporalEntityAttributeUuid), + eq("incoming"), + match { + it.size == 4 + }, + listOf(APIC_COMPOUND_CONTEXT) + ) + } + confirmVerified(temporalEntityAttributeService) + confirmVerified(attributeInstanceService) + } + + @Test + fun `it should correctly handle many attributes with one instance`() { + val entityTemporalFragment = + loadSampleData("fragments/temporal_entity_fragment_many_attributes_one_instance.jsonld") + val temporalEntityAttributeUuid = UUID.randomUUID() + + every { temporalEntityAttributeService.getForEntityAndAttribute(any(), any()) } answers { + Mono.just(temporalEntityAttributeUuid) + } + every { attributeInstanceService.addAttributeInstance(any(), any(), any(), any()) } answers { + Mono.just(1) + } + + webClient.post() + .uri("/ngsi-ld/v1/temporal/entities/$entityUri/attrs") + .header("Link", buildContextLinkHeader(APIC_COMPOUND_CONTEXT)) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromValue(entityTemporalFragment)) + .exchange() + .expectStatus().isNoContent + + verify(exactly = 2) { + temporalEntityAttributeService.getForEntityAndAttribute( + eq(entityUri), + or( + "https://ontology.eglobalmark.com/apic#incoming", + "https://ontology.eglobalmark.com/apic#outgoing" + ) + ) + } + verify(exactly = 2) { + attributeInstanceService.addAttributeInstance( + eq(temporalEntityAttributeUuid), + or("incoming", "outgoing"), + match { + it.size == 4 + }, + listOf(APIC_COMPOUND_CONTEXT) + ) + } + confirmVerified(temporalEntityAttributeService) + confirmVerified(attributeInstanceService) + } + + @Test + fun `it should correctly handle many attributes with many instances`() { + val entityTemporalFragment = + loadSampleData("fragments/temporal_entity_fragment_many_attributes_many_instances.jsonld") + val temporalEntityAttributeUuid = UUID.randomUUID() + + every { temporalEntityAttributeService.getForEntityAndAttribute(any(), any()) } answers { + Mono.just(temporalEntityAttributeUuid) + } + every { attributeInstanceService.addAttributeInstance(any(), any(), any(), any()) } answers { + Mono.just(1) + } + + webClient.post() + .uri("/ngsi-ld/v1/temporal/entities/$entityUri/attrs") + .header("Link", buildContextLinkHeader(APIC_COMPOUND_CONTEXT)) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromValue(entityTemporalFragment)) + .exchange() + .expectStatus().isNoContent + + verify(exactly = 4) { + temporalEntityAttributeService.getForEntityAndAttribute( + eq(entityUri), + or( + "https://ontology.eglobalmark.com/apic#incoming", + "https://ontology.eglobalmark.com/apic#outgoing" + ) + ) + } + verify(exactly = 4) { + attributeInstanceService.addAttributeInstance( + eq(temporalEntityAttributeUuid), + or("incoming", "outgoing"), + match { + it.size == 4 + }, + listOf(APIC_COMPOUND_CONTEXT) ) } confirmVerified(temporalEntityAttributeService) diff --git a/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_many_instances.jsonld b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_many_instances.jsonld new file mode 100644 index 000000000..86386ae53 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_many_instances.jsonld @@ -0,0 +1,30 @@ +{ + "incoming": [ + { + "type": "Property", + "value": 20.7, + "unitCode": "CEL", + "observedAt": "2019-10-18T07:31:39.77Z" + }, + { + "type": "Property", + "value": 21.7, + "unitCode": "CEL", + "observedAt": "2019-10-19T07:31:39.77Z" + } + ], + "outgoing": [ + { + "type": "Property", + "value": 10.3, + "unitCode": "CEL", + "observedAt": "2019-10-19T07:31:39.77Z" + }, + { + "type": "Property", + "value": 12.3, + "unitCode": "CEL", + "observedAt": "2019-10-20T07:31:39.77Z" + } + ] +} diff --git a/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_one_instance.jsonld b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_one_instance.jsonld new file mode 100644 index 000000000..2c11640a2 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_many_attributes_one_instance.jsonld @@ -0,0 +1,18 @@ +{ + "incoming": [ + { + "type": "Property", + "value": 20.7, + "unitCode": "CEL", + "observedAt": "2019-10-18T07:31:39.77Z" + } + ], + "outgoing": [ + { + "type": "Property", + "value": 10.3, + "unitCode": "CEL", + "observedAt": "2019-10-19T07:31:39.77Z" + } + ] +} diff --git a/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_many_instances.jsonld b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_many_instances.jsonld new file mode 100644 index 000000000..c80f1e470 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_many_instances.jsonld @@ -0,0 +1,16 @@ +{ + "incoming": [ + { + "type": "Property", + "value": 20.7, + "unitCode": "CEL", + "observedAt": "2019-10-18T07:31:39.77Z" + }, + { + "type": "Property", + "value": 22.7, + "unitCode": "CEL", + "observedAt": "2019-10-20T07:31:39.77Z" + } + ] +} diff --git a/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_one_instance.jsonld b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_one_instance.jsonld new file mode 100644 index 000000000..fb6d28d97 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/temporal_entity_fragment_one_attribute_one_instance.jsonld @@ -0,0 +1,10 @@ +{ + "incoming": [ + { + "type": "Property", + "value": 20.7, + "unitCode": "CEL", + "observedAt": "2019-10-18T07:31:39.77Z" + } + ] +} diff --git a/search-service/src/test/resources/ngsild/observation.jsonld b/search-service/src/test/resources/ngsild/observation.jsonld deleted file mode 100644 index 6ea403640..000000000 --- a/search-service/src/test/resources/ngsild/observation.jsonld +++ /dev/null @@ -1,8 +0,0 @@ -{ - "incoming": { - "type": "Property", - "value": 20.7, - "unitCode": "CEL", - "observedAt": "2019-10-18T07:31:39.77Z" - } -} diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index e77cb81d4..fb2efd9d8 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -298,6 +298,9 @@ object JsonLdUtils { return compactedFragment.keys.first() } + fun compactFragment(value: Map, context: List): Map = + JsonLdProcessor.compact(value, mapOf(JSONLD_CONTEXT to context), JsonLdOptions()) + fun compactAndStringifyFragment(key: String, value: Any, context: List): String = compactAndStringifyFragment(mapOf(key to value), context) From 90407f4d465448eeddd713e292f5b30915d747fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Aug 2021 09:00:28 +0200 Subject: [PATCH 33/56] chore(deps): bump wiremock-standalone from 2.25.1 to 2.27.2 (#477) Bumps [wiremock-standalone](https://github.com/tomakehurst/wiremock) from 2.25.1 to 2.27.2. - [Release notes](https://github.com/tomakehurst/wiremock/releases) - [Commits](https://github.com/tomakehurst/wiremock/compare/2.25.1...2.27.2) --- updated-dependencies: - dependency-name: com.github.tomakehurst:wiremock-standalone dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- search-service/build.gradle.kts | 2 +- subscription-service/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search-service/build.gradle.kts b/search-service/build.gradle.kts index c087efc63..b0ccfd182 100644 --- a/search-service/build.gradle.kts +++ b/search-service/build.gradle.kts @@ -23,7 +23,7 @@ dependencies { runtimeOnly("org.postgresql:postgresql") - testImplementation("com.github.tomakehurst:wiremock-standalone:2.25.1") + testImplementation("com.github.tomakehurst:wiremock-standalone:2.27.2") testImplementation("org.testcontainers:postgresql") testImplementation("org.testcontainers:r2dbc") testImplementation(testFixtures(project(":shared"))) diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index f8ec47abb..610b21971 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -24,7 +24,7 @@ dependencies { runtimeOnly("org.postgresql:postgresql") - testImplementation("com.github.tomakehurst:wiremock-standalone:2.25.1") + testImplementation("com.github.tomakehurst:wiremock-standalone:2.27.2") testImplementation("org.testcontainers:postgresql") testImplementation("org.testcontainers:r2dbc") testImplementation(testFixtures(project(":shared"))) From 8911d06f472bedf6c2e5acbadcd4ae8a778a3ff2 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 8 Aug 2021 08:44:16 +0200 Subject: [PATCH 34/56] feat(common): return a LdContextNotAvailable error if the JSON-LD context is not resolvable (#480) --- .../com/egm/stellio/entity/web/EntityHandlerTests.kt | 10 +++++----- .../com/egm/stellio/shared/model/APiExceptions.kt | 1 + .../com/egm/stellio/shared/model/ErrorResponse.kt | 6 ++---- .../kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt | 5 ++++- .../com/egm/stellio/shared/web/ExceptionHandler.kt | 4 ++++ .../com/egm/stellio/shared/util/JsonLdUtilsTests.kt | 9 +++++---- .../egm/stellio/shared/web/ExceptionHandlerTests.kt | 7 ++++--- 7 files changed, 25 insertions(+), 17 deletions(-) diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt index c7382f978..e2fddaf1b 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt @@ -1655,7 +1655,7 @@ class EntityHandlerTests { } @Test - fun `entity attributes update should return a 400 if JSON-LD context is not correct`() { + fun `entity attributes update should return a 503 if JSON-LD context is not correct`() { val payload = """ { @@ -1677,13 +1677,13 @@ class EntityHandlerTests { .contentType(MediaType.APPLICATION_JSON) .bodyValue(payload) .exchange() - .expectStatus().isBadRequest + .expectStatus().isEqualTo(HttpStatus.SERVICE_UNAVAILABLE) .expectBody().json( """ { - "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData", - "title": "The request includes input data which does not meet the requirements of the operation", - "detail": "Unexpected error while parsing payload (cause was: com.github.jsonldjava.core.JsonLdError: loading remote context failed: http://easyglobalmarket.com/contexts/diat.jsonld)" + "type": "https://uri.etsi.org/ngsi-ld/errors/LdContextNotAvailable", + "title": "A remote JSON-LD @context referenced in a request cannot be retrieved by the NGSI-LD Broker and expansion or compaction cannot be performed", + "detail": "Unable to load remote context (cause was: com.github.jsonldjava.core.JsonLdError: loading remote context failed: http://easyglobalmarket.com/contexts/diat.jsonld)" } """.trimIndent() ) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/APiExceptions.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/APiExceptions.kt index 651080ac7..b88064b99 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/APiExceptions.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/APiExceptions.kt @@ -14,3 +14,4 @@ data class TooComplexQueryException(override val message: String) : APiException data class TooManyResultsException(override val message: String) : APiException(message) data class AccessDeniedException(override val message: String) : APiException(message) data class NotImplementedException(override val message: String) : APiException(message) +data class LdContextNotAvailableException(override val message: String) : APiException(message) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt index bc3f1986f..38b66fad9 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ErrorResponse.kt @@ -47,10 +47,8 @@ data class TooManyResultsResponse(override val detail: String) : ErrorResponse( data class LdContextNotAvailableResponse(override val detail: String) : ErrorResponse( ErrorType.LD_CONTEXT_NOT_AVAILABLE.type, - """ - A remote JSON-LD @context referenced in a request cannot be retrieved by the NGSI-LD Broker and expansion or - compaction cannot be performed - """, + "A remote JSON-LD @context referenced in a request cannot be retrieved by the NGSI-LD Broker and " + + "expansion or compaction cannot be performed", detail ) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index fb2efd9d8..d6629ac31 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -520,7 +520,10 @@ fun parseAndExpandJsonLdFragment(fragment: String, jsonLdOptions: JsonLdOptions? else JsonLdProcessor.expand(parsedFragment) } catch (e: JsonLdError) { - throw BadRequestDataException("Unexpected error while parsing payload (cause was: $e)") + if (e.type == JsonLdError.Error.LOADING_REMOTE_CONTEXT_FAILED) + throw LdContextNotAvailableException("Unable to load remote context (cause was: $e)") + else + throw BadRequestDataException("Unexpected error while parsing payload (cause was: $e)") } if (expandedFragment.isEmpty()) throw BadRequestDataException("Unable to parse input payload") diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt b/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt index 694e1b9f7..17659323d 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/web/ExceptionHandler.kt @@ -48,6 +48,10 @@ class ExceptionHandler { HttpStatus.NOT_IMPLEMENTED, NotImplementedResponse(cause.message) ) + is LdContextNotAvailableException -> generateErrorResponse( + HttpStatus.SERVICE_UNAVAILABLE, + LdContextNotAvailableResponse(cause.message) + ) else -> generateErrorResponse( HttpStatus.INTERNAL_SERVER_ERROR, InternalErrorResponse("$cause") diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt index def19d2e8..6656f23d0 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonLdUtilsTests.kt @@ -3,6 +3,7 @@ package com.egm.stellio.shared.util import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.CompactedJsonLdEntity import com.egm.stellio.shared.model.InvalidRequestException +import com.egm.stellio.shared.model.LdContextNotAvailableException import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_HAS_OBJECT import com.egm.stellio.shared.util.JsonLdUtils.compact import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdFragment @@ -141,7 +142,7 @@ class JsonLdUtilsTests { } @Test - fun `it should throw a BadRequestData exception if the JSON-LD fragment is not a valid JSON-LD document`() { + fun `it should throw a LdContextNotAvailable exception if the provided JSON-LD context is not available`() { val rawEntity = """ { @@ -153,12 +154,12 @@ class JsonLdUtilsTests { } """.trimIndent() - val exception = assertThrows { + val exception = assertThrows { parseAndExpandJsonLdFragment(rawEntity) } assertEquals( - "Unexpected error while parsing payload " + - "(cause was: com.github.jsonldjava.core.JsonLdError: loading remote context failed: unknownContext)", + "Unable to load remote context (cause was: com.github.jsonldjava.core.JsonLdError: " + + "loading remote context failed: unknownContext)", exception.message ) } diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/web/ExceptionHandlerTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/web/ExceptionHandlerTests.kt index 988c06bfe..5dfb9a13b 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/web/ExceptionHandlerTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/web/ExceptionHandlerTests.kt @@ -35,7 +35,7 @@ class ExceptionHandlerTests { } @Test - fun `it should raise an error of type BadRequestData if the request payload is not a valid JSON-LD fragment`() { + fun `it should raise an error of type LdContextNotAvailable if the context is not resolvable`() { val invalidJsonPayload = """ { @@ -51,8 +51,9 @@ class ExceptionHandlerTests { .uri("/router/mockkedroute/validate-json-ld-fragment") .bodyValue(invalidJsonPayload) .exchange() - .expectStatus().isEqualTo(HttpStatus.BAD_REQUEST) + .expectStatus().isEqualTo(HttpStatus.SERVICE_UNAVAILABLE) .expectBody() - .jsonPath("$..type").isEqualTo("https://uri.etsi.org/ngsi-ld/errors/BadRequestData") + .jsonPath("$..type") + .isEqualTo("https://uri.etsi.org/ngsi-ld/errors/LdContextNotAvailable") } } From 6f952ca2d2968c48ca2003c2c1c59f1caeb73631 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 8 Aug 2021 09:15:57 +0200 Subject: [PATCH 35/56] chore(deps): bump org.springframework.boot from 2.5.2 to 2.5.3 (#479) Bumps [org.springframework.boot](https://github.com/spring-projects/spring-boot) from 2.5.2 to 2.5.3. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.5.2...v2.5.3) --- updated-dependencies: - dependency-name: org.springframework.boot dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 2700970ff..7194bf9eb 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,7 +12,7 @@ plugins { java // why did I have to add that ?! // only apply the plugin in the subprojects requiring it because it expects a Spring Boot app // and the shared lib is obviously not one - id("org.springframework.boot") version "2.5.2" apply false + id("org.springframework.boot") version "2.5.3" apply false id("io.spring.dependency-management") version "1.0.11.RELEASE" apply false kotlin("jvm") version "1.5.21" apply false kotlin("plugin.spring") version "1.5.21" apply false From 25403bb8303ee8c417e5991e7ae7d2df201bd88a Mon Sep 17 00:00:00 2001 From: Houcem Kacem Date: Mon, 9 Aug 2021 07:45:21 +0200 Subject: [PATCH 36/56] fix(common): replace page by offset in pagination queries #428 (#429) Co-authored-by: Benoit Orihuela --- entity-service/config/detekt/baseline.xml | 6 +- .../repository/Neo4jSearchRepository.kt | 6 +- .../stellio/entity/repository/QueryUtils.kt | 8 +- .../entity/repository/SearchRepository.kt | 6 +- .../StandaloneNeo4jSearchRepository.kt | 4 +- .../stellio/entity/service/EntityService.kt | 8 +- .../egm/stellio/entity/web/EntityHandler.kt | 20 ++--- .../repository/Neo4jSearchRepositoryTests.kt | 24 +++--- .../StandaloneNeo4jSearchRepositoryTests.kt | 74 +++++++++---------- .../util/QueryEntitiesParameterizedTests.kt | 8 +- .../stellio/entity/web/EntityHandlerTests.kt | 22 +++--- .../com/egm/stellio/shared/util/ApiUtils.kt | 2 +- .../egm/stellio/shared/util/PagingUtils.kt | 18 +++-- .../stellio/shared/util/PagingUtilsTests.kt | 18 ++--- .../subscription/web/SubscriptionHandler.kt | 14 ++-- .../web/SubscriptionHandlerTests.kt | 36 +++++---- 16 files changed, 144 insertions(+), 130 deletions(-) diff --git a/entity-service/config/detekt/baseline.xml b/entity-service/config/detekt/baseline.xml index 0c03ac340..676ec7fc6 100644 --- a/entity-service/config/detekt/baseline.xml +++ b/entity-service/config/detekt/baseline.xml @@ -11,9 +11,9 @@ LargeClass:StandaloneNeo4jSearchRepositoryTests.kt$StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer LongMethod:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap<String, String> ): ResponseEntity<*> LongMethod:EntityServiceTests.kt$EntityServiceTests$@Test fun `it should create a new multi attribute property`() - LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, page: Int, limit: Int, contexts: List<String> ): String - LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, page: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean ) - LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, page: Int, limit: Int, contexts: List<String>, includeSysAttrs: Boolean ) + LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, offset: Int, limit: Int, contexts: List<String> ): String + LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, offset: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean ) + LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, offset: Int, limit: Int, contexts: List<String>, includeSysAttrs: Boolean ) MaxLineLength:EntityHandlerTests.kt$EntityHandlerTests$ MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (a: MaxLineLength:Neo4jRepository.kt$Neo4jRepository$ MATCH (entity: diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt index 46535e672..ccc378228 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepository.kt @@ -19,14 +19,14 @@ class Neo4jSearchRepository( override fun getEntities( queryParams: QueryParams, userSub: String, - page: Int, + offset: Int, limit: Int, contexts: List ): Pair> { val query = if (neo4jAuthorizationService.userIsAdmin(userSub)) - QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, page, limit, contexts) + QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, offset, limit, contexts) else - QueryUtils.prepareQueryForEntitiesWithAuthentication(queryParams, page, limit, contexts) + QueryUtils.prepareQueryForEntitiesWithAuthentication(queryParams, offset, limit, contexts) val result = neo4jClient.query(query).bind(USER_PREFIX + userSub).to("userId").fetch().all() return if (limit == 0) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt index 1ca18894e..0d05e5164 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt @@ -18,7 +18,7 @@ object QueryUtils { fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, - page: Int, + offset: Int, limit: Int, contexts: List ): String { @@ -87,7 +87,7 @@ object QueryUtils { UNWIND entityIds as id RETURN id, count ORDER BY id - SKIP ${(page - 1) * limit} LIMIT $limit + SKIP $offset LIMIT $limit """.trimIndent() return """ @@ -106,7 +106,7 @@ object QueryUtils { fun prepareQueryForEntitiesWithoutAuthentication( queryParams: QueryParams, - page: Int, + offset: Int, limit: Int, contexts: List ): String { @@ -133,7 +133,7 @@ object QueryUtils { UNWIND entities as entity RETURN entity.id as id, count ORDER BY id - SKIP ${(page - 1) * limit} LIMIT $limit + SKIP $offset LIMIT $limit """.trimIndent() return """ diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt index 5f7e90159..530bddc7f 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/SearchRepository.kt @@ -11,8 +11,8 @@ interface SearchRepository { * * @param queryParams query parameters. * @param userSub to be used in authentication enabled mode to apply permissions checks. - * @param page page number for pagination. - * @param limit limit number for pagination. + * @param offset offset for pagination. + * @param limit limit for pagination. * @param contexts list of JSON-LD contexts for term to URI expansion. * @return [Pair] * @property first count of all matching entities in the database. @@ -22,7 +22,7 @@ interface SearchRepository { fun getEntities( queryParams: QueryParams, userSub: String, - page: Int, + offset: Int, limit: Int, contexts: List ): Pair> diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt index 31d96c7c4..0be972edc 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepository.kt @@ -16,11 +16,11 @@ class StandaloneNeo4jSearchRepository( override fun getEntities( queryParams: QueryParams, userSub: String, - page: Int, + offset: Int, limit: Int, contexts: List ): Pair> { - val query = QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, page, limit, contexts) + val query = QueryUtils.prepareQueryForEntitiesWithoutAuthentication(queryParams, offset, limit, contexts) val result = neo4jClient.query(query).fetch().all() return if (limit == 0) Pair( diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 0f876da35..5560f90d1 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -235,12 +235,12 @@ class EntityService( fun searchEntities( queryParams: QueryParams, userSub: String, - page: Int, + offset: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean ): Pair> = - searchEntities(queryParams, userSub, page, limit, listOf(contextLink), includeSysAttrs) + searchEntities(queryParams, userSub, offset, limit, listOf(contextLink), includeSysAttrs) /** * Search entities by type and query parameters @@ -255,7 +255,7 @@ class EntityService( fun searchEntities( queryParams: QueryParams, userSub: String, - page: Int, + offset: Int, limit: Int, contexts: List, includeSysAttrs: Boolean @@ -263,7 +263,7 @@ class EntityService( val result = searchRepository.getEntities( queryParams, userSub, - page, + offset, limit, contexts ) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt index 8777c0a6f..5a94db2b0 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt @@ -85,7 +85,7 @@ class EntityHandler( @RequestParam params: MultiValueMap ): ResponseEntity<*> { val count = params.getFirst(QUERY_PARAM_COUNT)?.toBoolean() ?: false - val page = params.getFirst(QUERY_PARAM_PAGE)?.toIntOrNull() ?: 1 + val offset = params.getFirst(QUERY_PARAM_OFFSET)?.toIntOrNull() ?: 0 val limit = params.getFirst(QUERY_PARAM_LIMIT)?.toIntOrNull() ?: applicationProperties.pagination.limitDefault val ids = params.getFirst(QUERY_PARAM_ID)?.split(",") val type = params.getFirst(QUERY_PARAM_TYPE) @@ -100,18 +100,20 @@ class EntityHandler( val mediaType = getApplicableMediaType(httpHeaders) val userId = extractSubjectOrEmpty().awaitFirst() - if (!count && (limit <= 0 || page <= 0)) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body(BadRequestDataResponse("Page number and Limit must be strictly greater than zero")) - - if (count && (limit < 0 || page <= 0)) + if (!count && (limit <= 0 || offset < 0)) return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) .body( BadRequestDataResponse( - "Page number must be strictly greater than zero and Limit must be greater than zero" + "Offset must be greater than zero and limit must be strictly greater than zero" ) ) + if (count && (limit < 0 || offset < 0)) + return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) + .body( + BadRequestDataResponse("Offset and limit must be greater than zero") + ) + if (limit > applicationProperties.pagination.limitMax) return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) .body( @@ -138,7 +140,7 @@ class EntityHandler( val countAndEntities = entityService.searchEntities( QueryParams(ids, type?.let { expandJsonLdKey(type, contextLink) }, idPattern, q?.decode(), expandedAttrs), userId, - page, + offset, limit, contextLink, includeSysAttrs @@ -172,7 +174,7 @@ class EntityHandler( "/ngsi-ld/v1/entities", params, countAndEntities.first, - page, + offset, limit ) return PagingUtils.buildPaginationResponse( diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt index 2f6b59515..22d59ef9a 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt @@ -54,7 +54,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { private val serviceAccountUri = "urn:ngsi-ld:User:01".toUri() private val expandedNameProperty = expandJsonLdKey("name", DEFAULT_CONTEXTS)!! private val sub = "01" - private val page = 1 + private val offset = 0 private val limit = 20 @AfterEach @@ -87,7 +87,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -116,7 +116,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -135,7 +135,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -172,7 +172,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { var entities = searchRepository.getEntities( queryParams, sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -182,7 +182,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { entities = searchRepository.getEntities( queryParams.copy(expandedAttrs = setOf(expandedNameProperty)), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -218,7 +218,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -249,7 +249,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -283,7 +283,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entitiesCount = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:Beekeeper:0.*2$"), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).first @@ -310,7 +310,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val countAndEntities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:Beekeeper:0.*2$"), sub, - page, + offset, 0, DEFAULT_CONTEXTS ) @@ -323,7 +323,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { @MethodSource("com.egm.stellio.entity.util.QueryEntitiesParameterizedTests#rawResultsProvider") fun `it should only return matching entities requested by pagination`( idPattern: String?, - page: Int, + offset: Int, limit: Int, expectedEntitiesIds: List ) { @@ -350,7 +350,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = idPattern), sub, - page, + offset, limit, DEFAULT_CONTEXTS ).second diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt index 39c607bf5..ad8581f9e 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt @@ -45,7 +45,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { private val partialTargetEntityUri = "urn:ngsi-ld:Entity:4567".toUri() private val expandedNameProperty = expandJsonLdKey("name", DEFAULT_CONTEXTS)!! private val userId = "" - private val page = 1 + private val offset = 0 private val limit = 20 @AfterEach @@ -64,7 +64,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"Scalpa\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -83,7 +83,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name==\"ScalpaXYZ\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -102,7 +102,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "fishNumber==500"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -121,7 +121,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "fishNumber==499"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -140,7 +140,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "fishWeight==120.50"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -159,7 +159,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "fishWeight==-120"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -177,7 +177,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "fishWeight>180.9"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -195,7 +195,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "fishWeight>=255"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -213,7 +213,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "DeadFishes", q = "name!=\"ScalpaXYZ\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -231,7 +231,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "name!=\"ScalpaXYZ\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -258,7 +258,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { q = "testedAt.observedAt>2018-12-04T00:00:00Z;testedAt.observedAt<2018-12-04T18:00:00Z" ), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -280,7 +280,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==2018-12-04T12:00:00Z"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -303,7 +303,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==2018-12-04"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -326,7 +326,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==2018-12-07"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -349,7 +349,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==12:00:00"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -373,7 +373,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==12:00:00;name==\"beekeeper\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -398,7 +398,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { var entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==13:00:00;name==\"beekeeper\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -408,7 +408,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==12:00:00;name==\"beekeeperx\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -433,7 +433,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { var entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", q = "testedAt==12:00:00;observedBy==\"urn:ngsi-ld:Entity:4567\""), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -445,7 +445,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { q = "(testedAt==12:00:00;observedBy==\"urn:ngsi-ld:Entity:4567\");name==\"beekeeper\"" ), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -457,7 +457,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { q = "(testedAt==12:00:00;observedBy==\"urn:ngsi-ld:Entity:4567\")|name==\"beekeeper\"" ), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -469,7 +469,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { q = "(testedAt==13:00:00;observedBy==\"urn:ngsi-ld:Entity:4567\")|name==\"beekeeper\"" ), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -492,7 +492,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(id = listOf("urn:ngsi-ld:Beekeeper:1231"), expandedType = "Beekeeper"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -514,7 +514,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(id = listOf("urn:ngsi-ld:Beekeeper:1231")), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -539,7 +539,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entitiesCount = searchRepository.getEntities( QueryParams(id = listOf("urn:ngsi-ld:Beekeeper:1231"), q = "createdAt>2021-07-10T00:00:00Z"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).first @@ -564,7 +564,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entitiesCount = searchRepository.getEntities( QueryParams(q = "createdAt>$now"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).first @@ -598,7 +598,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { expandedType = "Beekeeper" ), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -628,7 +628,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:Beekeeper:0.*2$"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -658,7 +658,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:BeeHive:*"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -681,7 +681,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedAttrs = setOf(expandedNameProperty)), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -701,7 +701,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedAttrs = setOf("observedBy")), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -721,7 +721,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(q = "name==\"Scalpa\"", expandedAttrs = setOf("observedBy")), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -741,7 +741,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", expandedAttrs = setOf("observedBy")), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second @@ -769,7 +769,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entitiesCount = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = "^urn:ngsi-ld:Beekeeper:0.*2$"), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).first @@ -781,7 +781,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { @MethodSource("com.egm.stellio.entity.util.QueryEntitiesParameterizedTests#rawResultsProvider") fun `it should only return matching entities requested by pagination`( idPattern: String?, - page: Int, + offset: Int, limit: Int, expectedEntitiesIds: List ) { @@ -804,7 +804,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { val entities = searchRepository.getEntities( QueryParams(expandedType = "Beekeeper", idPattern = idPattern), userId, - page, + offset, limit, DEFAULT_CONTEXTS ).second diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/util/QueryEntitiesParameterizedTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/util/QueryEntitiesParameterizedTests.kt index ce30586ea..ed362c7a7 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/util/QueryEntitiesParameterizedTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/util/QueryEntitiesParameterizedTests.kt @@ -14,7 +14,7 @@ class QueryEntitiesParameterizedTests private constructor() { return Stream.of( Arguments.arguments( null, - 1, + 0, 50, listOf( "urn:ngsi-ld:Beekeeper:01231".toUri(), @@ -24,7 +24,7 @@ class QueryEntitiesParameterizedTests private constructor() { ), Arguments.arguments( null, - 1, + 0, 2, listOf( "urn:ngsi-ld:Beekeeper:01231".toUri(), @@ -45,13 +45,13 @@ class QueryEntitiesParameterizedTests private constructor() { ), Arguments.arguments( "^urn:ngsi-ld:Beekeeper:0.*2$", - 1, + 0, 1, listOf("urn:ngsi-ld:Beekeeper:01232".toUri()) ), Arguments.arguments( "^urn:ngsi-ld:Beekeeper:0.*2$", - 2, + 1, 1, listOf("urn:ngsi-ld:Beekeeper:03432".toUri()) ), diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt index e2fddaf1b..13aae8ed1 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityHandlerTests.kt @@ -559,16 +559,16 @@ class EntityHandlerTests { webClient.get() .uri( "/ngsi-ld/v1/entities/?type=Beehive" + - "&id=urn:ngsi-ld:Beehive:TESTC,urn:ngsi-ld:Beehive:TESTB,urn:ngsi-ld:Beehive:TESTD&limit=1&page=2" + "&id=urn:ngsi-ld:Beehive:TESTC,urn:ngsi-ld:Beehive:TESTB,urn:ngsi-ld:Beehive:TESTD&limit=1&offset=1" ) .exchange() .expectStatus().isOk .expectHeader().valueEquals( "Link", ";rel=\"prev\";type=\"application/ld+json\"", + "urn:ngsi-ld:Beehive:TESTD&limit=1&offset=0>;rel=\"prev\";type=\"application/ld+json\"", ";rel=\"next\";type=\"application/ld+json\"" + "urn:ngsi-ld:Beehive:TESTD&limit=1&offset=2>;rel=\"next\";type=\"application/ld+json\"" ) .expectBody().json( """[ @@ -583,14 +583,14 @@ class EntityHandlerTests { } @Test - fun `get entities should return 200 and empty response if requested page does not exists`() { + fun `get entities should return 200 and empty response if requested offset does not exists`() { every { entityService.exists(any()) } returns true every { entityService.searchEntities(any(), any(), any(), any(), any(), any()) } returns Pair(0, emptyList()) webClient.get() - .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=1&page=9") + .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=1&offset=9") .exchange() .expectStatus().isOk .expectBody().json("[]") @@ -599,7 +599,7 @@ class EntityHandlerTests { @Test fun `get entities should return 400 if limit is equal or less than zero`() { webClient.get() - .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=-1&page=1") + .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=-1&offset=1") .exchange() .expectStatus().isBadRequest .expectBody().json( @@ -607,7 +607,7 @@ class EntityHandlerTests { { "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", "title":"The request includes input data which does not meet the requirements of the operation", - "detail":"Page number and Limit must be strictly greater than zero" + "detail":"Offset must be greater than zero and limit must be strictly greater than zero" } """.trimIndent() ) @@ -616,7 +616,7 @@ class EntityHandlerTests { @Test fun `get entities should return 400 if limit is greater than the maximum authorized limit`() { webClient.get() - .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=200&page=1") + .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=200&offset=1") .exchange() .expectStatus().isBadRequest .expectBody().json( @@ -639,7 +639,7 @@ class EntityHandlerTests { ) webClient.get() - .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=0&page=1&count=true") + .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=0&offset=1&count=true") .exchange() .expectStatus().isOk .expectHeader().valueEquals(RESULTS_COUNT_HEADER, "3") @@ -649,7 +649,7 @@ class EntityHandlerTests { @Test fun `get entities should return 400 if the number of results is requested with a limit less than zero`() { webClient.get() - .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=-1&page=1&count=true") + .uri("/ngsi-ld/v1/entities/?type=Beehive&limit=-1&offset=1&count=true") .exchange() .expectStatus().isBadRequest .expectBody().json( @@ -657,7 +657,7 @@ class EntityHandlerTests { { "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", "title":"The request includes input data which does not meet the requirements of the operation", - "detail":"Page number must be strictly greater than zero and Limit must be greater than zero" + "detail":"Offset and limit must be greater than zero" } """.trimIndent() ) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt index 4bd31deab..e542245e2 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt @@ -26,7 +26,7 @@ const val RESULTS_COUNT_HEADER = "NGSILD-Results-Count" const val JSON_LD_CONTENT_TYPE = "application/ld+json" const val JSON_MERGE_PATCH_CONTENT_TYPE = "application/merge-patch+json" const val QUERY_PARAM_COUNT: String = "count" -const val QUERY_PARAM_PAGE: String = "page" +const val QUERY_PARAM_OFFSET: String = "offset" const val QUERY_PARAM_LIMIT: String = "limit" const val QUERY_PARAM_ID: String = "id" const val QUERY_PARAM_TYPE: String = "type" diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/PagingUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/PagingUtils.kt index c2cbdc216..0d095baf1 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/PagingUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/PagingUtils.kt @@ -11,20 +11,22 @@ object PagingUtils { resourceUrl: String, requestParams: MultiValueMap, resourcesCount: Int, - pageNumber: Int, + offset: Int, limit: Int ): Pair { var prevLink: String? = null var nextLink: String? = null - if (pageNumber > 1 && (resourcesCount > (pageNumber - 1) * limit)) + if (offset > 0 && resourcesCount > offset - limit) { + val prevOffset = if (offset > limit) offset - limit else 0 prevLink = - "<$resourceUrl?${requestParams.toEncodedUrl(page = pageNumber - 1, limit = limit)}>;" + + "<$resourceUrl?${requestParams.toEncodedUrl(offset = prevOffset, limit = limit)}>;" + "rel=\"prev\";type=\"application/ld+json\"" + } - if (resourcesCount > pageNumber * limit) + if (resourcesCount > offset + limit) nextLink = - "<$resourceUrl?${requestParams.toEncodedUrl(page = pageNumber + 1, limit = limit)}>;" + + "<$resourceUrl?${requestParams.toEncodedUrl(offset = offset + limit, limit = limit)}>;" + "rel=\"next\";type=\"application/ld+json\"" return Pair(prevLink, nextLink) @@ -56,10 +58,10 @@ object PagingUtils { else responseHeaders.body(body) } - private fun MultiValueMap.toEncodedUrl(page: Int, limit: Int): String { - val requestParams = this.entries.filter { !listOf("page", "limit").contains(it.key) }.toMutableList() + private fun MultiValueMap.toEncodedUrl(offset: Int, limit: Int): String { + val requestParams = this.entries.filter { !listOf("offset", "limit").contains(it.key) }.toMutableList() requestParams.addAll(mutableMapOf("limit" to listOf(limit.toString())).entries) - requestParams.addAll(mutableMapOf("page" to listOf(page.toString())).entries) + requestParams.addAll(mutableMapOf("offset" to listOf(offset.toString())).entries) return requestParams.joinToString("&") { it.key + "=" + it.value.joinToString(",") } } } diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/util/PagingUtilsTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/util/PagingUtilsTests.kt index ef4ad600f..278575903 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/util/PagingUtilsTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/util/PagingUtilsTests.kt @@ -14,7 +14,7 @@ class PagingUtilsTests { subscriptionResourceUrl, LinkedMultiValueMap(), 8, - 1, + 0, 10 ) @@ -27,13 +27,13 @@ class PagingUtilsTests { subscriptionResourceUrl, LinkedMultiValueMap(), 8, - 2, + 5, 5 ) assertEquals( links, - Pair(";rel=\"prev\";type=\"application/ld+json\"", null) + Pair(";rel=\"prev\";type=\"application/ld+json\"", null) ) } @@ -43,13 +43,13 @@ class PagingUtilsTests { subscriptionResourceUrl, LinkedMultiValueMap(), 8, - 1, + 0, 5 ) assertEquals( links, - Pair(null, ";rel=\"next\";type=\"application/ld+json\"") + Pair(null, ";rel=\"next\";type=\"application/ld+json\"") ) } @@ -59,15 +59,15 @@ class PagingUtilsTests { subscriptionResourceUrl, LinkedMultiValueMap(), 8, - 3, - 1 + 2, + 3 ) assertEquals( links, Pair( - ";rel=\"prev\";type=\"application/ld+json\"", - ";rel=\"next\";type=\"application/ld+json\"" + ";rel=\"prev\";type=\"application/ld+json\"", + ";rel=\"next\";type=\"application/ld+json\"" ) ) } diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt index bcebdfe81..ee1c3c17d 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt @@ -77,14 +77,18 @@ class SubscriptionHandler( @RequestParam params: MultiValueMap, @RequestParam options: Optional ): ResponseEntity<*> { - val page = params.getFirst(QUERY_PARAM_PAGE)?.toIntOrNull() ?: 1 + val offset = params.getFirst(QUERY_PARAM_OFFSET)?.toIntOrNull() ?: 0 val limit = params.getFirst(QUERY_PARAM_LIMIT)?.toIntOrNull() ?: applicationProperties.pagination.limitDefault val includeSysAttrs = options.filter { it.contains("sysAttrs") }.isPresent val contextLink = getContextFromLinkHeaderOrDefault(httpHeaders) val mediaType = getApplicableMediaType(httpHeaders) - if (limit <= 0 || page <= 0) + if (limit <= 0 || offset < 0) return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body(BadRequestDataResponse("Page number and Limit must be greater than zero")) + .body( + BadRequestDataResponse( + "Offset must be greater than zero and limit must be strictly greater than zero" + ) + ) if (limit > applicationProperties.pagination.limitMax) return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) @@ -96,14 +100,14 @@ class SubscriptionHandler( ) val userId = extractSubjectOrEmpty().awaitFirst() - val subscriptions = subscriptionService.getSubscriptions(limit, (page - 1) * limit, userId) + val subscriptions = subscriptionService.getSubscriptions(limit, offset, userId) .collectList().awaitFirst().toJson(contextLink, mediaType, includeSysAttrs) val subscriptionsCount = subscriptionService.getSubscriptionsCount(userId).awaitFirst() val prevAndNextLinks = getPagingLinks( "/ngsi-ld/v1/subscriptions", params, subscriptionsCount, - page, + offset, limit ) diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt index e31195cd8..5fa466639 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/web/SubscriptionHandlerTests.kt @@ -291,11 +291,14 @@ class SubscriptionHandlerTests { every { subscriptionService.getSubscriptions(any(), any(), any()) } returns Flux.just(subscription) webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=1&page=2") + .uri("/ngsi-ld/v1/subscriptions/?limit=1&offset=2") .exchange() .expectStatus().isOk .expectHeader() - .valueEquals("Link", ";rel=\"prev\";type=\"application/ld+json\"") + .valueEquals( + "Link", + ";rel=\"prev\";type=\"application/ld+json\"" + ) } @Test @@ -306,11 +309,14 @@ class SubscriptionHandlerTests { every { subscriptionService.getSubscriptions(any(), any(), any()) } returns Flux.just(subscription) webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=1&page=1") + .uri("/ngsi-ld/v1/subscriptions/?limit=1&offset=0") .exchange() .expectStatus().isOk .expectHeader() - .valueEquals("Link", ";rel=\"next\";type=\"application/ld+json\"") + .valueEquals( + "Link", + ";rel=\"next\";type=\"application/ld+json\"" + ) } @Test @@ -321,37 +327,37 @@ class SubscriptionHandlerTests { every { subscriptionService.getSubscriptions(any(), any(), any()) } returns Flux.just(subscription) webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=1&page=2") + .uri("/ngsi-ld/v1/subscriptions/?limit=1&offset=1") .exchange() .expectStatus().isOk .expectHeader().valueEquals( "Link", - ";rel=\"prev\";type=\"application/ld+json\"", - ";rel=\"next\";type=\"application/ld+json\"" + ";rel=\"prev\";type=\"application/ld+json\"", + ";rel=\"next\";type=\"application/ld+json\"" ) } @Test - fun `query subscriptions should return 200 and empty response if requested page does not exists`() { + fun `query subscriptions should return 200 and empty response if requested offset does not exists`() { every { subscriptionService.getSubscriptionsCount(any()) } returns Mono.just(2) every { subscriptionService.getSubscriptions(any(), any(), any()) } returns Flux.empty() webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=1&page=9") + .uri("/ngsi-ld/v1/subscriptions/?limit=1&offset=9") .exchange() .expectStatus().isOk .expectBody().json("[]") } @Test - fun `query subscriptions should return 400 if requested page is equal or less than zero`() { + fun `query subscriptions should return 400 if requested offset is less than zero`() { val subscription = gimmeRawSubscription() every { subscriptionService.getSubscriptionsCount(any()) } returns Mono.just(2) every { subscriptionService.getSubscriptions(any(), any(), any()) } returns Flux.just(subscription) webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=1&page=0") + .uri("/ngsi-ld/v1/subscriptions/?limit=1&offset=-1") .exchange() .expectStatus().isBadRequest .expectBody().json( @@ -359,7 +365,7 @@ class SubscriptionHandlerTests { { "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", "title":"The request includes input data which does not meet the requirements of the operation", - "detail":"Page number and Limit must be greater than zero" + "detail":"Offset must be greater than zero and limit must be strictly greater than zero" } """.trimIndent() ) @@ -373,7 +379,7 @@ class SubscriptionHandlerTests { every { subscriptionService.getSubscriptions(any(), any(), any()) } returns Flux.just(subscription) webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=-1&page=1") + .uri("/ngsi-ld/v1/subscriptions/?limit=-1&offset=1") .exchange() .expectStatus().isBadRequest .expectBody().json( @@ -381,7 +387,7 @@ class SubscriptionHandlerTests { { "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", "title":"The request includes input data which does not meet the requirements of the operation", - "detail":"Page number and Limit must be greater than zero" + "detail":"Offset must be greater than zero and limit must be strictly greater than zero" } """.trimIndent() ) @@ -390,7 +396,7 @@ class SubscriptionHandlerTests { @Test fun `query subscriptions should return 400 if limit is greater than the maximum authorized limit`() { webClient.get() - .uri("/ngsi-ld/v1/subscriptions/?limit=200&page=1") + .uri("/ngsi-ld/v1/subscriptions/?limit=200&offset=1") .exchange() .expectStatus().isBadRequest .expectBody().json( From 97b86ab9e73410a4c8ae948a4ac980212419aa8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 08:20:01 +0200 Subject: [PATCH 37/56] chore(deps): bump springmockk from 2.0.0 to 3.0.1 (#468) * chore(deps): bump springmockk from 2.0.0 to 3.0.1 Bumps [springmockk](https://github.com/Ninja-Squad/springmockk) from 2.0.0 to 3.0.1. - [Release notes](https://github.com/Ninja-Squad/springmockk/releases) - [Commits](https://github.com/Ninja-Squad/springmockk/compare/2.0.0...3.0.1) --- updated-dependencies: - dependency-name: com.ninja-squad:springmockk dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * chore: upgrade mockk version for compatibility with SB 2.5 / Kotlin 1.5 Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Benoit Orihuela --- build.gradle.kts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7194bf9eb..4ffded928 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -83,7 +83,8 @@ subprojects { // to ensure we are using mocks and spies from springmockk lib instead exclude(module = "mockito-core") } - testImplementation("com.ninja-squad:springmockk:2.0.0") + testImplementation("com.ninja-squad:springmockk:3.0.1") + testImplementation("io.mockk:mockk:1.12.0") testImplementation("io.projectreactor:reactor-test") testImplementation("org.springframework.security:spring-security-test") testImplementation("org.testcontainers:testcontainers") From b6242975485d065645605e683305766376e27edc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 17:53:46 +0200 Subject: [PATCH 38/56] chore(deps): Bump neo4j-migrations-spring-boot-starter (#482) Bumps neo4j-migrations-spring-boot-starter from 0.1.4 to 0.3.1. --- updated-dependencies: - dependency-name: eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- entity-service/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entity-service/build.gradle.kts b/entity-service/build.gradle.kts index a09a3ffa7..28dd222a8 100644 --- a/entity-service/build.gradle.kts +++ b/entity-service/build.gradle.kts @@ -13,7 +13,7 @@ plugins { dependencies { implementation("org.springframework.boot:spring-boot-starter-data-neo4j") - implementation("eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter:0.1.4") + implementation("eu.michael-simons.neo4j:neo4j-migrations-spring-boot-starter:0.3.1") implementation(project(":shared")) developmentOnly("org.springframework.boot:spring-boot-devtools") From 7cd7beef92f4a143a2603590650ea3df86d38304 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Tue, 10 Aug 2021 07:19:09 +0200 Subject: [PATCH 39/56] feat: add Docker images for linux/arm64 architecture #473 (#474) --- api-gateway/build.gradle.kts | 3 + build.gradle.kts | 10 +- entity-service/build.gradle.kts | 10 +- .../src/main/jib/database/wait-for-it.sh | 182 ++++++++++++++++++ .../src/main/jib/database/wait-for-neo4j.sh | 79 -------- search-service/build.gradle.kts | 3 + subscription-service/build.gradle.kts | 3 + 7 files changed, 207 insertions(+), 83 deletions(-) create mode 100644 entity-service/src/main/jib/database/wait-for-it.sh delete mode 100755 entity-service/src/main/jib/database/wait-for-neo4j.sh diff --git a/api-gateway/build.gradle.kts b/api-gateway/build.gradle.kts index bd91dc13e..95e05c3a6 100644 --- a/api-gateway/build.gradle.kts +++ b/api-gateway/build.gradle.kts @@ -1,3 +1,5 @@ +import com.google.cloud.tools.jib.gradle.PlatformParameters + plugins { id("com.google.cloud.tools.jib") id("org.springframework.boot") @@ -9,6 +11,7 @@ dependencies { } jib.from.image = project.ext["jibFromImage"].toString() +jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-api-gateway" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8080") diff --git a/build.gradle.kts b/build.gradle.kts index 4ffded928..04bf42ae9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,4 @@ +import com.google.cloud.tools.jib.gradle.PlatformParameters import io.gitlab.arturbosch.detekt.detekt import io.spring.gradle.dependencymanagement.dsl.DependencyManagementExtension import org.jetbrains.kotlin.gradle.tasks.KotlinCompile @@ -143,7 +144,14 @@ subprojects { } } - project.ext.set("jibFromImage", "adoptopenjdk:11-jre") + project.ext.set("jibFromImage", "openjdk:11-jre-slim-buster") + project.ext.set( + "jibFromPlatforms", + listOf( + PlatformParameters().apply { os = "linux"; architecture = "arm64" }, + PlatformParameters().apply { os = "linux"; architecture = "amd64" } + ) + ) project.ext.set("jibContainerJvmFlags", listOf("-Xms256m", "-Xmx768m")) project.ext.set("jibContainerCreationTime", "USE_CURRENT_TIMESTAMP") project.ext.set( diff --git a/entity-service/build.gradle.kts b/entity-service/build.gradle.kts index 28dd222a8..21b0d03da 100644 --- a/entity-service/build.gradle.kts +++ b/entity-service/build.gradle.kts @@ -1,3 +1,5 @@ +import com.google.cloud.tools.jib.gradle.PlatformParameters + val mainClass = "com.egm.stellio.entity.EntityServiceApplicationKt" configurations { @@ -29,12 +31,14 @@ tasks.bootRun { environment("SPRING_PROFILES_ACTIVE", "dev") } -jib.from.image = "adoptopenjdk/openjdk11:alpine-jre" +// use a non-slim image to have wget (used by the wait-for script below) +jib.from.image = "openjdk:11-jre-buster" +jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-entity-service" jib.container.entrypoint = listOf( "/bin/sh", "-c", - "/database/wait-for-neo4j.sh neo4j:7687 -t \$NEO4J_WAIT_TIMEOUT -- " + + "/database/wait-for-it.sh neo4j:7687 -t \$NEO4J_WAIT_TIMEOUT -- " + "java " + (project.ext["jibContainerJvmFlags"] as List).joinToString(" ") + " -cp /app/resources:/app/classes:/app/libs/* " + mainClass @@ -43,4 +47,4 @@ jib.container.environment = mapOf("NEO4J_WAIT_TIMEOUT" to "100") jib.container.ports = listOf("8082") jib.container.creationTime = project.ext["jibContainerCreationTime"].toString() jib.container.labels.putAll(project.ext["jibContainerLabels"] as Map) -jib.extraDirectories.permissions = mapOf("/database/wait-for-neo4j.sh" to "775") +jib.extraDirectories.permissions = mapOf("/database/wait-for-it.sh" to "775") diff --git a/entity-service/src/main/jib/database/wait-for-it.sh b/entity-service/src/main/jib/database/wait-for-it.sh new file mode 100644 index 000000000..d990e0d36 --- /dev/null +++ b/entity-service/src/main/jib/database/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/entity-service/src/main/jib/database/wait-for-neo4j.sh b/entity-service/src/main/jib/database/wait-for-neo4j.sh deleted file mode 100755 index bf6a0719a..000000000 --- a/entity-service/src/main/jib/database/wait-for-neo4j.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh - -TIMEOUT=15 -QUIET=0 - -echoerr() { - if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi -} - -usage() { - exitcode="$1" - cat << USAGE >&2 -Usage: - $cmdname host:port [-t timeout] [-- command args] - -q | --quiet Do not output any status messages - -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit "$exitcode" -} - -wait_for() { - for i in `seq $TIMEOUT` ; do - nc -z "$HOST" "$PORT" > /dev/null 2>&1 - - result=$? - if [ $result -eq 0 ] ; then - if [ $# -gt 0 ] ; then - exec "$@" - fi - exit 0 - fi - sleep 1 - done - echo "Operation timed out" >&2 - exit 1 -} - -while [ $# -gt 0 ] -do - case "$1" in - *:* ) - HOST=$(printf "%s\n" "$1"| cut -d : -f 1) - PORT=$(printf "%s\n" "$1"| cut -d : -f 2) - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -t) - TIMEOUT="$2" - if [ "$TIMEOUT" = "" ]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - break - ;; - --help) - usage 0 - ;; - *) - echoerr "Unknown argument: $1" - usage 1 - ;; - esac -done - -if [ "$HOST" = "" -o "$PORT" = "" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 -fi - -wait_for "$@" \ No newline at end of file diff --git a/search-service/build.gradle.kts b/search-service/build.gradle.kts index b0ccfd182..78a8b98d2 100644 --- a/search-service/build.gradle.kts +++ b/search-service/build.gradle.kts @@ -1,3 +1,5 @@ +import com.google.cloud.tools.jib.gradle.PlatformParameters + configurations { compileOnly { extendsFrom(configurations.annotationProcessor.get()) @@ -36,6 +38,7 @@ tasks.bootRun { } jib.from.image = project.ext["jibFromImage"].toString() +jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-search-service" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8083") diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index 610b21971..9c5e96497 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -1,3 +1,5 @@ +import com.google.cloud.tools.jib.gradle.PlatformParameters + configurations { compileOnly { extendsFrom(configurations.annotationProcessor.get()) @@ -37,6 +39,7 @@ tasks.bootRun { } jib.from.image = project.ext["jibFromImage"].toString() +jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-subscription-service" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8084") From 16af0853f8e7bb349970c0c45ab7584110a2e0c6 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Wed, 11 Aug 2021 08:28:56 +0200 Subject: [PATCH 40/56] Revert "feat: add Docker images for linux/arm64 architecture #473 (#474)" This reverts commit 7cd7beef92f4a143a2603590650ea3df86d38304. --- api-gateway/build.gradle.kts | 3 - build.gradle.kts | 10 +- entity-service/build.gradle.kts | 10 +- .../src/main/jib/database/wait-for-it.sh | 182 ------------------ .../src/main/jib/database/wait-for-neo4j.sh | 79 ++++++++ search-service/build.gradle.kts | 3 - subscription-service/build.gradle.kts | 3 - 7 files changed, 83 insertions(+), 207 deletions(-) delete mode 100644 entity-service/src/main/jib/database/wait-for-it.sh create mode 100755 entity-service/src/main/jib/database/wait-for-neo4j.sh diff --git a/api-gateway/build.gradle.kts b/api-gateway/build.gradle.kts index 95e05c3a6..bd91dc13e 100644 --- a/api-gateway/build.gradle.kts +++ b/api-gateway/build.gradle.kts @@ -1,5 +1,3 @@ -import com.google.cloud.tools.jib.gradle.PlatformParameters - plugins { id("com.google.cloud.tools.jib") id("org.springframework.boot") @@ -11,7 +9,6 @@ dependencies { } jib.from.image = project.ext["jibFromImage"].toString() -jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-api-gateway" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8080") diff --git a/build.gradle.kts b/build.gradle.kts index 04bf42ae9..4ffded928 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,3 @@ -import com.google.cloud.tools.jib.gradle.PlatformParameters import io.gitlab.arturbosch.detekt.detekt import io.spring.gradle.dependencymanagement.dsl.DependencyManagementExtension import org.jetbrains.kotlin.gradle.tasks.KotlinCompile @@ -144,14 +143,7 @@ subprojects { } } - project.ext.set("jibFromImage", "openjdk:11-jre-slim-buster") - project.ext.set( - "jibFromPlatforms", - listOf( - PlatformParameters().apply { os = "linux"; architecture = "arm64" }, - PlatformParameters().apply { os = "linux"; architecture = "amd64" } - ) - ) + project.ext.set("jibFromImage", "adoptopenjdk:11-jre") project.ext.set("jibContainerJvmFlags", listOf("-Xms256m", "-Xmx768m")) project.ext.set("jibContainerCreationTime", "USE_CURRENT_TIMESTAMP") project.ext.set( diff --git a/entity-service/build.gradle.kts b/entity-service/build.gradle.kts index 21b0d03da..28dd222a8 100644 --- a/entity-service/build.gradle.kts +++ b/entity-service/build.gradle.kts @@ -1,5 +1,3 @@ -import com.google.cloud.tools.jib.gradle.PlatformParameters - val mainClass = "com.egm.stellio.entity.EntityServiceApplicationKt" configurations { @@ -31,14 +29,12 @@ tasks.bootRun { environment("SPRING_PROFILES_ACTIVE", "dev") } -// use a non-slim image to have wget (used by the wait-for script below) -jib.from.image = "openjdk:11-jre-buster" -jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) +jib.from.image = "adoptopenjdk/openjdk11:alpine-jre" jib.to.image = "stellio/stellio-entity-service" jib.container.entrypoint = listOf( "/bin/sh", "-c", - "/database/wait-for-it.sh neo4j:7687 -t \$NEO4J_WAIT_TIMEOUT -- " + + "/database/wait-for-neo4j.sh neo4j:7687 -t \$NEO4J_WAIT_TIMEOUT -- " + "java " + (project.ext["jibContainerJvmFlags"] as List).joinToString(" ") + " -cp /app/resources:/app/classes:/app/libs/* " + mainClass @@ -47,4 +43,4 @@ jib.container.environment = mapOf("NEO4J_WAIT_TIMEOUT" to "100") jib.container.ports = listOf("8082") jib.container.creationTime = project.ext["jibContainerCreationTime"].toString() jib.container.labels.putAll(project.ext["jibContainerLabels"] as Map) -jib.extraDirectories.permissions = mapOf("/database/wait-for-it.sh" to "775") +jib.extraDirectories.permissions = mapOf("/database/wait-for-neo4j.sh" to "775") diff --git a/entity-service/src/main/jib/database/wait-for-it.sh b/entity-service/src/main/jib/database/wait-for-it.sh deleted file mode 100644 index d990e0d36..000000000 --- a/entity-service/src/main/jib/database/wait-for-it.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# Check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) - -WAITFORIT_BUSYTIMEFLAG="" -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - # Check if busybox timeout uses -t flag - # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then - WAITFORIT_BUSYTIMEFLAG="-t" - fi -else - WAITFORIT_ISBUSY=0 -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi diff --git a/entity-service/src/main/jib/database/wait-for-neo4j.sh b/entity-service/src/main/jib/database/wait-for-neo4j.sh new file mode 100755 index 000000000..bf6a0719a --- /dev/null +++ b/entity-service/src/main/jib/database/wait-for-neo4j.sh @@ -0,0 +1,79 @@ +#!/bin/sh + +TIMEOUT=15 +QUIET=0 + +echoerr() { + if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi +} + +usage() { + exitcode="$1" + cat << USAGE >&2 +Usage: + $cmdname host:port [-t timeout] [-- command args] + -q | --quiet Do not output any status messages + -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit "$exitcode" +} + +wait_for() { + for i in `seq $TIMEOUT` ; do + nc -z "$HOST" "$PORT" > /dev/null 2>&1 + + result=$? + if [ $result -eq 0 ] ; then + if [ $# -gt 0 ] ; then + exec "$@" + fi + exit 0 + fi + sleep 1 + done + echo "Operation timed out" >&2 + exit 1 +} + +while [ $# -gt 0 ] +do + case "$1" in + *:* ) + HOST=$(printf "%s\n" "$1"| cut -d : -f 1) + PORT=$(printf "%s\n" "$1"| cut -d : -f 2) + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -t) + TIMEOUT="$2" + if [ "$TIMEOUT" = "" ]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + break + ;; + --help) + usage 0 + ;; + *) + echoerr "Unknown argument: $1" + usage 1 + ;; + esac +done + +if [ "$HOST" = "" -o "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 +fi + +wait_for "$@" \ No newline at end of file diff --git a/search-service/build.gradle.kts b/search-service/build.gradle.kts index 78a8b98d2..b0ccfd182 100644 --- a/search-service/build.gradle.kts +++ b/search-service/build.gradle.kts @@ -1,5 +1,3 @@ -import com.google.cloud.tools.jib.gradle.PlatformParameters - configurations { compileOnly { extendsFrom(configurations.annotationProcessor.get()) @@ -38,7 +36,6 @@ tasks.bootRun { } jib.from.image = project.ext["jibFromImage"].toString() -jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-search-service" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8083") diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index 9c5e96497..610b21971 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -1,5 +1,3 @@ -import com.google.cloud.tools.jib.gradle.PlatformParameters - configurations { compileOnly { extendsFrom(configurations.annotationProcessor.get()) @@ -39,7 +37,6 @@ tasks.bootRun { } jib.from.image = project.ext["jibFromImage"].toString() -jib.from.platforms.addAll(project.ext["jibFromPlatforms"] as List) jib.to.image = "stellio/stellio-subscription-service" jib.container.jvmFlags = project.ext["jibContainerJvmFlags"] as List jib.container.ports = listOf("8084") From f61ea4ab34daad8f1434bfe99ca0fb1139cb5baf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Aug 2021 06:40:36 +0200 Subject: [PATCH 41/56] chore(deps): Bump com.google.cloud.tools.jib from 3.1.2 to 3.1.3 (#484) Bumps com.google.cloud.tools.jib from 3.1.2 to 3.1.3. --- updated-dependencies: - dependency-name: com.google.cloud.tools.jib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4ffded928..cb3a420ed 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,7 +17,7 @@ plugins { kotlin("jvm") version "1.5.21" apply false kotlin("plugin.spring") version "1.5.21" apply false id("org.jlleitschuh.gradle.ktlint") version "10.1.0" - id("com.google.cloud.tools.jib") version "3.1.2" apply false + id("com.google.cloud.tools.jib") version "3.1.3" apply false kotlin("kapt") version "1.5.21" apply false id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false id("org.sonarqube") version "3.3" From d7cf66a99dffb5f35974445e0ef597b4bbae9f2f Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 14 Aug 2021 09:14:01 +0200 Subject: [PATCH 42/56] fix(common): remove custom trailing of zeroes for milliseconds in dates formatting (#486) --- .../service/SubscriptionEventListenerServiceTest.kt | 2 +- .../main/kotlin/com/egm/stellio/shared/util/DateUtils.kt | 9 ++------- .../kotlin/com/egm/stellio/shared/util/DateUtilsTests.kt | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/SubscriptionEventListenerServiceTest.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/SubscriptionEventListenerServiceTest.kt index 031ff6ef7..7a6c306e9 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/SubscriptionEventListenerServiceTest.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/SubscriptionEventListenerServiceTest.kt @@ -70,7 +70,7 @@ class SubscriptionEventListenerServiceTest { { "type": "Notification", "value": "urn:ngsi-ld:BeeHive:TESTC,urn:ngsi-ld:BeeHive:TESTD", - "observedAt": "2020-03-10T00:00:00.000Z", + "observedAt": "2020-03-10T00:00:00Z", "instanceId": "urn:ngsi-ld:Notification:1234" } """.trimIndent() diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt index e78d23a95..e651b31b4 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt @@ -5,10 +5,5 @@ import java.time.format.DateTimeFormatter val formatter: DateTimeFormatter = DateTimeFormatter.ISO_INSTANT -fun ZonedDateTime.toNgsiLdFormat(): String { - val formatted = formatter.format(this) - return if (this.nano == 0) - formatted.removeSuffix("Z").plus(".000Z") - else - formatted -} +fun ZonedDateTime.toNgsiLdFormat(): String = + formatter.format(this) diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/util/DateUtilsTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/util/DateUtilsTests.kt index 6518dcfe6..f185cf31d 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/util/DateUtilsTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/util/DateUtilsTests.kt @@ -9,7 +9,7 @@ class DateUtilsTests { @Test fun `it should correctly render a datetime with milliseconds at zero`() { val datetime = ZonedDateTime.parse("2020-12-26T10:54:00.000Z") - assertEquals("2020-12-26T10:54:00.000Z", datetime.toNgsiLdFormat()) + assertEquals("2020-12-26T10:54:00Z", datetime.toNgsiLdFormat()) } @Test From dced6a3cb95961ae7d4e00b61c0f9000ebe74c84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Aug 2021 10:14:23 +0200 Subject: [PATCH 43/56] chore(deps): Bump detekt from 1.17.1 to 1.18.0 (#485) * chore(deps): Bump detekt-formatting from 1.17.1 to 1.18.0 * chore(deps): Bump detekt from 1.17.1 to 1.18.0 Bumps [detekt-formatting](https://github.com/detekt/detekt) from 1.17.1 to 1.18.0. - [Release notes](https://github.com/detekt/detekt/releases) - [Commits](https://github.com/detekt/detekt/compare/v1.17.1...v1.18.0) --- updated-dependencies: - dependency-name: io.gitlab.arturbosch.detekt:detekt-formatting dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * feat: also upgrade detekt base lib Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Benoit Orihuela --- build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index cb3a420ed..efc1e92da 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,7 +19,7 @@ plugins { id("org.jlleitschuh.gradle.ktlint") version "10.1.0" id("com.google.cloud.tools.jib") version "3.1.3" apply false kotlin("kapt") version "1.5.21" apply false - id("io.gitlab.arturbosch.detekt") version "1.17.1" apply false + id("io.gitlab.arturbosch.detekt") version "1.18.0" apply false id("org.sonarqube") version "3.3" jacoco } @@ -72,7 +72,7 @@ subprojects { "kapt"("io.arrow-kt:arrow-meta:0.10.4") - "detektPlugins"("io.gitlab.arturbosch.detekt:detekt-formatting:1.17.1") + "detektPlugins"("io.gitlab.arturbosch.detekt:detekt-formatting:1.18.0") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") @@ -115,8 +115,8 @@ subprojects { } detekt { - toolVersion = "1.17.1" - input = files("src/main/kotlin", "src/test/kotlin") + toolVersion = "1.18.0" + source = files("src/main/kotlin", "src/test/kotlin") config = files(detektConfigFile) buildUponDefaultConfig = true baseline = file("$projectDir/config/detekt/baseline.xml") From d416576c9ac982f72d532932900a64c162cd280d Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 14 Aug 2021 10:23:17 +0200 Subject: [PATCH 44/56] fix(search): ensure deletion of instances is done before deleting temporal entity attributes (#487) --- .../search/service/TemporalEntityAttributeService.kt | 4 +--- .../search/service/TemporalEntityAttributeServiceTests.kt | 7 ++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt index f2fc64d64..fd244eaf1 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt @@ -159,9 +159,7 @@ class TemporalEntityAttributeService( fun deleteTemporalEntityReferences(entityId: URI): Mono = attributeInstanceService.deleteAttributeInstancesOfEntity(entityId) .zipWith(deleteEntityPayload(entityId)) - .map { it.t1 + it.t2 } - .zipWith(deleteTemporalAttributesOfEntity(entityId)) - .map { it.t1 + it.t2 } + .then(deleteTemporalAttributesOfEntity(entityId)) fun deleteTemporalAttributesOfEntity(entityId: URI): Mono = r2dbcEntityTemplate.delete(TemporalEntityAttribute::class.java) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt index b36d7bcb2..e7348f68e 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt @@ -394,7 +394,12 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val deletedRecords = temporalEntityAttributeService.deleteTemporalEntityReferences(entityId).block() - assert(deletedRecords == 5) + assert(deletedRecords == 2) + + verify { + attributeInstanceService.deleteAttributeInstancesOfEntity(entityId) + } + confirmVerified(attributeInstanceService) } @Test From 9dae58f5405ea09ed6a899b37a1d462781c8ee9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Aug 2021 18:34:11 +0200 Subject: [PATCH 45/56] chore(deps): Bump com.google.cloud.tools.jib from 3.1.3 to 3.1.4 (#489) Bumps com.google.cloud.tools.jib from 3.1.3 to 3.1.4. --- updated-dependencies: - dependency-name: com.google.cloud.tools.jib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index efc1e92da..4d314baa2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -17,7 +17,7 @@ plugins { kotlin("jvm") version "1.5.21" apply false kotlin("plugin.spring") version "1.5.21" apply false id("org.jlleitschuh.gradle.ktlint") version "10.1.0" - id("com.google.cloud.tools.jib") version "3.1.3" apply false + id("com.google.cloud.tools.jib") version "3.1.4" apply false kotlin("kapt") version "1.5.21" apply false id("io.gitlab.arturbosch.detekt") version "1.18.0" apply false id("org.sonarqube") version "3.3" From 99263c31d1dbf526c0345d11e1d9ca69efc397cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 21:18:41 +0200 Subject: [PATCH 46/56] chore(deps): Bump org.springframework.boot from 2.5.3 to 2.5.4 (#490) Bumps [org.springframework.boot](https://github.com/spring-projects/spring-boot) from 2.5.3 to 2.5.4. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v2.5.3...v2.5.4) --- updated-dependencies: - dependency-name: org.springframework.boot dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4d314baa2..a8e0bfdc9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,7 +12,7 @@ plugins { java // why did I have to add that ?! // only apply the plugin in the subprojects requiring it because it expects a Spring Boot app // and the shared lib is obviously not one - id("org.springframework.boot") version "2.5.3" apply false + id("org.springframework.boot") version "2.5.4" apply false id("io.spring.dependency-management") version "1.0.11.RELEASE" apply false kotlin("jvm") version "1.5.21" apply false kotlin("plugin.spring") version "1.5.21" apply false From 56efbc3298546f66e8e0c8c98dff3d660fb0e950 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 27 Aug 2021 16:58:32 +0200 Subject: [PATCH 47/56] refactor(entity): improve performance of search by loading all results at once (#492) - also mark associated tx as read only to avoid unnecessary commits --- .../com/egm/stellio/entity/model/Entity.kt | 2 +- .../stellio/entity/service/EntityService.kt | 62 +++++++++++++------ 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt index c7f12927f..5913dca0a 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Entity.kt @@ -58,7 +58,7 @@ data class Entity( ) { - fun serializeCoreProperties(includeSysAttrs: Boolean): MutableMap { + fun serializeCoreProperties(includeSysAttrs: Boolean): Map { val resultEntity = mutableMapOf() resultEntity[JSONLD_ID] = id.toString() resultEntity[JSONLD_TYPE] = type diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 5560f90d1..e2ead052e 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -161,17 +161,11 @@ class EntityService( fun existsAsPartial(entityId: URI): Boolean = partialEntityRepository.existsById(entityId) - /** - * @return a pair consisting of a map representing the entity keys and attributes and the list of contexts - * associated to the entity - * @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity - */ - fun getFullEntityById(entityId: URI, includeSysAttrs: Boolean = false): JsonLdEntity? { - val entity = entityRepository.findById(entityId) - .orElseThrow { ResourceNotFoundException(entityNotFoundMessage(entityId.toString())) } - val resultEntity = entity.serializeCoreProperties(includeSysAttrs) - - entity.properties + private fun serializeEntityProperties( + properties: List, + includeSysAttrs: Boolean = false + ): Map = + properties .map { property -> val serializedProperty = property.serializeCoreProperties(includeSysAttrs) @@ -188,11 +182,12 @@ class EntityService( Pair(property.name, serializedProperty) } .groupBy({ it.first }, { it.second }) - .forEach { (propertyName, propertyValues) -> - resultEntity[propertyName] = propertyValues - } - entity.relationships + private fun serializeEntityRelationships( + relationships: List, + includeSysAttrs: Boolean = false + ): Map = + relationships .map { relationship -> val serializedRelationship = serializeRelationship(relationship, includeSysAttrs) @@ -209,11 +204,36 @@ class EntityService( Pair(relationship.relationshipType(), serializedRelationship) } .groupBy({ it.first }, { it.second }) - .forEach { (relationshipName, relationshipValues) -> - resultEntity[relationshipName] = relationshipValues + + fun getFullEntitiesById(entitiesIds: List, includeSysAttrs: Boolean = false): List = + entityRepository.findAllById(entitiesIds) + .map { + JsonLdEntity( + it.serializeCoreProperties(includeSysAttrs) + .plus(serializeEntityProperties(it.properties)) + .plus(serializeEntityRelationships(it.relationships)), + it.contexts + ) + }.sortedBy { + // as findAllById does not preserve order of the results, sort them back by id (search order) + it.id } - return JsonLdEntity(resultEntity, entity.contexts) + /** + * @return a pair consisting of a map representing the entity keys and attributes and the list of contexts + * associated to the entity + * @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity + */ + fun getFullEntityById(entityId: URI, includeSysAttrs: Boolean = false): JsonLdEntity? { + val entity = entityRepository.findById(entityId) + .orElseThrow { ResourceNotFoundException(entityNotFoundMessage(entityId.toString())) } + + return JsonLdEntity( + entity.serializeCoreProperties(includeSysAttrs) + .plus(serializeEntityProperties(entity.properties)) + .plus(serializeEntityRelationships(entity.relationships)), + entity.contexts + ) } fun getEntityCoreProperties(entityId: URI) = entityRepository.getEntityCoreById(entityId.toString())!! @@ -232,6 +252,7 @@ class EntityService( /** @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity */ + @Transactional(readOnly = true) fun searchEntities( queryParams: QueryParams, userSub: String, @@ -251,7 +272,7 @@ class EntityService( * @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity * @return a list of entities represented as per #getFullEntityById result */ - @Transactional + @Transactional(readOnly = true) fun searchEntities( queryParams: QueryParams, userSub: String, @@ -267,7 +288,8 @@ class EntityService( limit, contexts ) - return Pair(result.first, result.second.mapNotNull { getFullEntityById(it, includeSysAttrs) }) + + return Pair(result.first, getFullEntitiesById(result.second, includeSysAttrs)) } @Transactional From 01036defdebe968d18cb044ae3d5586f98da6ef4 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 27 Aug 2021 18:23:24 +0200 Subject: [PATCH 48/56] fixup(entity): propagation of includeSysAttrs parameter to attributes serialization functions --- .../com/egm/stellio/entity/service/EntityService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index e2ead052e..51160bf6e 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -210,8 +210,8 @@ class EntityService( .map { JsonLdEntity( it.serializeCoreProperties(includeSysAttrs) - .plus(serializeEntityProperties(it.properties)) - .plus(serializeEntityRelationships(it.relationships)), + .plus(serializeEntityProperties(it.properties, includeSysAttrs)) + .plus(serializeEntityRelationships(it.relationships, includeSysAttrs)), it.contexts ) }.sortedBy { @@ -230,8 +230,8 @@ class EntityService( return JsonLdEntity( entity.serializeCoreProperties(includeSysAttrs) - .plus(serializeEntityProperties(entity.properties)) - .plus(serializeEntityRelationships(entity.relationships)), + .plus(serializeEntityProperties(entity.properties, includeSysAttrs)) + .plus(serializeEntityRelationships(entity.relationships, includeSysAttrs)), entity.contexts ) } From 52863e8603d6d74c47ecd3428faf60a3292d5e3d Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 28 Aug 2021 09:18:03 +0200 Subject: [PATCH 49/56] chore: add logging for info received from cim.observations topics --- .../com/egm/stellio/entity/service/EntityAttributeService.kt | 5 +++++ .../egm/stellio/entity/service/ObservationEventListener.kt | 3 ++- .../main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt index 9aca339fa..05a294151 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityAttributeService.kt @@ -9,6 +9,7 @@ import com.egm.stellio.entity.repository.RelationshipRepository import com.egm.stellio.shared.model.* import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdKey +import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.net.URI @@ -21,6 +22,8 @@ class EntityAttributeService( private val relationshipRepository: RelationshipRepository ) { + private val logger = LoggerFactory.getLogger(javaClass) + @Transactional fun partialUpdateEntityAttribute( entityId: URI, @@ -30,6 +33,8 @@ class EntityAttributeService( val expandedAttributeName = expandedPayload.keys.first() val attributeValues = expandedPayload.values.first() + logger.debug("Updating attribute $expandedAttributeName of entity $entityId with values: $attributeValues") + val updateResult = attributeValues.map { attributeInstanceValues -> val datasetId = attributeInstanceValues.getDatasetId() when { diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt index 46083c338..1df2c3525 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt @@ -20,6 +20,7 @@ class ObservationEventListener( @KafkaListener(topicPattern = "cim.observation.*", groupId = "observations") fun processMessage(content: String) { + logger.debug("Received event: $content") when (val observationEvent = deserializeAs(content)) { is EntityCreateEvent -> handleEntityCreate(observationEvent) is AttributeUpdateEvent -> handleAttributeUpdateEvent(observationEvent) @@ -107,7 +108,7 @@ class ObservationEventListener( observationEvent.attributeName, observationEvent.datasetId, observationEvent.operationPayload, - compactAndSerialize(updatedEntity!!, observationEvent.contexts, MediaType.APPLICATION_JSON), + compactAndSerialize(updatedEntity, observationEvent.contexts, MediaType.APPLICATION_JSON), observationEvent.contexts, observationEvent.overwrite ), diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index d6629ac31..72f3d3354 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -544,8 +544,8 @@ fun extractAttributeInstanceFromCompactedEntity( else { // Since some attributes cannot be well compacted, to be improved later logger.warn( - "Received expanded attribute $attributeName, " + - "extracting instance for ${attributeName.extractShortTypeFromExpanded()}" + "Could not find entry for attribute: $attributeName, " + + "trying on the 'guessed' short form instead: ${attributeName.extractShortTypeFromExpanded()}" ) compactedJsonLdEntity[attributeName.extractShortTypeFromExpanded()] as CompactedJsonLdAttribute } From 3d5f844c1baebb1292690e938b2b2772a03c8b1f Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 28 Aug 2021 18:05:51 +0200 Subject: [PATCH 50/56] fix(entity): do not propagate attribute update events if no update has been done - for instance, when the property does not exist in the target entity --- .../entity/service/ObservationEventListener.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt index 1df2c3525..cb84d22d4 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/ObservationEventListener.kt @@ -56,11 +56,20 @@ class ObservationEventListener( ) try { - entityAttributeService.partialUpdateEntityAttribute( + val updateResult = entityAttributeService.partialUpdateEntityAttribute( observationEvent.entityId, expandedPayload, observationEvent.contexts ) + // TODO things could be more fine-grained (e.g., one instance updated and not the other one) + // so we should also check the notUpdated data and remove them from the propagated payload + if (updateResult.updated.isEmpty()) { + logger.info( + "Nothing has been updated for attribute ${observationEvent.attributeName}" + + " in entity ${observationEvent.entityId}, returning" + ) + return + } } catch (e: ResourceNotFoundException) { logger.error("Entity or attribute not found in observation : ${e.message}") return From ff9b6b4f02c192a8a621961cb838ae88b0361870 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sat, 28 Aug 2021 18:06:28 +0200 Subject: [PATCH 51/56] chore(entity): remove last use of NOT EXISTS in Cypher query (deprecated) --- .../kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt index 4b158759c..feb2651b1 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt @@ -262,7 +262,7 @@ class Neo4jRepository( val query = if (datasetId == null) """ MATCH (a:${subjectNodeInfo.label} { id: ${'$'}attributeId })-[:HAS_OBJECT]->(rel:Relationship)-[:$relationshipType]->() - WHERE NOT EXISTS (rel.datasetId) + WHERE rel.datasetId IS NULL RETURN a.id """.trimIndent() else From 3df1a2b78217a1bd2baf912604df9fb7cfbb2251 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 29 Aug 2021 11:24:08 +0200 Subject: [PATCH 52/56] feat(entity): store object of a relationship along with the relationship node (#493) - thus, it avoids doing one more DB request per relationship when retrieving results of a search query --- .../Neo4jAuthorizationService.kt | 1 + .../egm/stellio/entity/model/Relationship.kt | 20 +++++++++- .../entity/repository/Neo4jRepository.kt | 18 +-------- .../stellio/entity/service/EntityService.kt | 40 +++++-------------- .../service/SubscriptionHandlerService.kt | 1 + ...V08__add_object_id_in_relationships.cypher | 2 + .../Neo4jAuthorizationRepositoryTest.kt | 4 +- .../entity/repository/Neo4jRepositoryTests.kt | 25 +++++++++++- .../repository/Neo4jSearchRepositoryTests.kt | 2 +- .../StandaloneNeo4jSearchRepositoryTests.kt | 2 +- .../service/EntityAttributeServiceTests.kt | 19 +++++---- 11 files changed, 73 insertions(+), 61 deletions(-) create mode 100644 entity-service/src/main/resources/neo4j/migrations/V08__add_object_id_in_relationships.cypher diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationService.kt index d4f228722..740f3bc4a 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationService.kt @@ -135,6 +135,7 @@ class Neo4jAuthorizationService( override fun createAdminLinks(entitiesId: List, userSub: String) { val relationships = entitiesId.map { Relationship( + objectId = it, type = listOf(R_CAN_ADMIN), datasetId = "urn:ngsi-ld:Dataset:rCanAdmin:$it".toUri() ) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt index 99c5bda77..a63dc963c 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/model/Relationship.kt @@ -3,7 +3,9 @@ package com.egm.stellio.entity.model import com.egm.stellio.entity.config.Neo4jUriPropertyConverter import com.egm.stellio.shared.model.NgsiLdRelationshipInstance import com.egm.stellio.shared.util.JsonLdUtils +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_HAS_OBJECT import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsDateTime +import com.egm.stellio.shared.util.JsonLdUtils.getPropertyValueFromMapAsString import com.egm.stellio.shared.util.extractShortTypeFromExpanded import com.egm.stellio.shared.util.toNgsiLdFormat import com.egm.stellio.shared.util.toUri @@ -27,6 +29,9 @@ data class Relationship( @ConvertWith(converter = Neo4jUriPropertyConverter::class) val id: URI = "urn:ngsi-ld:Relationship:${UUID.randomUUID()}".toUri(), + // keep a copy of the target object URI to avoid unnecessary DB calls to retrieve it + var objectId: URI, + var observedAt: ZonedDateTime? = null, @JsonIgnore @@ -53,6 +58,7 @@ data class Relationship( constructor(type: String, ngsiLdRelationshipInstance: NgsiLdRelationshipInstance) : this( + objectId = ngsiLdRelationshipInstance.objectId, type = listOf(type), observedAt = ngsiLdRelationshipInstance.observedAt, datasetId = ngsiLdRelationshipInstance.datasetId @@ -86,6 +92,9 @@ data class Relationship( ) } + resultEntity[JsonLdUtils.NGSILD_RELATIONSHIP_HAS_OBJECT] = mapOf( + JsonLdUtils.JSONLD_ID to objectId.toString() + ) resultEntity[JsonLdUtils.JSONLD_TYPE] = JsonLdUtils.NGSILD_RELATIONSHIP_TYPE.uri return resultEntity @@ -94,6 +103,7 @@ data class Relationship( override fun nodeProperties(): MutableMap { val nodeProperties = mutableMapOf( "id" to id.toString(), + "objectId" to objectId.toString(), "createdAt" to createdAt ) @@ -114,7 +124,10 @@ data class Relationship( override fun id(): URI = id - private fun updateValues(observedAt: ZonedDateTime?): Relationship { + private fun updateValues(objectId: URI?, observedAt: ZonedDateTime?): Relationship { + objectId?.let { + this.objectId = it + } observedAt?.let { this.observedAt = observedAt } @@ -122,7 +135,10 @@ data class Relationship( } fun updateValues(updateFragment: Map>): Relationship = - updateValues(getPropertyValueFromMapAsDateTime(updateFragment, JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY)) + updateValues( + getPropertyValueFromMapAsString(updateFragment, NGSILD_RELATIONSHIP_HAS_OBJECT)?.toUri(), + getPropertyValueFromMapAsDateTime(updateFragment, JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY) + ) // neo4j forces us to have a list but we know we have only one dynamic label fun relationshipType(): String = diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt index feb2651b1..279facebb 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt @@ -177,23 +177,6 @@ class Neo4jRepository( return neo4jClient.query(query).bind(entityId.toString()).to("entityId").run().counters().propertiesSet() } - fun getTargetObjectIdOfRelationship(relationshipId: URI, relationshipType: String): URI { - val query = - """ - MATCH (r:Relationship { id: ${'$'}relationshipId })-[:$relationshipType]->(e) - RETURN e.id as entityId - """.trimIndent() - - val result = neo4jClient.query(query) - .bind(relationshipId.toString()).to("relationshipId") - .fetch() - .one() - - return result.map { - (it["entityId"] as String).toUri() - }.orElse(null) - } - fun hasRelationshipOfType(subjectNodeInfo: SubjectNodeInfo, relationshipType: String): Boolean { val query = """ @@ -324,6 +307,7 @@ class Neo4jRepository( ON CREATE SET target:PartialEntity DETACH DELETE v MERGE (a)-[:$relationshipType]->(target) + SET a.objectId = ${'$'}newRelationshipObjectId """.trimIndent() val parameters = mapOf( diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 51160bf6e..6a7650100 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -8,8 +8,6 @@ import com.egm.stellio.entity.repository.EntitySubjectNode import com.egm.stellio.entity.repository.Neo4jRepository import com.egm.stellio.entity.repository.PartialEntityRepository import com.egm.stellio.shared.model.* -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID -import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_HAS_OBJECT import com.egm.stellio.shared.util.entityNotFoundMessage import com.egm.stellio.shared.util.extractShortTypeFromExpanded import org.slf4j.LoggerFactory @@ -42,8 +40,7 @@ class EntityService( createEntityRelationship( ngsiLdEntity.id, ngsiLdRelationship.name, - ngsiLdRelationshipInstance, - ngsiLdRelationshipInstance.objectId + ngsiLdRelationshipInstance ) } } @@ -95,13 +92,12 @@ class EntityService( private fun createEntityRelationship( entityId: URI, relationshipType: String, - ngsiLdRelationshipInstance: NgsiLdRelationshipInstance, - targetEntityId: URI + ngsiLdRelationshipInstance: NgsiLdRelationshipInstance ): URI { val rawRelationship = Relationship(relationshipType, ngsiLdRelationshipInstance) neo4jRepository.createRelationshipOfSubject( - EntitySubjectNode(entityId), rawRelationship, targetEntityId + EntitySubjectNode(entityId), rawRelationship, ngsiLdRelationshipInstance.objectId ) createAttributeProperties(rawRelationship.id, ngsiLdRelationshipInstance.properties) @@ -175,7 +171,7 @@ class EntityService( } property.relationships.forEach { innerRelationship -> - val serializedSubRelationship = serializeRelationship(innerRelationship, includeSysAttrs) + val serializedSubRelationship = innerRelationship.serializeCoreProperties(includeSysAttrs) serializedProperty[innerRelationship.relationshipType()] = serializedSubRelationship } @@ -189,7 +185,7 @@ class EntityService( ): Map = relationships .map { relationship -> - val serializedRelationship = serializeRelationship(relationship, includeSysAttrs) + val serializedRelationship = relationship.serializeCoreProperties(includeSysAttrs) relationship.properties.forEach { innerProperty -> val serializedSubProperty = innerProperty.serializeCoreProperties(includeSysAttrs) @@ -197,7 +193,7 @@ class EntityService( } relationship.relationships.forEach { innerRelationship -> - val serializedSubRelationship = serializeRelationship(innerRelationship, includeSysAttrs) + val serializedSubRelationship = innerRelationship.serializeCoreProperties(includeSysAttrs) serializedRelationship[innerRelationship.relationshipType()] = serializedSubRelationship } @@ -238,18 +234,6 @@ class EntityService( fun getEntityCoreProperties(entityId: URI) = entityRepository.getEntityCoreById(entityId.toString())!! - private fun serializeRelationship(relationship: Relationship, includeSysAttrs: Boolean): MutableMap { - val serializedRelationship = relationship.serializeCoreProperties(includeSysAttrs) - // TODO not perfect as it makes one more DB request per relationship to get the target entity id - val targetEntityId = - neo4jRepository.getTargetObjectIdOfRelationship( - relationship.id, - relationship.relationshipType().toRelationshipTypeName() - ) - serializedRelationship[NGSILD_RELATIONSHIP_HAS_OBJECT] = mapOf(JSONLD_ID to targetEntityId.toString()) - return serializedRelationship - } - /** @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity */ @Transactional(readOnly = true) @@ -266,8 +250,7 @@ class EntityService( /** * Search entities by type and query parameters * - * @param type the short-hand type (e.g "Measure") - * @param query the list of raw query parameters (e.g "name==test") + * @param queryParams the list of raw query parameters (e.g. type, idPattern,...) * @param contexts the list of contexts to consider * @param includeSysAttrs true if createdAt and modifiedAt have to be displayed in the entity * @return a list of entities represented as per #getFullEntityById result @@ -341,8 +324,7 @@ class EntityService( createEntityRelationship( entityId, ngsiLdRelationship.name, - ngsiLdRelationshipInstance, - ngsiLdRelationshipInstance.objectId + ngsiLdRelationshipInstance ) UpdateAttributeResult( ngsiLdRelationship.name, @@ -371,8 +353,7 @@ class EntityService( createEntityRelationship( entityId, ngsiLdRelationship.name, - ngsiLdRelationshipInstance, - ngsiLdRelationshipInstance.objectId + ngsiLdRelationshipInstance ) UpdateAttributeResult( ngsiLdRelationship.name, @@ -516,8 +497,7 @@ class EntityService( createEntityRelationship( entityId, ngsiLdRelationship.name, - ngsiLdRelationshipInstance, - ngsiLdRelationshipInstance.objectId + ngsiLdRelationshipInstance ) UpdateAttributeResult( ngsiLdRelationship.name, diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt index a3be5d179..efdbb54b0 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/SubscriptionHandlerService.kt @@ -108,6 +108,7 @@ class SubscriptionHandlerService( entityService.deleteEntity(lastNotification.id) } else { val rawRelationship = Relationship( + objectId = notification.id, type = listOf(JsonLdUtils.EGM_RAISED_NOTIFICATION) ) diff --git a/entity-service/src/main/resources/neo4j/migrations/V08__add_object_id_in_relationships.cypher b/entity-service/src/main/resources/neo4j/migrations/V08__add_object_id_in_relationships.cypher new file mode 100644 index 000000000..142486f66 --- /dev/null +++ b/entity-service/src/main/resources/neo4j/migrations/V08__add_object_id_in_relationships.cypher @@ -0,0 +1,2 @@ +MATCH (r:Attribute:Relationship)-[]->(e:Entity) +SET r.objectId = e.id diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt index e1c937901..652dfa3e2 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/authorization/Neo4jAuthorizationRepositoryTest.kt @@ -395,6 +395,7 @@ class Neo4jAuthorizationRepositoryTest : WithNeo4jContainer { userUri, targetIds.map { Relationship( + objectId = it, type = listOf(R_CAN_ADMIN), datasetId = "urn:ngsi-ld:Dataset:rCanAdmin:$it".toUri() ) @@ -426,6 +427,7 @@ class Neo4jAuthorizationRepositoryTest : WithNeo4jContainer { serviceAccountUri, targetIds.map { Relationship( + objectId = it, type = listOf(R_CAN_ADMIN), datasetId = "urn:ngsi-ld:Dataset:rCanAdmin:$it".toUri() ) @@ -442,7 +444,7 @@ class Neo4jAuthorizationRepositoryTest : WithNeo4jContainer { } fun createRelationship(subjectNodeInfo: SubjectNodeInfo, relationshipType: String, objectId: URI): Relationship { - val relationship = Relationship(type = listOf(relationshipType)) + val relationship = Relationship(objectId = objectId, type = listOf(relationshipType)) neo4jRepository.createRelationshipOfSubject(subjectNodeInfo, relationship, objectId) diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt index a43f3a9dd..10c1b4b6b 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt @@ -378,6 +378,27 @@ class Neo4jRepositoryTests : WithNeo4jContainer { ) } + @Test + fun `it should update the target relationship of a relationship`() { + val property = createProperty("https://uri.etsi.org/ngsi-ld/temperature", 36) + propertyRepository.save(property) + val relationship = createRelationship( + AttributeSubjectNode(property.id), + EGM_OBSERVED_BY, + "urn:ngsi-ld:Entity:123".toUri() + ) + + neo4jRepository.updateTargetOfRelationship( + relationship.id, + EGM_OBSERVED_BY.toRelationshipTypeName(), + "urn:ngsi-ld:Entity:123".toUri(), + "urn:ngsi-ld:Entity:456".toUri() + ) + + val updatedRelationship = relationshipRepository.findById(relationship.id) + assertEquals("urn:ngsi-ld:Entity:456".toUri(), updatedRelationship.get().objectId) + } + @Test fun `it should add modifiedAt value when creating a new property`() { val property = Property(name = "name", value = "Scalpa") @@ -387,7 +408,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { @Test fun `it should add modifiedAt value when saving a new relationship`() { - val relationship = Relationship(type = listOf("connectsTo")) + val relationship = Relationship(objectId = "urn:ngsi-ld:Entity:target".toUri(), type = listOf("connectsTo")) relationshipRepository.save(relationship) assertNotNull(relationshipRepository.findById(relationship.id).get().modifiedAt) } @@ -1157,7 +1178,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { objectId: URI, datasetId: URI? = null ): Relationship { - val relationship = Relationship(type = listOf(relationshipType), datasetId = datasetId) + val relationship = Relationship(objectId = objectId, type = listOf(relationshipType), datasetId = datasetId) neo4jRepository.createRelationshipOfSubject(subjectNodeInfo, relationship, objectId) return relationship diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt index 22d59ef9a..748555a99 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jSearchRepositoryTests.kt @@ -374,7 +374,7 @@ class Neo4jSearchRepositoryTests : WithNeo4jContainer { objectId: URI, datasetId: URI? = null ): Relationship { - val relationship = Relationship(type = listOf(relationshipType), datasetId = datasetId) + val relationship = Relationship(objectId = objectId, type = listOf(relationshipType), datasetId = datasetId) neo4jRepository.createRelationshipOfSubject(subjectNodeInfo, relationship, objectId) return relationship diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt index ad8581f9e..0b27cc302 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/StandaloneNeo4jSearchRepositoryTests.kt @@ -828,7 +828,7 @@ class StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer { objectId: URI, datasetId: URI? = null ): Relationship { - val relationship = Relationship(type = listOf(relationshipType), datasetId = datasetId) + val relationship = Relationship(objectId = objectId, type = listOf(relationshipType), datasetId = datasetId) neo4jRepository.createRelationshipOfSubject(subjectNodeInfo, relationship, objectId) return relationship diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt index 7c38f6046..c3314e284 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityAttributeServiceTests.kt @@ -22,6 +22,8 @@ import org.junit.jupiter.api.assertThrows import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles +import java.net.URI +import java.util.UUID @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [EntityAttributeService::class]) @ActiveProfiles("test") @@ -174,7 +176,7 @@ class EntityAttributeServiceTests { val expandedPayload = parseAndExpandAttributeFragment(propertyName, payload, listOf(AQUAC_COMPOUND_CONTEXT)) val property = Property(name = propertyName, unitCode = "years", value = 0) - val relationship = Relationship(type = listOf("measuredBy")) + val relationship = Relationship(objectId = generateRandomObjectId(), type = listOf("measuredBy")) every { neo4jRepository.hasRelationshipInstance(match { it.label == "Entity" }, any(), any()) } returns false every { neo4jRepository.hasPropertyInstance(any(), any(), any()) } returns true @@ -247,7 +249,7 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(type = listOf("isContainedIn")) + val relationship = Relationship(objectId = generateRandomObjectId(), type = listOf(relationshipType)) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true every { relationshipRepository.getRelationshipOfSubject(any(), any()) } returns relationship @@ -281,7 +283,7 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(type = listOf("isContainedIn")) + val relationship = Relationship(objectId = generateRandomObjectId(), type = listOf(relationshipType)) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true every { relationshipRepository.getRelationshipOfSubject(any(), any(), any()) } returns relationship @@ -320,7 +322,7 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(type = listOf("isContainedIn")) + val relationship = Relationship(objectId = generateRandomObjectId(), type = listOf(relationshipType)) val property = Property(name = "depth", value = 0) every { neo4jRepository.hasRelationshipInstance(match { it.label == "Entity" }, any(), any()) } returns true @@ -366,8 +368,8 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(type = listOf("isContainedIn")) - val measuredByRelationship = Relationship(type = listOf("measuredBy")) + val relationship = Relationship(objectId = generateRandomObjectId(), type = listOf(relationshipType)) + val measuredByRelationship = Relationship(objectId = generateRandomObjectId(), type = listOf(relationshipType)) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true every { relationshipRepository.getRelationshipOfSubject(fishUri, any()) } returns relationship @@ -497,7 +499,7 @@ class EntityAttributeServiceTests { """.trimIndent() val expandedPayload = parseAndExpandAttributeFragment(relationshipType, payload, listOf(AQUAC_COMPOUND_CONTEXT)) - val relationship = Relationship(type = listOf("isContainedIn")) + val relationship = Relationship(objectId = generateRandomObjectId(), type = listOf(relationshipType)) every { neo4jRepository.hasRelationshipInstance(any(), any(), any()) } returns true every { relationshipRepository.getRelationshipOfSubject(any(), any(), any()) } returns relationship every { neo4jRepository.updateRelationshipTargetOfSubject(any(), any(), any()) } returns true @@ -541,4 +543,7 @@ class EntityAttributeServiceTests { confirmVerified() } + + private fun generateRandomObjectId(): URI = + "urn:ngsi-ld:Entity:${UUID.randomUUID()}".toUri() } From acea4afc58e4ad9c2e31cd572e346693ee1c38c6 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 29 Aug 2021 11:45:08 +0200 Subject: [PATCH 53/56] fix(entity): migrate object of a relationship also when targeting a partial entity --- ...V09__add_object_id_in_relationships_to_partial_entity.cypher | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 entity-service/src/main/resources/neo4j/migrations/V09__add_object_id_in_relationships_to_partial_entity.cypher diff --git a/entity-service/src/main/resources/neo4j/migrations/V09__add_object_id_in_relationships_to_partial_entity.cypher b/entity-service/src/main/resources/neo4j/migrations/V09__add_object_id_in_relationships_to_partial_entity.cypher new file mode 100644 index 000000000..101644012 --- /dev/null +++ b/entity-service/src/main/resources/neo4j/migrations/V09__add_object_id_in_relationships_to_partial_entity.cypher @@ -0,0 +1,2 @@ +MATCH (r:Attribute:Relationship)-[]->(e:PartialEntity) +SET r.objectId = e.id From 315eac65e54ab786b8eced0d1666b49f5d2bc934 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 29 Aug 2021 17:04:03 +0200 Subject: [PATCH 54/56] fix(entity): switch back to individual findById for search on entities - findAllById is doing a very long last query taking up to 4 seconds for 30 results... --- .../egm/stellio/entity/service/EntityService.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 6a7650100..5309327b6 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -202,13 +202,18 @@ class EntityService( .groupBy({ it.first }, { it.second }) fun getFullEntitiesById(entitiesIds: List, includeSysAttrs: Boolean = false): List = - entityRepository.findAllById(entitiesIds) + entitiesIds .map { + entityRepository.findById(it) + } + .filter { it.isPresent } + .map { + val entity = it.get() JsonLdEntity( - it.serializeCoreProperties(includeSysAttrs) - .plus(serializeEntityProperties(it.properties, includeSysAttrs)) - .plus(serializeEntityRelationships(it.relationships, includeSysAttrs)), - it.contexts + entity.serializeCoreProperties(includeSysAttrs) + .plus(serializeEntityProperties(entity.properties, includeSysAttrs)) + .plus(serializeEntityRelationships(entity.relationships, includeSysAttrs)), + entity.contexts ) }.sortedBy { // as findAllById does not preserve order of the results, sort them back by id (search order) From d50154f0a965ba47c7202227e860eafaec7ea3a0 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 12 Sep 2021 09:54:04 +0200 Subject: [PATCH 55/56] fix: add an index on id of partial entities to prevent from unexpected duplicates (#495) - split into 2 migration scripts as schema modifications can't be done after a write query --- .../migrations/V10__merge_identical_partial_entities.cypher | 5 +++++ .../neo4j/migrations/V11__add_index_partial_entity.cypher | 1 + 2 files changed, 6 insertions(+) create mode 100644 entity-service/src/main/resources/neo4j/migrations/V10__merge_identical_partial_entities.cypher create mode 100644 entity-service/src/main/resources/neo4j/migrations/V11__add_index_partial_entity.cypher diff --git a/entity-service/src/main/resources/neo4j/migrations/V10__merge_identical_partial_entities.cypher b/entity-service/src/main/resources/neo4j/migrations/V10__merge_identical_partial_entities.cypher new file mode 100644 index 000000000..4d39029bc --- /dev/null +++ b/entity-service/src/main/resources/neo4j/migrations/V10__merge_identical_partial_entities.cypher @@ -0,0 +1,5 @@ +// merge any duplicated partial entity before adding the index +MATCH (n:PartialEntity) +WITH n.id as id, COLLECT(n) AS ns WHERE size(ns) > 1 +CALL apoc.refactor.mergeNodes(ns) YIELD node +RETURN node; diff --git a/entity-service/src/main/resources/neo4j/migrations/V11__add_index_partial_entity.cypher b/entity-service/src/main/resources/neo4j/migrations/V11__add_index_partial_entity.cypher new file mode 100644 index 000000000..783724a6f --- /dev/null +++ b/entity-service/src/main/resources/neo4j/migrations/V11__add_index_partial_entity.cypher @@ -0,0 +1 @@ +CREATE INDEX partial_entity_id_index IF NOT EXISTS FOR (e:PartialEntity) ON (e.id); From 2d2ebd884354ef9fd71cb5dcfce0016574f2471b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Sep 2021 06:52:14 +0200 Subject: [PATCH 56/56] chore(deps): Bump zaproxy/action-baseline from 0.4.0 to 0.5.0 (#496) Bumps [zaproxy/action-baseline](https://github.com/zaproxy/action-baseline) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/zaproxy/action-baseline/releases) - [Changelog](https://github.com/zaproxy/action-baseline/blob/master/CHANGELOG.md) - [Commits](https://github.com/zaproxy/action-baseline/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: zaproxy/action-baseline dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/zap_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zap_scan.yml b/.github/workflows/zap_scan.yml index b7d82a393..4fed35f43 100644 --- a/.github/workflows/zap_scan.yml +++ b/.github/workflows/zap_scan.yml @@ -20,7 +20,7 @@ jobs: - name: Launh docker-compose stack run: docker-compose -f docker-compose.yml up -d - name: ZAP Scan - uses: zaproxy/action-baseline@v0.4.0 + uses: zaproxy/action-baseline@v0.5.0 # Then run the scan against the locally running instance with: target: 'http://localhost:8080/ngsi-ld/v1/entities'