From ed7e1113a352349d4a8dda6aa7edc4329b537d0c Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 29 May 2024 18:22:09 +0200 Subject: [PATCH 01/14] feat(mqtt notification): --- .../egm/stellio/shared/model/ApiExceptions.kt | 2 + subscription-service/build.gradle.kts | 2 + .../service/NotificationService.kt | 69 +- .../service/mqtt/MQTTNotificationData.kt | 19 + .../service/mqtt/MQTTNotificationService.kt | 68 ++ .../service/mqtt/MQTTVersionService.kt | 50 + .../stellio/subscription/service/mqtt/Mqtt.kt | 26 + .../src/main/resources/application.properties | 21 +- .../service/MqttNotificationServiceTest.kt | 138 +++ .../service/MqttVersionServiceTest.kt | 72 ++ .../service/NotificationServiceTests.kt | 62 ++ .../support/WithMosquittoContainer.kt | 27 + .../test/resources/mosquitto/mosquitto.conf | 906 ++++++++++++++++++ 13 files changed, 1425 insertions(+), 37 deletions(-) create mode 100644 subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt create mode 100644 subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt create mode 100644 subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt create mode 100644 subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt create mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt create mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt create mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt create mode 100644 subscription-service/src/test/resources/mosquitto/mosquitto.conf diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt index e96b511082..41d0a796c8 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ApiExceptions.kt @@ -20,6 +20,7 @@ data class NotImplementedException(override val message: String) : APIException( data class LdContextNotAvailableException(override val message: String) : APIException(message) data class NonexistentTenantException(override val message: String) : APIException(message) data class NotAcceptableException(override val message: String) : APIException(message) +data class BadSchemeException(override val message: String) : APIException(message) fun Throwable.toAPIException(specificMessage: String? = null): APIException = when (this) { @@ -28,5 +29,6 @@ fun Throwable.toAPIException(specificMessage: String? = null): APIException = if (this.code == JsonLdErrorCode.LOADING_REMOTE_CONTEXT_FAILED) LdContextNotAvailableException(specificMessage ?: "Unable to load remote context (cause was: $this)") else BadRequestDataException("Unexpected error while parsing payload (cause was: $this)") + else -> BadRequestDataException(specificMessage ?: this.localizedMessage) } diff --git a/subscription-service/build.gradle.kts b/subscription-service/build.gradle.kts index 4b8eec5dff..cee56e3201 100644 --- a/subscription-service/build.gradle.kts +++ b/subscription-service/build.gradle.kts @@ -20,6 +20,8 @@ dependencies { implementation("org.postgresql:r2dbc-postgresql") implementation("com.jayway.jsonpath:json-path:2.9.0") implementation(project(":shared")) + implementation("org.eclipse.paho:org.eclipse.paho.client.mqttv3:1.2.5") + implementation("org.eclipse.paho:org.eclipse.paho.mqttv5.client:1.2.5") detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.23.6") diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt index bef5e57996..d2a632a208 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt @@ -13,6 +13,8 @@ import com.egm.stellio.subscription.model.Notification import com.egm.stellio.subscription.model.NotificationParams import com.egm.stellio.subscription.model.NotificationTrigger import com.egm.stellio.subscription.model.Subscription +import com.egm.stellio.subscription.service.mqtt.MQTTNotificationService +import com.egm.stellio.subscription.service.mqtt.Mqtt import org.slf4j.LoggerFactory import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus @@ -23,7 +25,8 @@ import org.springframework.web.reactive.function.client.awaitExchange @Service class NotificationService( - private val subscriptionService: SubscriptionService + private val subscriptionService: SubscriptionService, + private val mqttNotificationService: MQTTNotificationService ) { private val logger = LoggerFactory.getLogger(javaClass) @@ -70,32 +73,50 @@ class NotificationService( ) val uri = subscription.notification.endpoint.uri.toString() logger.info("Notification is about to be sent to $uri for subscription ${subscription.id}") - val request = - WebClient.create(uri).post().contentType(mediaType).headers { - if (mediaType == MediaType.APPLICATION_JSON) { - it.set(HttpHeaders.LINK, subscriptionService.getContextsLink(subscription)) - } - if (tenantName != DEFAULT_TENANT_NAME) - it.set(NGSILD_TENANT_HEADER, tenantName) - subscription.notification.endpoint.receiverInfo?.forEach { endpointInfo -> - it.set(endpointInfo.key, endpointInfo.value) + + val headerMap: MutableMap = emptyMap().toMutableMap() + if (mediaType == MediaType.APPLICATION_JSON) { + headerMap[HttpHeaders.LINK] = subscriptionService.getContextsLink(subscription) + } + if (tenantName != DEFAULT_TENANT_NAME) + headerMap[NGSILD_TENANT_HEADER] = tenantName + subscription.notification.endpoint.receiverInfo?.forEach { endpointInfo -> + headerMap[endpointInfo.key] = endpointInfo.value + } + + val result = + kotlin.runCatching { + if (uri.startsWith(Mqtt.SCHEME.MQTT)) { + headerMap["Content-Type"] = mediaType.toString() // could be common with line 99 ? + Triple( + subscription, + notification, + mqttNotificationService.mqttNotifier( + notification = notification, + subscription = subscription, + headers = headerMap + ) + ) + } else { + val request = + WebClient.create(uri).post().contentType(mediaType).headers { it.setAll(headerMap) } + request + .bodyValue(serializeObject(notification)) + .awaitExchange { response -> + val success = response.statusCode() == HttpStatus.OK + logger.info( + "The notification sent has been received with ${if (success) "success" else "failure"}" + ) + if (!success) { + logger.error("Failed to send notification to $uri: ${response.statusCode()}") + } + Triple(subscription, notification, success) + } } + }.getOrElse { + Triple(subscription, notification, false) } - val result = kotlin.runCatching { - request - .bodyValue(serializeObject(notification)) - .awaitExchange { response -> - val success = response.statusCode() == HttpStatus.OK - logger.info("The notification sent has been received with ${if (success) "success" else "failure"}") - if (!success) { - logger.error("Failed to send notification to $uri: ${response.statusCode()}") - } - Triple(subscription, notification, success) - } - }.getOrElse { - Triple(subscription, notification, false) - } subscriptionService.updateSubscriptionNotification(result.first, result.second, result.third) return result } diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt new file mode 100644 index 0000000000..9c68832e5d --- /dev/null +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt @@ -0,0 +1,19 @@ +package com.egm.stellio.subscription.service.mqtt + +import com.egm.stellio.subscription.model.Notification + +data class MQTTNotificationData( + val topic: String, + val mqttMessage: MqttMessage, + val qos: Int, + val brokerUrl: String, + val clientId: String, + val username: String, + val password: String? = null, +) { + + data class MqttMessage( + val body: Notification, + val metadata: Map = emptyMap(), + ) +} diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt new file mode 100644 index 0000000000..fd440fb900 --- /dev/null +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt @@ -0,0 +1,68 @@ +package com.egm.stellio.subscription.service.mqtt + +import com.egm.stellio.shared.model.BadSchemeException +import com.egm.stellio.subscription.model.Notification +import com.egm.stellio.subscription.model.Subscription +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Service +import org.eclipse.paho.client.mqttv3.MqttException as MqttExceptionV3 +import org.eclipse.paho.mqttv5.common.MqttException as MqttExceptionV5 + +@Service +class MQTTNotificationService( + @Value("\${mqtt.clientId}") + private val clientId: String = "stellio-context-brokerUrl", + private val mqttVersionService: MQTTVersionService +) { + + private val logger = LoggerFactory.getLogger(javaClass) + + suspend fun mqttNotifier( + subscription: Subscription, + notification: Notification, + headers: Map + ): Boolean { + val endpoint = subscription.notification.endpoint + val uri = endpoint.uri + val userInfo = uri.userInfo.split(':') + val username = userInfo.getOrNull(0) ?: "" + val password = userInfo.getOrNull(1) + val brokerScheme = Mqtt.SCHEME.brokerSchemeMap[uri.scheme] + ?: throw BadSchemeException("${uri.scheme} is not a valid mqtt scheme") + + val brokerPort = if (uri.port != -1) uri.port else Mqtt.SCHEME.defaultPortMap[uri.scheme] + + val brokerUrl = "$brokerScheme://${uri.host}:$brokerPort" + val notifierInfo = endpoint.notifierInfo?.map { it.key to it.value }?.toMap() ?: emptyMap() + val qos = + notifierInfo[Mqtt.QualityOfService.KEY]?.let { Integer.parseInt(it) } ?: Mqtt.QualityOfService.AT_MOST_ONCE + + val data = MQTTNotificationData( + topic = uri.path, + brokerUrl = brokerUrl, + clientId = clientId, + qos = qos, + mqttMessage = MQTTNotificationData.MqttMessage(notification, headers), + username = username, + password = password + ) + + try { + val mqttVersion = notifierInfo[Mqtt.Version.KEY] + when (mqttVersion) { + Mqtt.Version.V3 -> mqttVersionService.callMqttV3(data) + Mqtt.Version.V5 -> mqttVersionService.callMqttV5(data) + else -> mqttVersionService.callMqttV5(data) + } + logger.info("successfull mqtt notification for uri : ${data.brokerUrl} version: $mqttVersion") + return true + } catch (e: MqttExceptionV3) { + logger.error("failed mqttv3 notification for uri : ${data.brokerUrl}", e) + return false + } catch (e: MqttExceptionV5) { + logger.error("failed mqttv5 notification for uri : ${data.brokerUrl}", e) + return false + } + } +} diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt new file mode 100644 index 0000000000..2d05bccd40 --- /dev/null +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt @@ -0,0 +1,50 @@ +package com.egm.stellio.subscription.service.mqtt + +import com.egm.stellio.shared.util.JsonUtils.serializeObject +import org.eclipse.paho.mqttv5.client.MqttConnectionOptions +import org.springframework.stereotype.Service +import org.eclipse.paho.client.mqttv3.MqttClient as MqttClientv3 +import org.eclipse.paho.client.mqttv3.MqttConnectOptions as MqttConnectOptionsv3 +import org.eclipse.paho.client.mqttv3.MqttMessage as MqttMessagev3 +import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence as MemoryPersistencev3 +import org.eclipse.paho.mqttv5.client.IMqttToken as IMqttTokenv5 +import org.eclipse.paho.mqttv5.client.MqttAsyncClient as MqttAsyncClientv5 +import org.eclipse.paho.mqttv5.client.persist.MemoryPersistence as MemoryPersistencev5 +import org.eclipse.paho.mqttv5.common.MqttMessage as MqttMessagev5 + +@Service +class MQTTVersionService { + + internal suspend fun callMqttV3(data: MQTTNotificationData) { + val persistence = MemoryPersistencev3() + val sampleClient = MqttClientv3(data.brokerUrl, data.clientId, persistence) + val connOpts = MqttConnectOptionsv3() + connOpts.isCleanSession = true + connOpts.userName = data.username + connOpts.password = data.password?.toCharArray() ?: "".toCharArray() + sampleClient.connect(connOpts) + val message = MqttMessagev3( + serializeObject(data.mqttMessage).toByteArray() + ) + message.qos = data.qos + sampleClient.publish(data.topic, message) + sampleClient.disconnect() + } + + internal suspend fun callMqttV5(data: MQTTNotificationData) { + val persistence = MemoryPersistencev5() + val sampleClient = MqttAsyncClientv5(data.brokerUrl, data.clientId, persistence) + val connOpts = MqttConnectionOptions() + connOpts.isCleanStart = true + connOpts.userName = data.username + connOpts.password = data.password?.toByteArray() + var token: IMqttTokenv5 = sampleClient.connect(connOpts) + token.waitForCompletion() + val message = MqttMessagev5(serializeObject(data.mqttMessage).toByteArray()) + message.qos = data.qos + token = sampleClient.publish(data.topic, message) + token.waitForCompletion() + sampleClient.disconnect() + sampleClient.close() + } +} diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt new file mode 100644 index 0000000000..610385daea --- /dev/null +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt @@ -0,0 +1,26 @@ +package com.egm.stellio.subscription.service.mqtt + +object Mqtt { + + object Version { + const val KEY = "MQTT-Version" + const val V3 = "mqtt3.1.1" + const val V5 = "mqtt5.0" + } + + object QualityOfService { + const val KEY = "MQTT-QoS" + const val AT_MOST_ONCE = 0 +// const val AT_LEAST_ONCE = 1 +// const val EXACTLY_ONCE = 2 + } + + object SCHEME { + const val MQTT = "mqtt" + const val MQTTS = "mqtts" + const val MQTT_DEFAULT_PORT = 1883 + const val MQTTS_DEFAULT_PORT = 8883 + val defaultPortMap = mapOf(MQTT to MQTT_DEFAULT_PORT, MQTTS to MQTTS_DEFAULT_PORT) + val brokerSchemeMap = mapOf(MQTT to "tcp", MQTTS to "ssl") + } +} diff --git a/subscription-service/src/main/resources/application.properties b/subscription-service/src/main/resources/application.properties index 39ddb7865b..730f430621 100644 --- a/subscription-service/src/main/resources/application.properties +++ b/subscription-service/src/main/resources/application.properties @@ -1,23 +1,18 @@ spring.config.import=classpath:/shared.properties - -spring.r2dbc.url = r2dbc:postgresql://localhost/stellio_subscription -spring.r2dbc.username = stellio -spring.r2dbc.password = stellio_password - +spring.r2dbc.url=r2dbc:postgresql://localhost/stellio_subscription +spring.r2dbc.username=stellio +spring.r2dbc.password=stellio_password # Required for Flyway to know where the DB is located -spring.flyway.url = jdbc:postgresql://localhost/stellio_subscription -spring.flyway.user = ${spring.r2dbc.username} -spring.flyway.password = ${spring.r2dbc.password} - +spring.flyway.url=jdbc:postgresql://localhost/stellio_subscription +spring.flyway.user=${spring.r2dbc.username} +spring.flyway.password=${spring.r2dbc.password} # Client registration used to get entities from search-service spring.security.oauth2.client.registration.keycloak.authorization-grant-type=client_credentials spring.security.oauth2.client.registration.keycloak.client-id=client-id spring.security.oauth2.client.registration.keycloak.client-secret=client-secret spring.security.oauth2.client.provider.keycloak.token-uri=https://my.sso/token - subscription.entity-service-url=http://localhost:8083 - # Stellio url used to form the link to get the contexts associated to a notification subscription.stellio-url=http://localhost:8080 - -server.port = 8084 +server.port=8084 +mqtt.clientId=stellio-mqtt-client diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt new file mode 100644 index 0000000000..2968e93216 --- /dev/null +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt @@ -0,0 +1,138 @@ +package com.egm.stellio.subscription.service + +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SUBSCRIPTION_TERM +import com.egm.stellio.shared.util.toUri +import com.egm.stellio.subscription.model.* +import com.egm.stellio.subscription.service.mqtt.MQTTNotificationData +import com.egm.stellio.subscription.service.mqtt.MQTTNotificationService +import com.egm.stellio.subscription.service.mqtt.MQTTVersionService +import com.egm.stellio.subscription.service.mqtt.Mqtt +import com.ninjasquad.springmockk.MockkBean +import io.mockk.coEvery +import io.mockk.coVerify +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [MQTTNotificationService::class]) +@ActiveProfiles("test") +class MqttNotificationServiceTest { + + @Autowired + private lateinit var mqttNotificationService: MQTTNotificationService + + @MockkBean + private lateinit var mqttVersionService: MQTTVersionService + + private val mqttSubscription = Subscription( + type = NGSILD_SUBSCRIPTION_TERM, + subscriptionName = "My Subscription", + description = "My beautiful subscription", + entities = emptySet(), + notification = NotificationParams( + attributes = emptyList(), + endpoint = Endpoint( + uri = "mqtt://test@localhost:1883/notification".toUri(), + accept = Endpoint.AcceptType.JSONLD, + notifierInfo = listOf( + EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V3), + EndpointInfo(Mqtt.QualityOfService.KEY, Mqtt.QualityOfService.AT_MOST_ONCE.toString()) + ) + ) + ), + contexts = emptyList() + ) + + private val mqttSubscriptionV5 = Subscription( + type = NGSILD_SUBSCRIPTION_TERM, + entities = emptySet(), + notification = NotificationParams( + attributes = emptyList(), + endpoint = Endpoint( + uri = "mqtt://test@localhost:1883/notification".toUri(), + notifierInfo = listOf( + EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V5) + ) + ) + ), + contexts = emptyList() + ) + private val validMqttNotificationData = MQTTNotificationData( + brokerUrl = "tcp://localhost:1883", + clientId = "clientId", + username = "test", + topic = "/notification", + qos = 0, + mqttMessage = MQTTNotificationData.MqttMessage(getNotificationForSubscription(mqttSubscription), emptyMap()) + ) + + private fun getNotificationForSubscription(subscription: Subscription) = Notification( + subscriptionId = subscription.id, + data = listOf(mapOf("hello" to "world")) + ) + + @Test + fun `mqttNotifier should process endpoint uri to get connexion information`() = runTest { + val subscription = mqttSubscription + coEvery { mqttVersionService.callMqttV3(any()) } returns Unit + assert( + mqttNotificationService.mqttNotifier( + subscription, + getNotificationForSubscription(subscription), + mapOf() + ) + ) + + coVerify { + mqttVersionService.callMqttV3( + match { + it.username == validMqttNotificationData.username && + it.password == validMqttNotificationData.password && + it.topic == validMqttNotificationData.topic && + it.brokerUrl == validMqttNotificationData.brokerUrl + } + ) + } + } + + @Test + fun `mqttNotifier should use notifier info to choose the mqtt version`() = runTest { + val subscription = mqttSubscription + coEvery { mqttVersionService.callMqttV3(any()) } returns Unit + coEvery { mqttVersionService.callMqttV5(any()) } returns Unit + + mqttNotificationService.mqttNotifier( + subscription, + getNotificationForSubscription(subscription), + mapOf() + ) + coVerify(exactly = 1) { + mqttVersionService.callMqttV3( + any() + ) + } + coVerify(exactly = 0) { + mqttVersionService.callMqttV5( + any() + ) + } + + mqttNotificationService.mqttNotifier( + mqttSubscriptionV5, + getNotificationForSubscription(subscription), + mapOf() + ) + coVerify(exactly = 1) { + mqttVersionService.callMqttV5( + any() + ) + } + coVerify(exactly = 1) { + mqttVersionService.callMqttV3( + any() + ) + } + } +} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt new file mode 100644 index 0000000000..d3bbea23ce --- /dev/null +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt @@ -0,0 +1,72 @@ +package com.egm.stellio.subscription.service + +import com.egm.stellio.subscription.model.Notification +import com.egm.stellio.subscription.service.mqtt.MQTTNotificationData +import com.egm.stellio.subscription.service.mqtt.MQTTVersionService +import com.egm.stellio.subscription.service.mqtt.Mqtt +import com.egm.stellio.subscription.support.WithMosquittoContainer +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertDoesNotThrow +import org.junit.jupiter.api.assertThrows +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles +import java.net.URI +import org.eclipse.paho.client.mqttv3.MqttException as MqttExceptionV3 +import org.eclipse.paho.mqttv5.common.MqttException as MqttExceptionV5 + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [MQTTVersionService::class]) +@ActiveProfiles("test") +class MqttVersionServiceTest : WithMosquittoContainer { + + @Autowired + private lateinit var mqttVersionService: MQTTVersionService + + private val mqttContainerPort = WithMosquittoContainer.mosquittoContainer.getMappedPort( + Mqtt.SCHEME.MQTT_DEFAULT_PORT + ) + + private val notification = Notification( + subscriptionId = URI("1"), + data = listOf(mapOf("hello" to "world")) + ) + + private val validMqttNotificationData = MQTTNotificationData( + brokerUrl = "tcp://localhost:$mqttContainerPort", + clientId = "clientId", + username = "test", + topic = "notification", + qos = 0, + mqttMessage = MQTTNotificationData.MqttMessage(notification, emptyMap()) + ) + + private val invalidUriMqttNotificationData = MQTTNotificationData( + brokerUrl = "tcp://badHost:1883", + clientId = "clientId", + username = "test", + topic = "notification", + qos = 0, + mqttMessage = MQTTNotificationData.MqttMessage(notification, emptyMap()) + ) + + @Test + fun `sending mqttV3 notification with good uri should succeed`() = runTest { + assertDoesNotThrow { mqttVersionService.callMqttV3(validMqttNotificationData) } + } + + @Test + fun `sending mqttV3 notification with bad uri should throw`() = runTest { + assertThrows { mqttVersionService.callMqttV3(invalidUriMqttNotificationData) } + } + + @Test + fun `sending mqttV5 notification with good uri should succeed`() = runTest { + assertDoesNotThrow { mqttVersionService.callMqttV5(validMqttNotificationData) } + } + + @Test + fun `sending mqttV5 notification with bad uri should throw`() = runTest { + assertThrows { mqttVersionService.callMqttV5(invalidUriMqttNotificationData) } + } +} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt index ef72765580..c98228fa5c 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt @@ -13,9 +13,11 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_TERM import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap import com.egm.stellio.shared.web.NGSILD_TENANT_HEADER import com.egm.stellio.subscription.model.Endpoint +import com.egm.stellio.subscription.model.EndpointInfo import com.egm.stellio.subscription.model.NotificationParams import com.egm.stellio.subscription.model.NotificationParams.FormatType import com.egm.stellio.subscription.model.NotificationTrigger.* +import com.egm.stellio.subscription.service.mqtt.MQTTNotificationService import com.egm.stellio.subscription.support.gimmeRawSubscription import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.junit5.WireMockTest @@ -42,6 +44,9 @@ class NotificationServiceTests { @MockkBean private lateinit var subscriptionService: SubscriptionService + @MockkBean + private lateinit var mqttNotificationService: MQTTNotificationService + @Autowired private lateinit var notificationService: NotificationService @@ -558,4 +563,61 @@ class NotificationServiceTests { } } } + + @Test + fun `callSuscriber should ask mqttNotifier if the brokerUrl startWith mqtt`() = runTest { + val subscription = gimmeRawSubscription().copy( + notification = NotificationParams( + attributes = emptyList(), + endpoint = Endpoint( + uri = "mqtt://localhost:8089/notification".toUri(), + accept = Endpoint.AcceptType.JSON + ) + ) + ) + + coEvery { subscriptionService.getContextsLink(any()) } returns buildContextLinkHeader(NGSILD_TEST_CORE_CONTEXT) + coEvery { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } returns 1 + + coEvery { mqttNotificationService.mqttNotifier(any(), any(), any()) } returns true + + notificationService.callSubscriber(subscription, rawEntity.deserializeAsMap()) + + coVerify(exactly = 1) { mqttNotificationService.mqttNotifier(any(), any(), any()) } + } + + @Test + fun `callSuscriber should generate headers for mqtt notification`() = runTest { + val infoKey = "hello" + val infoValue = "world" + val subscription = gimmeRawSubscription().copy( + notification = NotificationParams( + attributes = emptyList(), + endpoint = Endpoint( + uri = "mqtt://localhost:8089/notification".toUri(), + accept = Endpoint.AcceptType.JSON, + receiverInfo = listOf(EndpointInfo(key = infoKey, value = infoValue)), + ) + ) + ) + + coEvery { mqttNotificationService.mqttNotifier(any(), any(), any()) } returns true + + coEvery { subscriptionService.getContextsLink(any()) } returns buildContextLinkHeader(NGSILD_TEST_CORE_CONTEXT) + coEvery { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } returns 1 + + notificationService.callSubscriber(subscription, rawEntity.deserializeAsMap()) + + coVerify { + mqttNotificationService.mqttNotifier( + any(), + any(), + match { + it["Content-Type"] == "application/json" && + it[infoKey] == infoValue && + it["Link"] != null + } + ) + } + } } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt new file mode 100644 index 0000000000..5db96c764a --- /dev/null +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt @@ -0,0 +1,27 @@ +package com.egm.stellio.subscription.support + +import org.testcontainers.containers.GenericContainer +import org.testcontainers.utility.DockerImageName +import org.testcontainers.utility.MountableFile + +interface WithMosquittoContainer { + + companion object { + + private val mosquittoImage: DockerImageName = + DockerImageName.parse("eclipse-mosquitto:2.0.18") + + val mosquittoContainer = GenericContainer(mosquittoImage).apply { + withReuse(true) + withExposedPorts(1883) + withCopyFileToContainer( + MountableFile.forClasspathResource("/mosquitto/mosquitto.conf"), + "/mosquitto/config/mosquitto.conf" + ) + } + + init { + mosquittoContainer.start() + } + } +} diff --git a/subscription-service/src/test/resources/mosquitto/mosquitto.conf b/subscription-service/src/test/resources/mosquitto/mosquitto.conf new file mode 100644 index 0000000000..5b0f2e9f29 --- /dev/null +++ b/subscription-service/src/test/resources/mosquitto/mosquitto.conf @@ -0,0 +1,906 @@ +# Config file for mosquitto +# +# See mosquitto.conf(5) for more information. +# +# Default values are shown, uncomment to change. +# +# Use the # character to indicate a comment, but only if it is the +# very first character on the line. + +# ================================================================= +# General configuration +# ================================================================= + +# Use per listener security settings. +# +# It is recommended this option be set before any other options. +# +# If this option is set to true, then all authentication and access control +# options are controlled on a per listener basis. The following options are +# affected: +# +# acl_file +allow_anonymous true +listener 1883 +listener 9001 +# allow_zero_length_clientid +# auto_id_prefix +# password_file +# plugin +# plugin_opt_* +# psk_file +# +# Note that if set to true, then a durable client (i.e. with clean session set +# to false) that has disconnected will use the ACL settings defined for the +# listener that it was most recently connected to. +# +# The default behaviour is for this to be set to false, which maintains the +# setting behaviour from previous versions of mosquitto. +#per_listener_settings false + + +# This option controls whether a client is allowed to connect with a zero +# length client id or not. This option only affects clients using MQTT v3.1.1 +# and later. If set to false, clients connecting with a zero length client id +# are disconnected. If set to true, clients will be allocated a client id by +# the broker. This means it is only useful for clients with clean session set +# to true. +#allow_zero_length_clientid true + +# If allow_zero_length_clientid is true, this option allows you to set a prefix +# to automatically generated client ids to aid visibility in logs. +# Defaults to 'auto-' +#auto_id_prefix auto- + +# This option affects the scenario when a client subscribes to a topic that has +# retained messages. It is possible that the client that published the retained +# message to the topic had access at the time they published, but that access +# has been subsequently removed. If check_retain_source is set to true, the +# default, the source of a retained message will be checked for access rights +# before it is republished. When set to false, no check will be made and the +# retained message will always be published. This affects all listeners. +#check_retain_source true + +# QoS 1 and 2 messages will be allowed inflight per client until this limit +# is exceeded. Defaults to 0. (No maximum) +# See also max_inflight_messages +#max_inflight_bytes 0 + +# The maximum number of QoS 1 and 2 messages currently inflight per +# client. +# This includes messages that are partway through handshakes and +# those that are being retried. Defaults to 20. Set to 0 for no +# maximum. Setting to 1 will guarantee in-order delivery of QoS 1 +# and 2 messages. +#max_inflight_messages 20 + +# For MQTT v5 clients, it is possible to have the server send a "server +# keepalive" value that will override the keepalive value set by the client. +# This is intended to be used as a mechanism to say that the server will +# disconnect the client earlier than it anticipated, and that the client should +# use the new keepalive value. The max_keepalive option allows you to specify +# that clients may only connect with keepalive less than or equal to this +# value, otherwise they will be sent a server keepalive telling them to use +# max_keepalive. This only applies to MQTT v5 clients. The default, and maximum +# value allowable, is 65535. +# +# Set to 0 to allow clients to set keepalive = 0, which means no keepalive +# checks are made and the client will never be disconnected by the broker if no +# messages are received. You should be very sure this is the behaviour that you +# want. +# +# For MQTT v3.1.1 and v3.1 clients, there is no mechanism to tell the client +# what keepalive value they should use. If an MQTT v3.1.1 or v3.1 client +# specifies a keepalive time greater than max_keepalive they will be sent a +# CONNACK message with the "identifier rejected" reason code, and disconnected. +# +#max_keepalive 65535 + +# For MQTT v5 clients, it is possible to have the server send a "maximum packet +# size" value that will instruct the client it will not accept MQTT packets +# with size greater than max_packet_size bytes. This applies to the full MQTT +# packet, not just the payload. Setting this option to a positive value will +# set the maximum packet size to that number of bytes. If a client sends a +# packet which is larger than this value, it will be disconnected. This applies +# to all clients regardless of the protocol version they are using, but v3.1.1 +# and earlier clients will of course not have received the maximum packet size +# information. Defaults to no limit. Setting below 20 bytes is forbidden +# because it is likely to interfere with ordinary client operation, even with +# very small payloads. +#max_packet_size 0 + +# QoS 1 and 2 messages above those currently in-flight will be queued per +# client until this limit is exceeded. Defaults to 0. (No maximum) +# See also max_queued_messages. +# If both max_queued_messages and max_queued_bytes are specified, packets will +# be queued until the first limit is reached. +#max_queued_bytes 0 + +# Set the maximum QoS supported. Clients publishing at a QoS higher than +# specified here will be disconnected. +#max_qos 2 + +# The maximum number of QoS 1 and 2 messages to hold in a queue per client +# above those that are currently in-flight. Defaults to 1000. Set +# to 0 for no maximum (not recommended). +# See also queue_qos0_messages. +# See also max_queued_bytes. +#max_queued_messages 1000 +# +# This option sets the maximum number of heap memory bytes that the broker will +# allocate, and hence sets a hard limit on memory use by the broker. Memory +# requests that exceed this value will be denied. The effect will vary +# depending on what has been denied. If an incoming message is being processed, +# then the message will be dropped and the publishing client will be +# disconnected. If an outgoing message is being sent, then the individual +# message will be dropped and the receiving client will be disconnected. +# Defaults to no limit. +#memory_limit 0 + +# This option sets the maximum publish payload size that the broker will allow. +# Received messages that exceed this size will not be accepted by the broker. +# The default value is 0, which means that all valid MQTT messages are +# accepted. MQTT imposes a maximum payload size of 268435455 bytes. +#message_size_limit 0 + +# This option allows the session of persistent clients (those with clean +# session set to false) that are not currently connected to be removed if they +# do not reconnect within a certain time frame. This is a non-standard option +# in MQTT v3.1. MQTT v3.1.1 and v5.0 allow brokers to remove client sessions. +# +# Badly designed clients may set clean session to false whilst using a randomly +# generated client id. This leads to persistent clients that connect once and +# never reconnect. This option allows these clients to be removed. This option +# allows persistent clients (those with clean session set to false) to be +# removed if they do not reconnect within a certain time frame. +# +# The expiration period should be an integer followed by one of h d w m y for +# hour, day, week, month and year respectively. For example +# +# persistent_client_expiration 2m +# persistent_client_expiration 14d +# persistent_client_expiration 1y +# +# The default if not set is to never expire persistent clients. +#persistent_client_expiration + +# Write process id to a file. Default is a blank string which means +# a pid file shouldn't be written. +# This should be set to /var/run/mosquitto/mosquitto.pid if mosquitto is +# being run automatically on boot with an init script and +# start-stop-daemon or similar. +#pid_file + +# Set to true to queue messages with QoS 0 when a persistent client is +# disconnected. These messages are included in the limit imposed by +# max_queued_messages and max_queued_bytes +# Defaults to false. +# This is a non-standard option for the MQTT v3.1 spec but is allowed in +# v3.1.1. +#queue_qos0_messages false + +# Set to false to disable retained message support. If a client publishes a +# message with the retain bit set, it will be disconnected if this is set to +# false. +#retain_available true + +# Disable Nagle's algorithm on client sockets. This has the effect of reducing +# latency of individual messages at the potential cost of increasing the number +# of packets being sent. +#set_tcp_nodelay false + +# Time in seconds between updates of the $SYS tree. +# Set to 0 to disable the publishing of the $SYS tree. +#sys_interval 10 + +# The MQTT specification requires that the QoS of a message delivered to a +# subscriber is never upgraded to match the QoS of the subscription. Enabling +# this option changes this behaviour. If upgrade_outgoing_qos is set true, +# messages sent to a subscriber will always match the QoS of its subscription. +# This is a non-standard option explicitly disallowed by the spec. +#upgrade_outgoing_qos false + +# When run as root, drop privileges to this user and its primary +# group. +# Set to root to stay as root, but this is not recommended. +# If set to "mosquitto", or left unset, and the "mosquitto" user does not exist +# then it will drop privileges to the "nobody" user instead. +# If run as a non-root user, this setting has no effect. +# Note that on Windows this has no effect and so mosquitto should be started by +# the user you wish it to run as. +#user mosquitto + +# ================================================================= +# Listeners +# ================================================================= + +# Listen on a port/ip address combination. By using this variable +# multiple times, mosquitto can listen on more than one port. If +# this variable is used and neither bind_address nor port given, +# then the default listener will not be started. +# The port number to listen on must be given. Optionally, an ip +# address or host name may be supplied as a second argument. In +# this case, mosquitto will attempt to bind the listener to that +# address and so restrict access to the associated network and +# interface. By default, mosquitto will listen on all interfaces. +# Note that for a websockets listener it is not possible to bind to a host +# name. +# +# On systems that support Unix Domain Sockets, it is also possible +# to create a # Unix socket rather than opening a TCP socket. In +# this case, the port number should be set to 0 and a unix socket +# path must be provided, e.g. +# listener 0 /tmp/mosquitto.sock +# +# listener port-number [ip address/host name/unix socket path] +#listener + +# By default, a listener will attempt to listen on all supported IP protocol +# versions. If you do not have an IPv4 or IPv6 interface you may wish to +# disable support for either of those protocol versions. In particular, note +# that due to the limitations of the websockets library, it will only ever +# attempt to open IPv6 sockets if IPv6 support is compiled in, and so will fail +# if IPv6 is not available. +# +# Set to `ipv4` to force the listener to only use IPv4, or set to `ipv6` to +# force the listener to only use IPv6. If you want support for both IPv4 and +# IPv6, then do not use the socket_domain option. +# +#socket_domain + +# Bind the listener to a specific interface. This is similar to +# the [ip address/host name] part of the listener definition, but is useful +# when an interface has multiple addresses or the address may change. If used +# with the [ip address/host name] part of the listener definition, then the +# bind_interface option will take priority. +# Not available on Windows. +# +# Example: bind_interface eth0 +#bind_interface + +# When a listener is using the websockets protocol, it is possible to serve +# http data as well. Set http_dir to a directory which contains the files you +# wish to serve. If this option is not specified, then no normal http +# connections will be possible. +#http_dir + +# The maximum number of client connections to allow. This is +# a per listener setting. +# Default is -1, which means unlimited connections. +# Note that other process limits mean that unlimited connections +# are not really possible. Typically the default maximum number of +# connections possible is around 1024. +#max_connections -1 + +# The listener can be restricted to operating within a topic hierarchy using +# the mount_point option. This is achieved be prefixing the mount_point string +# to all topics for any clients connected to this listener. This prefixing only +# happens internally to the broker; the client will not see the prefix. +#mount_point + +# Choose the protocol to use when listening. +# This can be either mqtt or websockets. +# Certificate based TLS may be used with websockets, except that only the +# cafile, certfile, keyfile, ciphers, and ciphers_tls13 options are supported. +#protocol mqtt + +# Set use_username_as_clientid to true to replace the clientid that a client +# connected with with its username. This allows authentication to be tied to +# the clientid, which means that it is possible to prevent one client +# disconnecting another by using the same clientid. +# If a client connects with no username it will be disconnected as not +# authorised when this option is set to true. +# Do not use in conjunction with clientid_prefixes. +# See also use_identity_as_username. +# This does not apply globally, but on a per-listener basis. +#use_username_as_clientid + +# Change the websockets headers size. This is a global option, it is not +# possible to set per listener. This option sets the size of the buffer used in +# the libwebsockets library when reading HTTP headers. If you are passing large +# header data such as cookies then you may need to increase this value. If left +# unset, or set to 0, then the default of 1024 bytes will be used. +#websockets_headers_size + +# ----------------------------------------------------------------- +# Certificate based SSL/TLS support +# ----------------------------------------------------------------- +# The following options can be used to enable certificate based SSL/TLS support +# for this listener. Note that the recommended port for MQTT over TLS is 8883, +# but this must be set manually. +# +# See also the mosquitto-tls man page and the "Pre-shared-key based SSL/TLS +# support" section. Only one of certificate or PSK encryption support can be +# enabled for any listener. + +# Both of certfile and keyfile must be defined to enable certificate based +# TLS encryption. + +# Path to the PEM encoded server certificate. +#certfile + +# Path to the PEM encoded keyfile. +#keyfile + +# If you wish to control which encryption ciphers are used, use the ciphers +# option. The list of available ciphers can be optained using the "openssl +# ciphers" command and should be provided in the same format as the output of +# that command. This applies to TLS 1.2 and earlier versions only. Use +# ciphers_tls1.3 for TLS v1.3. +#ciphers + +# Choose which TLS v1.3 ciphersuites are used for this listener. +# Defaults to "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" +#ciphers_tls1.3 + +# If you have require_certificate set to true, you can create a certificate +# revocation list file to revoke access to particular client certificates. If +# you have done this, use crlfile to point to the PEM encoded revocation file. +#crlfile + +# To allow the use of ephemeral DH key exchange, which provides forward +# security, the listener must load DH parameters. This can be specified with +# the dhparamfile option. The dhparamfile can be generated with the command +# e.g. "openssl dhparam -out dhparam.pem 2048" +#dhparamfile + +# By default an TLS enabled listener will operate in a similar fashion to a +# https enabled web server, in that the server has a certificate signed by a CA +# and the client will verify that it is a trusted certificate. The overall aim +# is encryption of the network traffic. By setting require_certificate to true, +# the client must provide a valid certificate in order for the network +# connection to proceed. This allows access to the broker to be controlled +# outside of the mechanisms provided by MQTT. +#require_certificate false + +# cafile and capath define methods of accessing the PEM encoded +# Certificate Authority certificates that will be considered trusted when +# checking incoming client certificates. +# cafile defines the path to a file containing the CA certificates. +# capath defines a directory that will be searched for files +# containing the CA certificates. For capath to work correctly, the +# certificate files must have ".crt" as the file ending and you must run +# "openssl rehash " each time you add/remove a certificate. +#cafile +#capath + + +# If require_certificate is true, you may set use_identity_as_username to true +# to use the CN value from the client certificate as a username. If this is +# true, the password_file option will not be used for this listener. +#use_identity_as_username false + +# ----------------------------------------------------------------- +# Pre-shared-key based SSL/TLS support +# ----------------------------------------------------------------- +# The following options can be used to enable PSK based SSL/TLS support for +# this listener. Note that the recommended port for MQTT over TLS is 8883, but +# this must be set manually. +# +# See also the mosquitto-tls man page and the "Certificate based SSL/TLS +# support" section. Only one of certificate or PSK encryption support can be +# enabled for any listener. + +# The psk_hint option enables pre-shared-key support for this listener and also +# acts as an identifier for this listener. The hint is sent to clients and may +# be used locally to aid authentication. The hint is a free form string that +# doesn't have much meaning in itself, so feel free to be creative. +# If this option is provided, see psk_file to define the pre-shared keys to be +# used or create a security plugin to handle them. +#psk_hint + +# When using PSK, the encryption ciphers used will be chosen from the list of +# available PSK ciphers. If you want to control which ciphers are available, +# use the "ciphers" option. The list of available ciphers can be optained +# using the "openssl ciphers" command and should be provided in the same format +# as the output of that command. +#ciphers + +# Set use_identity_as_username to have the psk identity sent by the client used +# as its username. Authentication will be carried out using the PSK rather than +# the MQTT username/password and so password_file will not be used for this +# listener. +#use_identity_as_username false + + +# ================================================================= +# Persistence +# ================================================================= + +# If persistence is enabled, save the in-memory database to disk +# every autosave_interval seconds. If set to 0, the persistence +# database will only be written when mosquitto exits. See also +# autosave_on_changes. +# Note that writing of the persistence database can be forced by +# sending mosquitto a SIGUSR1 signal. +#autosave_interval 1800 + +# If true, mosquitto will count the number of subscription changes, retained +# messages received and queued messages and if the total exceeds +# autosave_interval then the in-memory database will be saved to disk. +# If false, mosquitto will save the in-memory database to disk by treating +# autosave_interval as a time in seconds. +#autosave_on_changes false + +# Save persistent message data to disk (true/false). +# This saves information about all messages, including +# subscriptions, currently in-flight messages and retained +# messages. +# retained_persistence is a synonym for this option. +#persistence false + +# The filename to use for the persistent database, not including +# the path. +#persistence_file mosquitto.db + +# Location for persistent database. +# Default is an empty string (current directory). +# Set to e.g. /var/lib/mosquitto if running as a proper service on Linux or +# similar. +#persistence_location + + +# ================================================================= +# Logging +# ================================================================= + +# Places to log to. Use multiple log_dest lines for multiple +# logging destinations. +# Possible destinations are: stdout stderr syslog topic file dlt +# +# stdout and stderr log to the console on the named output. +# +# syslog uses the userspace syslog facility which usually ends up +# in /var/log/messages or similar. +# +# topic logs to the broker topic '$SYS/broker/log/', +# where severity is one of D, E, W, N, I, M which are debug, error, +# warning, notice, information and message. Message type severity is used by +# the subscribe/unsubscribe log_types and publishes log messages to +# $SYS/broker/log/M/susbcribe or $SYS/broker/log/M/unsubscribe. +# +# The file destination requires an additional parameter which is the file to be +# logged to, e.g. "log_dest file /var/log/mosquitto.log". The file will be +# closed and reopened when the broker receives a HUP signal. Only a single file +# destination may be configured. +# +# The dlt destination is for the automotive `Diagnostic Log and Trace` tool. +# This requires that Mosquitto has been compiled with DLT support. +# +# Note that if the broker is running as a Windows service it will default to +# "log_dest none" and neither stdout nor stderr logging is available. +# Use "log_dest none" if you wish to disable logging. +#log_dest stderr + +# Types of messages to log. Use multiple log_type lines for logging +# multiple types of messages. +# Possible types are: debug, error, warning, notice, information, +# none, subscribe, unsubscribe, websockets, all. +# Note that debug type messages are for decoding the incoming/outgoing +# network packets. They are not logged in "topics". +#log_type error +#log_type warning +#log_type notice +#log_type information + + +# If set to true, client connection and disconnection messages will be included +# in the log. +#connection_messages true + +# If using syslog logging (not on Windows), messages will be logged to the +# "daemon" facility by default. Use the log_facility option to choose which of +# local0 to local7 to log to instead. The option value should be an integer +# value, e.g. "log_facility 5" to use local5. +#log_facility + +# If set to true, add a timestamp value to each log message. +#log_timestamp true + +# Set the format of the log timestamp. If left unset, this is the number of +# seconds since the Unix epoch. +# This is a free text string which will be passed to the strftime function. To +# get an ISO 8601 datetime, for example: +# log_timestamp_format %Y-%m-%dT%H:%M:%S +#log_timestamp_format + +# Change the websockets logging level. This is a global option, it is not +# possible to set per listener. This is an integer that is interpreted by +# libwebsockets as a bit mask for its lws_log_levels enum. See the +# libwebsockets documentation for more details. "log_type websockets" must also +# be enabled. +#websockets_log_level 0 + + +# ================================================================= +# Security +# ================================================================= + +# If set, only clients that have a matching prefix on their +# clientid will be allowed to connect to the broker. By default, +# all clients may connect. +# For example, setting "secure-" here would mean a client "secure- +# client" could connect but another with clientid "mqtt" couldn't. +#clientid_prefixes + +# Boolean value that determines whether clients that connect +# without providing a username are allowed to connect. If set to +# false then a password file should be created (see the +# password_file option) to control authenticated client access. +# +# Defaults to false, unless there are no listeners defined in the configuration +# file, in which case it is set to true, but connections are only allowed from +# the local machine. +#allow_anonymous false + +# ----------------------------------------------------------------- +# Default authentication and topic access control +# ----------------------------------------------------------------- + +# Control access to the broker using a password file. This file can be +# generated using the mosquitto_passwd utility. If TLS support is not compiled +# into mosquitto (it is recommended that TLS support should be included) then +# plain text passwords are used, in which case the file should be a text file +# with lines in the format: +# username:password +# The password (and colon) may be omitted if desired, although this +# offers very little in the way of security. +# +# See the TLS client require_certificate and use_identity_as_username options +# for alternative authentication options. If a plugin is used as well as +# password_file, the plugin check will be made first. +#password_file + +# Access may also be controlled using a pre-shared-key file. This requires +# TLS-PSK support and a listener configured to use it. The file should be text +# lines in the format: +# identity:key +# The key should be in hexadecimal format without a leading "0x". +# If an plugin is used as well, the plugin check will be made first. +#psk_file + +# Control access to topics on the broker using an access control list +# file. If this parameter is defined then only the topics listed will +# have access. +# If the first character of a line of the ACL file is a # it is treated as a +# comment. +# Topic access is added with lines of the format: +# +# topic [read|write|readwrite|deny] +# +# The access type is controlled using "read", "write", "readwrite" or "deny". +# This parameter is optional (unless contains a space character) - if +# not given then the access is read/write. can contain the + or # +# wildcards as in subscriptions. +# +# The "deny" option can used to explicity deny access to a topic that would +# otherwise be granted by a broader read/write/readwrite statement. Any "deny" +# topics are handled before topics that grant read/write access. +# +# The first set of topics are applied to anonymous clients, assuming +# allow_anonymous is true. User specific topic ACLs are added after a +# user line as follows: +# +# user +# +# The username referred to here is the same as in password_file. It is +# not the clientid. +# +# +# If is also possible to define ACLs based on pattern substitution within the +# topic. The patterns available for substition are: +# +# %c to match the client id of the client +# %u to match the username of the client +# +# The substitution pattern must be the only text for that level of hierarchy. +# +# The form is the same as for the topic keyword, but using pattern as the +# keyword. +# Pattern ACLs apply to all users even if the "user" keyword has previously +# been given. +# +# If using bridges with usernames and ACLs, connection messages can be allowed +# with the following pattern: +# pattern write $SYS/broker/connection/%c/state +# +# pattern [read|write|readwrite] +# +# Example: +# +# pattern write sensor/%u/data +# +# If an plugin is used as well as acl_file, the plugin check will be +# made first. +#acl_file + +# ----------------------------------------------------------------- +# External authentication and topic access plugin options +# ----------------------------------------------------------------- + +# External authentication and access control can be supported with the +# plugin option. This is a path to a loadable plugin. See also the +# plugin_opt_* options described below. +# +# The plugin option can be specified multiple times to load multiple +# plugins. The plugins will be processed in the order that they are specified +# here. If the plugin option is specified alongside either of +# password_file or acl_file then the plugin checks will be made first. +# +# If the per_listener_settings option is false, the plugin will be apply to all +# listeners. If per_listener_settings is true, then the plugin will apply to +# the current listener being defined only. +# +# This option is also available as `auth_plugin`, but this use is deprecated +# and will be removed in the future. +# +#plugin + +# If the plugin option above is used, define options to pass to the +# plugin here as described by the plugin instructions. All options named +# using the format plugin_opt_* will be passed to the plugin, for example: +# +# This option is also available as `auth_opt_*`, but this use is deprecated +# and will be removed in the future. +# +# plugin_opt_db_host +# plugin_opt_db_port +# plugin_opt_db_username +# plugin_opt_db_password + + +# ================================================================= +# Bridges +# ================================================================= + +# A bridge is a way of connecting multiple MQTT brokers together. +# Create a new bridge using the "connection" option as described below. Set +# options for the bridges using the remaining parameters. You must specify the +# address and at least one topic to subscribe to. +# +# Each connection must have a unique name. +# +# The address line may have multiple host address and ports specified. See +# below in the round_robin description for more details on bridge behaviour if +# multiple addresses are used. Note that if you use an IPv6 address, then you +# are required to specify a port. +# +# The direction that the topic will be shared can be chosen by +# specifying out, in or both, where the default value is out. +# The QoS level of the bridged communication can be specified with the next +# topic option. The default QoS level is 0, to change the QoS the topic +# direction must also be given. +# +# The local and remote prefix options allow a topic to be remapped when it is +# bridged to/from the remote broker. This provides the ability to place a topic +# tree in an appropriate location. +# +# For more details see the mosquitto.conf man page. +# +# Multiple topics can be specified per connection, but be careful +# not to create any loops. +# +# If you are using bridges with cleansession set to false (the default), then +# you may get unexpected behaviour from incoming topics if you change what +# topics you are subscribing to. This is because the remote broker keeps the +# subscription for the old topic. If you have this problem, connect your bridge +# with cleansession set to true, then reconnect with cleansession set to false +# as normal. +#connection +#address [:] [[:]] +#topic [[[out | in | both] qos-level] local-prefix remote-prefix] + +# If you need to have the bridge connect over a particular network interface, +# use bridge_bind_address to tell the bridge which local IP address the socket +# should bind to, e.g. `bridge_bind_address 192.168.1.10` +#bridge_bind_address + +# If a bridge has topics that have "out" direction, the default behaviour is to +# send an unsubscribe request to the remote broker on that topic. This means +# that changing a topic direction from "in" to "out" will not keep receiving +# incoming messages. Sending these unsubscribe requests is not always +# desirable, setting bridge_attempt_unsubscribe to false will disable sending +# the unsubscribe request. +#bridge_attempt_unsubscribe true + +# Set the version of the MQTT protocol to use with for this bridge. Can be one +# of mqttv50, mqttv311 or mqttv31. Defaults to mqttv311. +#bridge_protocol_version mqttv311 + +# Set the clean session variable for this bridge. +# When set to true, when the bridge disconnects for any reason, all +# messages and subscriptions will be cleaned up on the remote +# broker. Note that with cleansession set to true, there may be a +# significant amount of retained messages sent when the bridge +# reconnects after losing its connection. +# When set to false, the subscriptions and messages are kept on the +# remote broker, and delivered when the bridge reconnects. +#cleansession false + +# Set the amount of time a bridge using the lazy start type must be idle before +# it will be stopped. Defaults to 60 seconds. +#idle_timeout 60 + +# Set the keepalive interval for this bridge connection, in +# seconds. +#keepalive_interval 60 + +# Set the clientid to use on the local broker. If not defined, this defaults to +# 'local.'. If you are bridging a broker to itself, it is important +# that local_clientid and clientid do not match. +#local_clientid + +# If set to true, publish notification messages to the local and remote brokers +# giving information about the state of the bridge connection. Retained +# messages are published to the topic $SYS/broker/connection//state +# unless the notification_topic option is used. +# If the message is 1 then the connection is active, or 0 if the connection has +# failed. +# This uses the last will and testament feature. +#notifications true + +# Choose the topic on which notification messages for this bridge are +# published. If not set, messages are published on the topic +# $SYS/broker/connection//state +#notification_topic + +# Set the client id to use on the remote end of this bridge connection. If not +# defined, this defaults to 'name.hostname' where name is the connection name +# and hostname is the hostname of this computer. +# This replaces the old "clientid" option to avoid confusion. "clientid" +# remains valid for the time being. +#remote_clientid + +# Set the password to use when connecting to a broker that requires +# authentication. This option is only used if remote_username is also set. +# This replaces the old "password" option to avoid confusion. "password" +# remains valid for the time being. +#remote_password + +# Set the username to use when connecting to a broker that requires +# authentication. +# This replaces the old "username" option to avoid confusion. "username" +# remains valid for the time being. +#remote_username + +# Set the amount of time a bridge using the automatic start type will wait +# until attempting to reconnect. +# This option can be configured to use a constant delay time in seconds, or to +# use a backoff mechanism based on "Decorrelated Jitter", which adds a degree +# of randomness to when the restart occurs. +# +# Set a constant timeout of 20 seconds: +# restart_timeout 20 +# +# Set backoff with a base (start value) of 10 seconds and a cap (upper limit) of +# 60 seconds: +# restart_timeout 10 30 +# +# Defaults to jitter with a base of 5 and cap of 30 +#restart_timeout 5 30 + +# If the bridge has more than one address given in the address/addresses +# configuration, the round_robin option defines the behaviour of the bridge on +# a failure of the bridge connection. If round_robin is false, the default +# value, then the first address is treated as the main bridge connection. If +# the connection fails, the other secondary addresses will be attempted in +# turn. Whilst connected to a secondary bridge, the bridge will periodically +# attempt to reconnect to the main bridge until successful. +# If round_robin is true, then all addresses are treated as equals. If a +# connection fails, the next address will be tried and if successful will +# remain connected until it fails +#round_robin false + +# Set the start type of the bridge. This controls how the bridge starts and +# can be one of three types: automatic, lazy and once. Note that RSMB provides +# a fourth start type "manual" which isn't currently supported by mosquitto. +# +# "automatic" is the default start type and means that the bridge connection +# will be started automatically when the broker starts and also restarted +# after a short delay (30 seconds) if the connection fails. +# +# Bridges using the "lazy" start type will be started automatically when the +# number of queued messages exceeds the number set with the "threshold" +# parameter. It will be stopped automatically after the time set by the +# "idle_timeout" parameter. Use this start type if you wish the connection to +# only be active when it is needed. +# +# A bridge using the "once" start type will be started automatically when the +# broker starts but will not be restarted if the connection fails. +#start_type automatic + +# Set the number of messages that need to be queued for a bridge with lazy +# start type to be restarted. Defaults to 10 messages. +# Must be less than max_queued_messages. +#threshold 10 + +# If try_private is set to true, the bridge will attempt to indicate to the +# remote broker that it is a bridge not an ordinary client. If successful, this +# means that loop detection will be more effective and that retained messages +# will be propagated correctly. Not all brokers support this feature so it may +# be necessary to set try_private to false if your bridge does not connect +# properly. +#try_private true + +# Some MQTT brokers do not allow retained messages. MQTT v5 gives a mechanism +# for brokers to tell clients that they do not support retained messages, but +# this is not possible for MQTT v3.1.1 or v3.1. If you need to bridge to a +# v3.1.1 or v3.1 broker that does not support retained messages, set the +# bridge_outgoing_retain option to false. This will remove the retain bit on +# all outgoing messages to that bridge, regardless of any other setting. +#bridge_outgoing_retain true + +# If you wish to restrict the size of messages sent to a remote bridge, use the +# bridge_max_packet_size option. This sets the maximum number of bytes for +# the total message, including headers and payload. +# Note that MQTT v5 brokers may provide their own maximum-packet-size property. +# In this case, the smaller of the two limits will be used. +# Set to 0 for "unlimited". +#bridge_max_packet_size 0 + + +# ----------------------------------------------------------------- +# Certificate based SSL/TLS support +# ----------------------------------------------------------------- +# Either bridge_cafile or bridge_capath must be defined to enable TLS support +# for this bridge. +# bridge_cafile defines the path to a file containing the +# Certificate Authority certificates that have signed the remote broker +# certificate. +# bridge_capath defines a directory that will be searched for files containing +# the CA certificates. For bridge_capath to work correctly, the certificate +# files must have ".crt" as the file ending and you must run "openssl rehash +# " each time you add/remove a certificate. +#bridge_cafile +#bridge_capath + + +# If the remote broker has more than one protocol available on its port, e.g. +# MQTT and WebSockets, then use bridge_alpn to configure which protocol is +# requested. Note that WebSockets support for bridges is not yet available. +#bridge_alpn + +# When using certificate based encryption, bridge_insecure disables +# verification of the server hostname in the server certificate. This can be +# useful when testing initial server configurations, but makes it possible for +# a malicious third party to impersonate your server through DNS spoofing, for +# example. Use this option in testing only. If you need to resort to using this +# option in a production environment, your setup is at fault and there is no +# point using encryption. +#bridge_insecure false + +# Path to the PEM encoded client certificate, if required by the remote broker. +#bridge_certfile + +# Path to the PEM encoded client private key, if required by the remote broker. +#bridge_keyfile + +# ----------------------------------------------------------------- +# PSK based SSL/TLS support +# ----------------------------------------------------------------- +# Pre-shared-key encryption provides an alternative to certificate based +# encryption. A bridge can be configured to use PSK with the bridge_identity +# and bridge_psk options. These are the client PSK identity, and pre-shared-key +# in hexadecimal format with no "0x". Only one of certificate and PSK based +# encryption can be used on one +# bridge at once. +#bridge_identity +#bridge_psk + + +# ================================================================= +# External config files +# ================================================================= + +# External configuration files may be included by using the +# include_dir option. This defines a directory that will be searched +# for config files. All files that end in '.conf' will be loaded as +# a configuration file. It is best to have this as the last option +# in the main file. This option will only be processed from the main +# configuration file. The directory specified must not contain the +# main configuration file. +# Files within include_dir will be loaded sorted in case-sensitive +# alphabetical order, with capital letters ordered first. If this option is +# given multiple times, all of the files from the first instance will be +# processed before the next instance. See the man page for examples. +#include_dir \ No newline at end of file From 81cf75906c47cd8a471b6ee1ac998ff083c0dbc9 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 3 Jun 2024 11:48:10 +0200 Subject: [PATCH 02/14] fix most comment --- .../service/NotificationService.kt | 4 +- .../service/mqtt/MQTTVersionService.kt | 50 - .../stellio/subscription/service/mqtt/Mqtt.kt | 2 - ...icationData.kt => MqttNotificationData.kt} | 2 +- ...nService.kt => MqttNotificationService.kt} | 54 +- .../service/MqttVersionServiceTest.kt | 72 -- .../service/NotificationServiceTests.kt | 6 +- .../{ => mqtt}/MqttNotificationServiceTest.kt | 98 +- .../test/resources/mosquitto/mosquitto.conf | 904 +----------------- 9 files changed, 122 insertions(+), 1070 deletions(-) delete mode 100644 subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt rename subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/{MQTTNotificationData.kt => MqttNotificationData.kt} (92%) rename subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/{MQTTNotificationService.kt => MqttNotificationService.kt} (52%) delete mode 100644 subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt rename subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/{ => mqtt}/MqttNotificationServiceTest.kt (52%) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt index d2a632a208..b697667254 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt @@ -13,8 +13,8 @@ import com.egm.stellio.subscription.model.Notification import com.egm.stellio.subscription.model.NotificationParams import com.egm.stellio.subscription.model.NotificationTrigger import com.egm.stellio.subscription.model.Subscription -import com.egm.stellio.subscription.service.mqtt.MQTTNotificationService import com.egm.stellio.subscription.service.mqtt.Mqtt +import com.egm.stellio.subscription.service.mqtt.MqttNotificationService import org.slf4j.LoggerFactory import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus @@ -26,7 +26,7 @@ import org.springframework.web.reactive.function.client.awaitExchange @Service class NotificationService( private val subscriptionService: SubscriptionService, - private val mqttNotificationService: MQTTNotificationService + private val mqttNotificationService: MqttNotificationService ) { private val logger = LoggerFactory.getLogger(javaClass) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt deleted file mode 100644 index 2d05bccd40..0000000000 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTVersionService.kt +++ /dev/null @@ -1,50 +0,0 @@ -package com.egm.stellio.subscription.service.mqtt - -import com.egm.stellio.shared.util.JsonUtils.serializeObject -import org.eclipse.paho.mqttv5.client.MqttConnectionOptions -import org.springframework.stereotype.Service -import org.eclipse.paho.client.mqttv3.MqttClient as MqttClientv3 -import org.eclipse.paho.client.mqttv3.MqttConnectOptions as MqttConnectOptionsv3 -import org.eclipse.paho.client.mqttv3.MqttMessage as MqttMessagev3 -import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence as MemoryPersistencev3 -import org.eclipse.paho.mqttv5.client.IMqttToken as IMqttTokenv5 -import org.eclipse.paho.mqttv5.client.MqttAsyncClient as MqttAsyncClientv5 -import org.eclipse.paho.mqttv5.client.persist.MemoryPersistence as MemoryPersistencev5 -import org.eclipse.paho.mqttv5.common.MqttMessage as MqttMessagev5 - -@Service -class MQTTVersionService { - - internal suspend fun callMqttV3(data: MQTTNotificationData) { - val persistence = MemoryPersistencev3() - val sampleClient = MqttClientv3(data.brokerUrl, data.clientId, persistence) - val connOpts = MqttConnectOptionsv3() - connOpts.isCleanSession = true - connOpts.userName = data.username - connOpts.password = data.password?.toCharArray() ?: "".toCharArray() - sampleClient.connect(connOpts) - val message = MqttMessagev3( - serializeObject(data.mqttMessage).toByteArray() - ) - message.qos = data.qos - sampleClient.publish(data.topic, message) - sampleClient.disconnect() - } - - internal suspend fun callMqttV5(data: MQTTNotificationData) { - val persistence = MemoryPersistencev5() - val sampleClient = MqttAsyncClientv5(data.brokerUrl, data.clientId, persistence) - val connOpts = MqttConnectionOptions() - connOpts.isCleanStart = true - connOpts.userName = data.username - connOpts.password = data.password?.toByteArray() - var token: IMqttTokenv5 = sampleClient.connect(connOpts) - token.waitForCompletion() - val message = MqttMessagev5(serializeObject(data.mqttMessage).toByteArray()) - message.qos = data.qos - token = sampleClient.publish(data.topic, message) - token.waitForCompletion() - sampleClient.disconnect() - sampleClient.close() - } -} diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt index 610385daea..a258e8a95f 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt @@ -11,8 +11,6 @@ object Mqtt { object QualityOfService { const val KEY = "MQTT-QoS" const val AT_MOST_ONCE = 0 -// const val AT_LEAST_ONCE = 1 -// const val EXACTLY_ONCE = 2 } object SCHEME { diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt similarity index 92% rename from subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt rename to subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt index 9c68832e5d..3d6ed21672 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationData.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt @@ -2,7 +2,7 @@ package com.egm.stellio.subscription.service.mqtt import com.egm.stellio.subscription.model.Notification -data class MQTTNotificationData( +data class MqttNotificationData( val topic: String, val mqttMessage: MqttMessage, val qos: Int, diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt similarity index 52% rename from subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt rename to subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt index fd440fb900..5290f9a1fb 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MQTTNotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt @@ -1,8 +1,16 @@ package com.egm.stellio.subscription.service.mqtt import com.egm.stellio.shared.model.BadSchemeException +import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.subscription.model.Notification import com.egm.stellio.subscription.model.Subscription +import org.eclipse.paho.client.mqttv3.MqttClient +import org.eclipse.paho.client.mqttv3.MqttConnectOptions +import org.eclipse.paho.client.mqttv3.MqttMessage +import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence +import org.eclipse.paho.mqttv5.client.IMqttToken +import org.eclipse.paho.mqttv5.client.MqttAsyncClient +import org.eclipse.paho.mqttv5.client.MqttConnectionOptions import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service @@ -10,10 +18,9 @@ import org.eclipse.paho.client.mqttv3.MqttException as MqttExceptionV3 import org.eclipse.paho.mqttv5.common.MqttException as MqttExceptionV5 @Service -class MQTTNotificationService( +class MqttNotificationService( @Value("\${mqtt.clientId}") private val clientId: String = "stellio-context-brokerUrl", - private val mqttVersionService: MQTTVersionService ) { private val logger = LoggerFactory.getLogger(javaClass) @@ -38,12 +45,12 @@ class MQTTNotificationService( val qos = notifierInfo[Mqtt.QualityOfService.KEY]?.let { Integer.parseInt(it) } ?: Mqtt.QualityOfService.AT_MOST_ONCE - val data = MQTTNotificationData( + val data = MqttNotificationData( topic = uri.path, brokerUrl = brokerUrl, clientId = clientId, qos = qos, - mqttMessage = MQTTNotificationData.MqttMessage(notification, headers), + mqttMessage = MqttNotificationData.MqttMessage(notification, headers), username = username, password = password ) @@ -51,9 +58,9 @@ class MQTTNotificationService( try { val mqttVersion = notifierInfo[Mqtt.Version.KEY] when (mqttVersion) { - Mqtt.Version.V3 -> mqttVersionService.callMqttV3(data) - Mqtt.Version.V5 -> mqttVersionService.callMqttV5(data) - else -> mqttVersionService.callMqttV5(data) + Mqtt.Version.V3 -> callMqttV3(data) + Mqtt.Version.V5 -> callMqttV5(data) + else -> callMqttV5(data) } logger.info("successfull mqtt notification for uri : ${data.brokerUrl} version: $mqttVersion") return true @@ -65,4 +72,37 @@ class MQTTNotificationService( return false } } + + internal suspend fun callMqttV3(data: MqttNotificationData) { + val persistence = MemoryPersistence() + val mqttClient = MqttClient(data.brokerUrl, data.clientId, persistence) + val connOpts = MqttConnectOptions() + connOpts.isCleanSession = true + connOpts.userName = data.username + connOpts.password = data.password?.toCharArray() ?: "".toCharArray() + mqttClient.connect(connOpts) + val message = MqttMessage( + serializeObject(data.mqttMessage).toByteArray() + ) + message.qos = data.qos + mqttClient.publish(data.topic, message) + mqttClient.disconnect() + } + + internal suspend fun callMqttV5(data: MqttNotificationData) { + val persistence = org.eclipse.paho.mqttv5.client.persist.MemoryPersistence() + val mqttClient = MqttAsyncClient(data.brokerUrl, data.clientId, persistence) + val connOpts = MqttConnectionOptions() + connOpts.isCleanStart = true + connOpts.userName = data.username + connOpts.password = data.password?.toByteArray() + var token: IMqttToken = mqttClient.connect(connOpts) + token.waitForCompletion() + val message = org.eclipse.paho.mqttv5.common.MqttMessage(serializeObject(data.mqttMessage).toByteArray()) + message.qos = data.qos + token = mqttClient.publish(data.topic, message) + token.waitForCompletion() + mqttClient.disconnect() + mqttClient.close() + } } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt deleted file mode 100644 index d3bbea23ce..0000000000 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttVersionServiceTest.kt +++ /dev/null @@ -1,72 +0,0 @@ -package com.egm.stellio.subscription.service - -import com.egm.stellio.subscription.model.Notification -import com.egm.stellio.subscription.service.mqtt.MQTTNotificationData -import com.egm.stellio.subscription.service.mqtt.MQTTVersionService -import com.egm.stellio.subscription.service.mqtt.Mqtt -import com.egm.stellio.subscription.support.WithMosquittoContainer -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertDoesNotThrow -import org.junit.jupiter.api.assertThrows -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.test.context.SpringBootTest -import org.springframework.test.context.ActiveProfiles -import java.net.URI -import org.eclipse.paho.client.mqttv3.MqttException as MqttExceptionV3 -import org.eclipse.paho.mqttv5.common.MqttException as MqttExceptionV5 - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [MQTTVersionService::class]) -@ActiveProfiles("test") -class MqttVersionServiceTest : WithMosquittoContainer { - - @Autowired - private lateinit var mqttVersionService: MQTTVersionService - - private val mqttContainerPort = WithMosquittoContainer.mosquittoContainer.getMappedPort( - Mqtt.SCHEME.MQTT_DEFAULT_PORT - ) - - private val notification = Notification( - subscriptionId = URI("1"), - data = listOf(mapOf("hello" to "world")) - ) - - private val validMqttNotificationData = MQTTNotificationData( - brokerUrl = "tcp://localhost:$mqttContainerPort", - clientId = "clientId", - username = "test", - topic = "notification", - qos = 0, - mqttMessage = MQTTNotificationData.MqttMessage(notification, emptyMap()) - ) - - private val invalidUriMqttNotificationData = MQTTNotificationData( - brokerUrl = "tcp://badHost:1883", - clientId = "clientId", - username = "test", - topic = "notification", - qos = 0, - mqttMessage = MQTTNotificationData.MqttMessage(notification, emptyMap()) - ) - - @Test - fun `sending mqttV3 notification with good uri should succeed`() = runTest { - assertDoesNotThrow { mqttVersionService.callMqttV3(validMqttNotificationData) } - } - - @Test - fun `sending mqttV3 notification with bad uri should throw`() = runTest { - assertThrows { mqttVersionService.callMqttV3(invalidUriMqttNotificationData) } - } - - @Test - fun `sending mqttV5 notification with good uri should succeed`() = runTest { - assertDoesNotThrow { mqttVersionService.callMqttV5(validMqttNotificationData) } - } - - @Test - fun `sending mqttV5 notification with bad uri should throw`() = runTest { - assertThrows { mqttVersionService.callMqttV5(invalidUriMqttNotificationData) } - } -} diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt index c98228fa5c..2b45990969 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt @@ -17,7 +17,7 @@ import com.egm.stellio.subscription.model.EndpointInfo import com.egm.stellio.subscription.model.NotificationParams import com.egm.stellio.subscription.model.NotificationParams.FormatType import com.egm.stellio.subscription.model.NotificationTrigger.* -import com.egm.stellio.subscription.service.mqtt.MQTTNotificationService +import com.egm.stellio.subscription.service.mqtt.MqttNotificationService import com.egm.stellio.subscription.support.gimmeRawSubscription import com.github.tomakehurst.wiremock.client.WireMock.* import com.github.tomakehurst.wiremock.junit5.WireMockTest @@ -45,7 +45,7 @@ class NotificationServiceTests { private lateinit var subscriptionService: SubscriptionService @MockkBean - private lateinit var mqttNotificationService: MQTTNotificationService + private lateinit var mqttNotificationService: MqttNotificationService @Autowired private lateinit var notificationService: NotificationService @@ -565,7 +565,7 @@ class NotificationServiceTests { } @Test - fun `callSuscriber should ask mqttNotifier if the brokerUrl startWith mqtt`() = runTest { + fun `callSuscriber should ask mqttNotifier if the endpoint uri startWith mqtt`() = runTest { val subscription = gimmeRawSubscription().copy( notification = NotificationParams( attributes = emptyList(), diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt similarity index 52% rename from subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt rename to subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt index 2968e93216..93640adfea 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/MqttNotificationServiceTest.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt @@ -1,32 +1,32 @@ -package com.egm.stellio.subscription.service +package com.egm.stellio.subscription.service.mqtt import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SUBSCRIPTION_TERM import com.egm.stellio.shared.util.toUri import com.egm.stellio.subscription.model.* -import com.egm.stellio.subscription.service.mqtt.MQTTNotificationData -import com.egm.stellio.subscription.service.mqtt.MQTTNotificationService -import com.egm.stellio.subscription.service.mqtt.MQTTVersionService -import com.egm.stellio.subscription.service.mqtt.Mqtt -import com.ninjasquad.springmockk.MockkBean +import com.egm.stellio.subscription.support.WithMosquittoContainer +import com.ninjasquad.springmockk.SpykBean import io.mockk.coEvery import io.mockk.coVerify import kotlinx.coroutines.test.runTest +import org.eclipse.paho.client.mqttv3.MqttException import org.junit.jupiter.api.Test -import org.springframework.beans.factory.annotation.Autowired +import org.junit.jupiter.api.assertDoesNotThrow +import org.junit.jupiter.api.assertThrows import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles +import java.net.URI -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [MQTTNotificationService::class]) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [MqttNotificationService::class]) @ActiveProfiles("test") -class MqttNotificationServiceTest { +class MqttNotificationServiceTest : WithMosquittoContainer { - @Autowired - private lateinit var mqttNotificationService: MQTTNotificationService + @SpykBean + private lateinit var mqttNotificationService: MqttNotificationService - @MockkBean - private lateinit var mqttVersionService: MQTTVersionService - - private val mqttSubscription = Subscription( + private val mqttContainerPort = WithMosquittoContainer.mosquittoContainer.getMappedPort( + Mqtt.SCHEME.MQTT_DEFAULT_PORT + ) + private val mqttSubscriptionV3 = Subscription( type = NGSILD_SUBSCRIPTION_TERM, subscriptionName = "My Subscription", description = "My beautiful subscription", @@ -34,7 +34,7 @@ class MqttNotificationServiceTest { notification = NotificationParams( attributes = emptyList(), endpoint = Endpoint( - uri = "mqtt://test@localhost:1883/notification".toUri(), + uri = "mqtt://test@localhost:$mqttContainerPort/notification".toUri(), accept = Endpoint.AcceptType.JSONLD, notifierInfo = listOf( EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V3), @@ -51,7 +51,7 @@ class MqttNotificationServiceTest { notification = NotificationParams( attributes = emptyList(), endpoint = Endpoint( - uri = "mqtt://test@localhost:1883/notification".toUri(), + uri = "mqtt://test@localhost:$mqttContainerPort/notification".toUri(), notifierInfo = listOf( EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V5) ) @@ -59,13 +59,27 @@ class MqttNotificationServiceTest { ), contexts = emptyList() ) - private val validMqttNotificationData = MQTTNotificationData( - brokerUrl = "tcp://localhost:1883", + private val validMqttNotificationData = MqttNotificationData( + brokerUrl = "tcp://localhost:$mqttContainerPort", clientId = "clientId", username = "test", topic = "/notification", qos = 0, - mqttMessage = MQTTNotificationData.MqttMessage(getNotificationForSubscription(mqttSubscription), emptyMap()) + mqttMessage = MqttNotificationData.MqttMessage(getNotificationForSubscription(mqttSubscriptionV3), emptyMap()) + ) + + private val notification = Notification( + subscriptionId = URI("1"), + data = listOf(mapOf("hello" to "world")) + ) + + private val invalidUriMqttNotificationData = MqttNotificationData( + brokerUrl = "tcp://badHost:1883", + clientId = "clientId", + username = "test", + topic = "notification", + qos = 0, + mqttMessage = MqttNotificationData.MqttMessage(notification, emptyMap()) ) private fun getNotificationForSubscription(subscription: Subscription) = Notification( @@ -75,8 +89,8 @@ class MqttNotificationServiceTest { @Test fun `mqttNotifier should process endpoint uri to get connexion information`() = runTest { - val subscription = mqttSubscription - coEvery { mqttVersionService.callMqttV3(any()) } returns Unit + val subscription = mqttSubscriptionV3 + coEvery { mqttNotificationService.callMqttV3(any()) } returns Unit assert( mqttNotificationService.mqttNotifier( subscription, @@ -86,7 +100,7 @@ class MqttNotificationServiceTest { ) coVerify { - mqttVersionService.callMqttV3( + mqttNotificationService.callMqttV3( match { it.username == validMqttNotificationData.username && it.password == validMqttNotificationData.password && @@ -99,9 +113,9 @@ class MqttNotificationServiceTest { @Test fun `mqttNotifier should use notifier info to choose the mqtt version`() = runTest { - val subscription = mqttSubscription - coEvery { mqttVersionService.callMqttV3(any()) } returns Unit - coEvery { mqttVersionService.callMqttV5(any()) } returns Unit + val subscription = mqttSubscriptionV3 + coEvery { mqttNotificationService.callMqttV3(any()) } returns Unit + coEvery { mqttNotificationService.callMqttV5(any()) } returns Unit mqttNotificationService.mqttNotifier( subscription, @@ -109,12 +123,12 @@ class MqttNotificationServiceTest { mapOf() ) coVerify(exactly = 1) { - mqttVersionService.callMqttV3( + mqttNotificationService.callMqttV3( any() ) } coVerify(exactly = 0) { - mqttVersionService.callMqttV5( + mqttNotificationService.callMqttV5( any() ) } @@ -125,14 +139,38 @@ class MqttNotificationServiceTest { mapOf() ) coVerify(exactly = 1) { - mqttVersionService.callMqttV5( + mqttNotificationService.callMqttV5( any() ) } coVerify(exactly = 1) { - mqttVersionService.callMqttV3( + mqttNotificationService.callMqttV3( any() ) } } + + @Test + fun `sending mqttV3 notification with good uri should succeed`() = runTest { + assertDoesNotThrow { mqttNotificationService.callMqttV3(validMqttNotificationData) } + } + + @Test + fun `sending mqttV3 notification with bad uri should throw an exception`() = runTest { + assertThrows { mqttNotificationService.callMqttV3(invalidUriMqttNotificationData) } + } + + @Test + fun `sending mqttV5 notification with good uri should succeed`() = runTest { + assertDoesNotThrow { mqttNotificationService.callMqttV5(validMqttNotificationData) } + } + + @Test + fun `sending mqttV5 notification with bad uri should throw an exception`() = runTest { + assertThrows { + mqttNotificationService.callMqttV5( + invalidUriMqttNotificationData + ) + } + } } diff --git a/subscription-service/src/test/resources/mosquitto/mosquitto.conf b/subscription-service/src/test/resources/mosquitto/mosquitto.conf index 5b0f2e9f29..65adc872cb 100644 --- a/subscription-service/src/test/resources/mosquitto/mosquitto.conf +++ b/subscription-service/src/test/resources/mosquitto/mosquitto.conf @@ -1,906 +1,4 @@ -# Config file for mosquitto -# -# See mosquitto.conf(5) for more information. -# -# Default values are shown, uncomment to change. -# -# Use the # character to indicate a comment, but only if it is the -# very first character on the line. - -# ================================================================= -# General configuration -# ================================================================= - -# Use per listener security settings. -# -# It is recommended this option be set before any other options. -# -# If this option is set to true, then all authentication and access control -# options are controlled on a per listener basis. The following options are -# affected: -# -# acl_file +# see possible config : https://mosquitto.org/man/mosquitto-conf-5.html allow_anonymous true listener 1883 listener 9001 -# allow_zero_length_clientid -# auto_id_prefix -# password_file -# plugin -# plugin_opt_* -# psk_file -# -# Note that if set to true, then a durable client (i.e. with clean session set -# to false) that has disconnected will use the ACL settings defined for the -# listener that it was most recently connected to. -# -# The default behaviour is for this to be set to false, which maintains the -# setting behaviour from previous versions of mosquitto. -#per_listener_settings false - - -# This option controls whether a client is allowed to connect with a zero -# length client id or not. This option only affects clients using MQTT v3.1.1 -# and later. If set to false, clients connecting with a zero length client id -# are disconnected. If set to true, clients will be allocated a client id by -# the broker. This means it is only useful for clients with clean session set -# to true. -#allow_zero_length_clientid true - -# If allow_zero_length_clientid is true, this option allows you to set a prefix -# to automatically generated client ids to aid visibility in logs. -# Defaults to 'auto-' -#auto_id_prefix auto- - -# This option affects the scenario when a client subscribes to a topic that has -# retained messages. It is possible that the client that published the retained -# message to the topic had access at the time they published, but that access -# has been subsequently removed. If check_retain_source is set to true, the -# default, the source of a retained message will be checked for access rights -# before it is republished. When set to false, no check will be made and the -# retained message will always be published. This affects all listeners. -#check_retain_source true - -# QoS 1 and 2 messages will be allowed inflight per client until this limit -# is exceeded. Defaults to 0. (No maximum) -# See also max_inflight_messages -#max_inflight_bytes 0 - -# The maximum number of QoS 1 and 2 messages currently inflight per -# client. -# This includes messages that are partway through handshakes and -# those that are being retried. Defaults to 20. Set to 0 for no -# maximum. Setting to 1 will guarantee in-order delivery of QoS 1 -# and 2 messages. -#max_inflight_messages 20 - -# For MQTT v5 clients, it is possible to have the server send a "server -# keepalive" value that will override the keepalive value set by the client. -# This is intended to be used as a mechanism to say that the server will -# disconnect the client earlier than it anticipated, and that the client should -# use the new keepalive value. The max_keepalive option allows you to specify -# that clients may only connect with keepalive less than or equal to this -# value, otherwise they will be sent a server keepalive telling them to use -# max_keepalive. This only applies to MQTT v5 clients. The default, and maximum -# value allowable, is 65535. -# -# Set to 0 to allow clients to set keepalive = 0, which means no keepalive -# checks are made and the client will never be disconnected by the broker if no -# messages are received. You should be very sure this is the behaviour that you -# want. -# -# For MQTT v3.1.1 and v3.1 clients, there is no mechanism to tell the client -# what keepalive value they should use. If an MQTT v3.1.1 or v3.1 client -# specifies a keepalive time greater than max_keepalive they will be sent a -# CONNACK message with the "identifier rejected" reason code, and disconnected. -# -#max_keepalive 65535 - -# For MQTT v5 clients, it is possible to have the server send a "maximum packet -# size" value that will instruct the client it will not accept MQTT packets -# with size greater than max_packet_size bytes. This applies to the full MQTT -# packet, not just the payload. Setting this option to a positive value will -# set the maximum packet size to that number of bytes. If a client sends a -# packet which is larger than this value, it will be disconnected. This applies -# to all clients regardless of the protocol version they are using, but v3.1.1 -# and earlier clients will of course not have received the maximum packet size -# information. Defaults to no limit. Setting below 20 bytes is forbidden -# because it is likely to interfere with ordinary client operation, even with -# very small payloads. -#max_packet_size 0 - -# QoS 1 and 2 messages above those currently in-flight will be queued per -# client until this limit is exceeded. Defaults to 0. (No maximum) -# See also max_queued_messages. -# If both max_queued_messages and max_queued_bytes are specified, packets will -# be queued until the first limit is reached. -#max_queued_bytes 0 - -# Set the maximum QoS supported. Clients publishing at a QoS higher than -# specified here will be disconnected. -#max_qos 2 - -# The maximum number of QoS 1 and 2 messages to hold in a queue per client -# above those that are currently in-flight. Defaults to 1000. Set -# to 0 for no maximum (not recommended). -# See also queue_qos0_messages. -# See also max_queued_bytes. -#max_queued_messages 1000 -# -# This option sets the maximum number of heap memory bytes that the broker will -# allocate, and hence sets a hard limit on memory use by the broker. Memory -# requests that exceed this value will be denied. The effect will vary -# depending on what has been denied. If an incoming message is being processed, -# then the message will be dropped and the publishing client will be -# disconnected. If an outgoing message is being sent, then the individual -# message will be dropped and the receiving client will be disconnected. -# Defaults to no limit. -#memory_limit 0 - -# This option sets the maximum publish payload size that the broker will allow. -# Received messages that exceed this size will not be accepted by the broker. -# The default value is 0, which means that all valid MQTT messages are -# accepted. MQTT imposes a maximum payload size of 268435455 bytes. -#message_size_limit 0 - -# This option allows the session of persistent clients (those with clean -# session set to false) that are not currently connected to be removed if they -# do not reconnect within a certain time frame. This is a non-standard option -# in MQTT v3.1. MQTT v3.1.1 and v5.0 allow brokers to remove client sessions. -# -# Badly designed clients may set clean session to false whilst using a randomly -# generated client id. This leads to persistent clients that connect once and -# never reconnect. This option allows these clients to be removed. This option -# allows persistent clients (those with clean session set to false) to be -# removed if they do not reconnect within a certain time frame. -# -# The expiration period should be an integer followed by one of h d w m y for -# hour, day, week, month and year respectively. For example -# -# persistent_client_expiration 2m -# persistent_client_expiration 14d -# persistent_client_expiration 1y -# -# The default if not set is to never expire persistent clients. -#persistent_client_expiration - -# Write process id to a file. Default is a blank string which means -# a pid file shouldn't be written. -# This should be set to /var/run/mosquitto/mosquitto.pid if mosquitto is -# being run automatically on boot with an init script and -# start-stop-daemon or similar. -#pid_file - -# Set to true to queue messages with QoS 0 when a persistent client is -# disconnected. These messages are included in the limit imposed by -# max_queued_messages and max_queued_bytes -# Defaults to false. -# This is a non-standard option for the MQTT v3.1 spec but is allowed in -# v3.1.1. -#queue_qos0_messages false - -# Set to false to disable retained message support. If a client publishes a -# message with the retain bit set, it will be disconnected if this is set to -# false. -#retain_available true - -# Disable Nagle's algorithm on client sockets. This has the effect of reducing -# latency of individual messages at the potential cost of increasing the number -# of packets being sent. -#set_tcp_nodelay false - -# Time in seconds between updates of the $SYS tree. -# Set to 0 to disable the publishing of the $SYS tree. -#sys_interval 10 - -# The MQTT specification requires that the QoS of a message delivered to a -# subscriber is never upgraded to match the QoS of the subscription. Enabling -# this option changes this behaviour. If upgrade_outgoing_qos is set true, -# messages sent to a subscriber will always match the QoS of its subscription. -# This is a non-standard option explicitly disallowed by the spec. -#upgrade_outgoing_qos false - -# When run as root, drop privileges to this user and its primary -# group. -# Set to root to stay as root, but this is not recommended. -# If set to "mosquitto", or left unset, and the "mosquitto" user does not exist -# then it will drop privileges to the "nobody" user instead. -# If run as a non-root user, this setting has no effect. -# Note that on Windows this has no effect and so mosquitto should be started by -# the user you wish it to run as. -#user mosquitto - -# ================================================================= -# Listeners -# ================================================================= - -# Listen on a port/ip address combination. By using this variable -# multiple times, mosquitto can listen on more than one port. If -# this variable is used and neither bind_address nor port given, -# then the default listener will not be started. -# The port number to listen on must be given. Optionally, an ip -# address or host name may be supplied as a second argument. In -# this case, mosquitto will attempt to bind the listener to that -# address and so restrict access to the associated network and -# interface. By default, mosquitto will listen on all interfaces. -# Note that for a websockets listener it is not possible to bind to a host -# name. -# -# On systems that support Unix Domain Sockets, it is also possible -# to create a # Unix socket rather than opening a TCP socket. In -# this case, the port number should be set to 0 and a unix socket -# path must be provided, e.g. -# listener 0 /tmp/mosquitto.sock -# -# listener port-number [ip address/host name/unix socket path] -#listener - -# By default, a listener will attempt to listen on all supported IP protocol -# versions. If you do not have an IPv4 or IPv6 interface you may wish to -# disable support for either of those protocol versions. In particular, note -# that due to the limitations of the websockets library, it will only ever -# attempt to open IPv6 sockets if IPv6 support is compiled in, and so will fail -# if IPv6 is not available. -# -# Set to `ipv4` to force the listener to only use IPv4, or set to `ipv6` to -# force the listener to only use IPv6. If you want support for both IPv4 and -# IPv6, then do not use the socket_domain option. -# -#socket_domain - -# Bind the listener to a specific interface. This is similar to -# the [ip address/host name] part of the listener definition, but is useful -# when an interface has multiple addresses or the address may change. If used -# with the [ip address/host name] part of the listener definition, then the -# bind_interface option will take priority. -# Not available on Windows. -# -# Example: bind_interface eth0 -#bind_interface - -# When a listener is using the websockets protocol, it is possible to serve -# http data as well. Set http_dir to a directory which contains the files you -# wish to serve. If this option is not specified, then no normal http -# connections will be possible. -#http_dir - -# The maximum number of client connections to allow. This is -# a per listener setting. -# Default is -1, which means unlimited connections. -# Note that other process limits mean that unlimited connections -# are not really possible. Typically the default maximum number of -# connections possible is around 1024. -#max_connections -1 - -# The listener can be restricted to operating within a topic hierarchy using -# the mount_point option. This is achieved be prefixing the mount_point string -# to all topics for any clients connected to this listener. This prefixing only -# happens internally to the broker; the client will not see the prefix. -#mount_point - -# Choose the protocol to use when listening. -# This can be either mqtt or websockets. -# Certificate based TLS may be used with websockets, except that only the -# cafile, certfile, keyfile, ciphers, and ciphers_tls13 options are supported. -#protocol mqtt - -# Set use_username_as_clientid to true to replace the clientid that a client -# connected with with its username. This allows authentication to be tied to -# the clientid, which means that it is possible to prevent one client -# disconnecting another by using the same clientid. -# If a client connects with no username it will be disconnected as not -# authorised when this option is set to true. -# Do not use in conjunction with clientid_prefixes. -# See also use_identity_as_username. -# This does not apply globally, but on a per-listener basis. -#use_username_as_clientid - -# Change the websockets headers size. This is a global option, it is not -# possible to set per listener. This option sets the size of the buffer used in -# the libwebsockets library when reading HTTP headers. If you are passing large -# header data such as cookies then you may need to increase this value. If left -# unset, or set to 0, then the default of 1024 bytes will be used. -#websockets_headers_size - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable certificate based SSL/TLS support -# for this listener. Note that the recommended port for MQTT over TLS is 8883, -# but this must be set manually. -# -# See also the mosquitto-tls man page and the "Pre-shared-key based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# Both of certfile and keyfile must be defined to enable certificate based -# TLS encryption. - -# Path to the PEM encoded server certificate. -#certfile - -# Path to the PEM encoded keyfile. -#keyfile - -# If you wish to control which encryption ciphers are used, use the ciphers -# option. The list of available ciphers can be optained using the "openssl -# ciphers" command and should be provided in the same format as the output of -# that command. This applies to TLS 1.2 and earlier versions only. Use -# ciphers_tls1.3 for TLS v1.3. -#ciphers - -# Choose which TLS v1.3 ciphersuites are used for this listener. -# Defaults to "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" -#ciphers_tls1.3 - -# If you have require_certificate set to true, you can create a certificate -# revocation list file to revoke access to particular client certificates. If -# you have done this, use crlfile to point to the PEM encoded revocation file. -#crlfile - -# To allow the use of ephemeral DH key exchange, which provides forward -# security, the listener must load DH parameters. This can be specified with -# the dhparamfile option. The dhparamfile can be generated with the command -# e.g. "openssl dhparam -out dhparam.pem 2048" -#dhparamfile - -# By default an TLS enabled listener will operate in a similar fashion to a -# https enabled web server, in that the server has a certificate signed by a CA -# and the client will verify that it is a trusted certificate. The overall aim -# is encryption of the network traffic. By setting require_certificate to true, -# the client must provide a valid certificate in order for the network -# connection to proceed. This allows access to the broker to be controlled -# outside of the mechanisms provided by MQTT. -#require_certificate false - -# cafile and capath define methods of accessing the PEM encoded -# Certificate Authority certificates that will be considered trusted when -# checking incoming client certificates. -# cafile defines the path to a file containing the CA certificates. -# capath defines a directory that will be searched for files -# containing the CA certificates. For capath to work correctly, the -# certificate files must have ".crt" as the file ending and you must run -# "openssl rehash " each time you add/remove a certificate. -#cafile -#capath - - -# If require_certificate is true, you may set use_identity_as_username to true -# to use the CN value from the client certificate as a username. If this is -# true, the password_file option will not be used for this listener. -#use_identity_as_username false - -# ----------------------------------------------------------------- -# Pre-shared-key based SSL/TLS support -# ----------------------------------------------------------------- -# The following options can be used to enable PSK based SSL/TLS support for -# this listener. Note that the recommended port for MQTT over TLS is 8883, but -# this must be set manually. -# -# See also the mosquitto-tls man page and the "Certificate based SSL/TLS -# support" section. Only one of certificate or PSK encryption support can be -# enabled for any listener. - -# The psk_hint option enables pre-shared-key support for this listener and also -# acts as an identifier for this listener. The hint is sent to clients and may -# be used locally to aid authentication. The hint is a free form string that -# doesn't have much meaning in itself, so feel free to be creative. -# If this option is provided, see psk_file to define the pre-shared keys to be -# used or create a security plugin to handle them. -#psk_hint - -# When using PSK, the encryption ciphers used will be chosen from the list of -# available PSK ciphers. If you want to control which ciphers are available, -# use the "ciphers" option. The list of available ciphers can be optained -# using the "openssl ciphers" command and should be provided in the same format -# as the output of that command. -#ciphers - -# Set use_identity_as_username to have the psk identity sent by the client used -# as its username. Authentication will be carried out using the PSK rather than -# the MQTT username/password and so password_file will not be used for this -# listener. -#use_identity_as_username false - - -# ================================================================= -# Persistence -# ================================================================= - -# If persistence is enabled, save the in-memory database to disk -# every autosave_interval seconds. If set to 0, the persistence -# database will only be written when mosquitto exits. See also -# autosave_on_changes. -# Note that writing of the persistence database can be forced by -# sending mosquitto a SIGUSR1 signal. -#autosave_interval 1800 - -# If true, mosquitto will count the number of subscription changes, retained -# messages received and queued messages and if the total exceeds -# autosave_interval then the in-memory database will be saved to disk. -# If false, mosquitto will save the in-memory database to disk by treating -# autosave_interval as a time in seconds. -#autosave_on_changes false - -# Save persistent message data to disk (true/false). -# This saves information about all messages, including -# subscriptions, currently in-flight messages and retained -# messages. -# retained_persistence is a synonym for this option. -#persistence false - -# The filename to use for the persistent database, not including -# the path. -#persistence_file mosquitto.db - -# Location for persistent database. -# Default is an empty string (current directory). -# Set to e.g. /var/lib/mosquitto if running as a proper service on Linux or -# similar. -#persistence_location - - -# ================================================================= -# Logging -# ================================================================= - -# Places to log to. Use multiple log_dest lines for multiple -# logging destinations. -# Possible destinations are: stdout stderr syslog topic file dlt -# -# stdout and stderr log to the console on the named output. -# -# syslog uses the userspace syslog facility which usually ends up -# in /var/log/messages or similar. -# -# topic logs to the broker topic '$SYS/broker/log/', -# where severity is one of D, E, W, N, I, M which are debug, error, -# warning, notice, information and message. Message type severity is used by -# the subscribe/unsubscribe log_types and publishes log messages to -# $SYS/broker/log/M/susbcribe or $SYS/broker/log/M/unsubscribe. -# -# The file destination requires an additional parameter which is the file to be -# logged to, e.g. "log_dest file /var/log/mosquitto.log". The file will be -# closed and reopened when the broker receives a HUP signal. Only a single file -# destination may be configured. -# -# The dlt destination is for the automotive `Diagnostic Log and Trace` tool. -# This requires that Mosquitto has been compiled with DLT support. -# -# Note that if the broker is running as a Windows service it will default to -# "log_dest none" and neither stdout nor stderr logging is available. -# Use "log_dest none" if you wish to disable logging. -#log_dest stderr - -# Types of messages to log. Use multiple log_type lines for logging -# multiple types of messages. -# Possible types are: debug, error, warning, notice, information, -# none, subscribe, unsubscribe, websockets, all. -# Note that debug type messages are for decoding the incoming/outgoing -# network packets. They are not logged in "topics". -#log_type error -#log_type warning -#log_type notice -#log_type information - - -# If set to true, client connection and disconnection messages will be included -# in the log. -#connection_messages true - -# If using syslog logging (not on Windows), messages will be logged to the -# "daemon" facility by default. Use the log_facility option to choose which of -# local0 to local7 to log to instead. The option value should be an integer -# value, e.g. "log_facility 5" to use local5. -#log_facility - -# If set to true, add a timestamp value to each log message. -#log_timestamp true - -# Set the format of the log timestamp. If left unset, this is the number of -# seconds since the Unix epoch. -# This is a free text string which will be passed to the strftime function. To -# get an ISO 8601 datetime, for example: -# log_timestamp_format %Y-%m-%dT%H:%M:%S -#log_timestamp_format - -# Change the websockets logging level. This is a global option, it is not -# possible to set per listener. This is an integer that is interpreted by -# libwebsockets as a bit mask for its lws_log_levels enum. See the -# libwebsockets documentation for more details. "log_type websockets" must also -# be enabled. -#websockets_log_level 0 - - -# ================================================================= -# Security -# ================================================================= - -# If set, only clients that have a matching prefix on their -# clientid will be allowed to connect to the broker. By default, -# all clients may connect. -# For example, setting "secure-" here would mean a client "secure- -# client" could connect but another with clientid "mqtt" couldn't. -#clientid_prefixes - -# Boolean value that determines whether clients that connect -# without providing a username are allowed to connect. If set to -# false then a password file should be created (see the -# password_file option) to control authenticated client access. -# -# Defaults to false, unless there are no listeners defined in the configuration -# file, in which case it is set to true, but connections are only allowed from -# the local machine. -#allow_anonymous false - -# ----------------------------------------------------------------- -# Default authentication and topic access control -# ----------------------------------------------------------------- - -# Control access to the broker using a password file. This file can be -# generated using the mosquitto_passwd utility. If TLS support is not compiled -# into mosquitto (it is recommended that TLS support should be included) then -# plain text passwords are used, in which case the file should be a text file -# with lines in the format: -# username:password -# The password (and colon) may be omitted if desired, although this -# offers very little in the way of security. -# -# See the TLS client require_certificate and use_identity_as_username options -# for alternative authentication options. If a plugin is used as well as -# password_file, the plugin check will be made first. -#password_file - -# Access may also be controlled using a pre-shared-key file. This requires -# TLS-PSK support and a listener configured to use it. The file should be text -# lines in the format: -# identity:key -# The key should be in hexadecimal format without a leading "0x". -# If an plugin is used as well, the plugin check will be made first. -#psk_file - -# Control access to topics on the broker using an access control list -# file. If this parameter is defined then only the topics listed will -# have access. -# If the first character of a line of the ACL file is a # it is treated as a -# comment. -# Topic access is added with lines of the format: -# -# topic [read|write|readwrite|deny] -# -# The access type is controlled using "read", "write", "readwrite" or "deny". -# This parameter is optional (unless contains a space character) - if -# not given then the access is read/write. can contain the + or # -# wildcards as in subscriptions. -# -# The "deny" option can used to explicity deny access to a topic that would -# otherwise be granted by a broader read/write/readwrite statement. Any "deny" -# topics are handled before topics that grant read/write access. -# -# The first set of topics are applied to anonymous clients, assuming -# allow_anonymous is true. User specific topic ACLs are added after a -# user line as follows: -# -# user -# -# The username referred to here is the same as in password_file. It is -# not the clientid. -# -# -# If is also possible to define ACLs based on pattern substitution within the -# topic. The patterns available for substition are: -# -# %c to match the client id of the client -# %u to match the username of the client -# -# The substitution pattern must be the only text for that level of hierarchy. -# -# The form is the same as for the topic keyword, but using pattern as the -# keyword. -# Pattern ACLs apply to all users even if the "user" keyword has previously -# been given. -# -# If using bridges with usernames and ACLs, connection messages can be allowed -# with the following pattern: -# pattern write $SYS/broker/connection/%c/state -# -# pattern [read|write|readwrite] -# -# Example: -# -# pattern write sensor/%u/data -# -# If an plugin is used as well as acl_file, the plugin check will be -# made first. -#acl_file - -# ----------------------------------------------------------------- -# External authentication and topic access plugin options -# ----------------------------------------------------------------- - -# External authentication and access control can be supported with the -# plugin option. This is a path to a loadable plugin. See also the -# plugin_opt_* options described below. -# -# The plugin option can be specified multiple times to load multiple -# plugins. The plugins will be processed in the order that they are specified -# here. If the plugin option is specified alongside either of -# password_file or acl_file then the plugin checks will be made first. -# -# If the per_listener_settings option is false, the plugin will be apply to all -# listeners. If per_listener_settings is true, then the plugin will apply to -# the current listener being defined only. -# -# This option is also available as `auth_plugin`, but this use is deprecated -# and will be removed in the future. -# -#plugin - -# If the plugin option above is used, define options to pass to the -# plugin here as described by the plugin instructions. All options named -# using the format plugin_opt_* will be passed to the plugin, for example: -# -# This option is also available as `auth_opt_*`, but this use is deprecated -# and will be removed in the future. -# -# plugin_opt_db_host -# plugin_opt_db_port -# plugin_opt_db_username -# plugin_opt_db_password - - -# ================================================================= -# Bridges -# ================================================================= - -# A bridge is a way of connecting multiple MQTT brokers together. -# Create a new bridge using the "connection" option as described below. Set -# options for the bridges using the remaining parameters. You must specify the -# address and at least one topic to subscribe to. -# -# Each connection must have a unique name. -# -# The address line may have multiple host address and ports specified. See -# below in the round_robin description for more details on bridge behaviour if -# multiple addresses are used. Note that if you use an IPv6 address, then you -# are required to specify a port. -# -# The direction that the topic will be shared can be chosen by -# specifying out, in or both, where the default value is out. -# The QoS level of the bridged communication can be specified with the next -# topic option. The default QoS level is 0, to change the QoS the topic -# direction must also be given. -# -# The local and remote prefix options allow a topic to be remapped when it is -# bridged to/from the remote broker. This provides the ability to place a topic -# tree in an appropriate location. -# -# For more details see the mosquitto.conf man page. -# -# Multiple topics can be specified per connection, but be careful -# not to create any loops. -# -# If you are using bridges with cleansession set to false (the default), then -# you may get unexpected behaviour from incoming topics if you change what -# topics you are subscribing to. This is because the remote broker keeps the -# subscription for the old topic. If you have this problem, connect your bridge -# with cleansession set to true, then reconnect with cleansession set to false -# as normal. -#connection -#address [:] [[:]] -#topic [[[out | in | both] qos-level] local-prefix remote-prefix] - -# If you need to have the bridge connect over a particular network interface, -# use bridge_bind_address to tell the bridge which local IP address the socket -# should bind to, e.g. `bridge_bind_address 192.168.1.10` -#bridge_bind_address - -# If a bridge has topics that have "out" direction, the default behaviour is to -# send an unsubscribe request to the remote broker on that topic. This means -# that changing a topic direction from "in" to "out" will not keep receiving -# incoming messages. Sending these unsubscribe requests is not always -# desirable, setting bridge_attempt_unsubscribe to false will disable sending -# the unsubscribe request. -#bridge_attempt_unsubscribe true - -# Set the version of the MQTT protocol to use with for this bridge. Can be one -# of mqttv50, mqttv311 or mqttv31. Defaults to mqttv311. -#bridge_protocol_version mqttv311 - -# Set the clean session variable for this bridge. -# When set to true, when the bridge disconnects for any reason, all -# messages and subscriptions will be cleaned up on the remote -# broker. Note that with cleansession set to true, there may be a -# significant amount of retained messages sent when the bridge -# reconnects after losing its connection. -# When set to false, the subscriptions and messages are kept on the -# remote broker, and delivered when the bridge reconnects. -#cleansession false - -# Set the amount of time a bridge using the lazy start type must be idle before -# it will be stopped. Defaults to 60 seconds. -#idle_timeout 60 - -# Set the keepalive interval for this bridge connection, in -# seconds. -#keepalive_interval 60 - -# Set the clientid to use on the local broker. If not defined, this defaults to -# 'local.'. If you are bridging a broker to itself, it is important -# that local_clientid and clientid do not match. -#local_clientid - -# If set to true, publish notification messages to the local and remote brokers -# giving information about the state of the bridge connection. Retained -# messages are published to the topic $SYS/broker/connection//state -# unless the notification_topic option is used. -# If the message is 1 then the connection is active, or 0 if the connection has -# failed. -# This uses the last will and testament feature. -#notifications true - -# Choose the topic on which notification messages for this bridge are -# published. If not set, messages are published on the topic -# $SYS/broker/connection//state -#notification_topic - -# Set the client id to use on the remote end of this bridge connection. If not -# defined, this defaults to 'name.hostname' where name is the connection name -# and hostname is the hostname of this computer. -# This replaces the old "clientid" option to avoid confusion. "clientid" -# remains valid for the time being. -#remote_clientid - -# Set the password to use when connecting to a broker that requires -# authentication. This option is only used if remote_username is also set. -# This replaces the old "password" option to avoid confusion. "password" -# remains valid for the time being. -#remote_password - -# Set the username to use when connecting to a broker that requires -# authentication. -# This replaces the old "username" option to avoid confusion. "username" -# remains valid for the time being. -#remote_username - -# Set the amount of time a bridge using the automatic start type will wait -# until attempting to reconnect. -# This option can be configured to use a constant delay time in seconds, or to -# use a backoff mechanism based on "Decorrelated Jitter", which adds a degree -# of randomness to when the restart occurs. -# -# Set a constant timeout of 20 seconds: -# restart_timeout 20 -# -# Set backoff with a base (start value) of 10 seconds and a cap (upper limit) of -# 60 seconds: -# restart_timeout 10 30 -# -# Defaults to jitter with a base of 5 and cap of 30 -#restart_timeout 5 30 - -# If the bridge has more than one address given in the address/addresses -# configuration, the round_robin option defines the behaviour of the bridge on -# a failure of the bridge connection. If round_robin is false, the default -# value, then the first address is treated as the main bridge connection. If -# the connection fails, the other secondary addresses will be attempted in -# turn. Whilst connected to a secondary bridge, the bridge will periodically -# attempt to reconnect to the main bridge until successful. -# If round_robin is true, then all addresses are treated as equals. If a -# connection fails, the next address will be tried and if successful will -# remain connected until it fails -#round_robin false - -# Set the start type of the bridge. This controls how the bridge starts and -# can be one of three types: automatic, lazy and once. Note that RSMB provides -# a fourth start type "manual" which isn't currently supported by mosquitto. -# -# "automatic" is the default start type and means that the bridge connection -# will be started automatically when the broker starts and also restarted -# after a short delay (30 seconds) if the connection fails. -# -# Bridges using the "lazy" start type will be started automatically when the -# number of queued messages exceeds the number set with the "threshold" -# parameter. It will be stopped automatically after the time set by the -# "idle_timeout" parameter. Use this start type if you wish the connection to -# only be active when it is needed. -# -# A bridge using the "once" start type will be started automatically when the -# broker starts but will not be restarted if the connection fails. -#start_type automatic - -# Set the number of messages that need to be queued for a bridge with lazy -# start type to be restarted. Defaults to 10 messages. -# Must be less than max_queued_messages. -#threshold 10 - -# If try_private is set to true, the bridge will attempt to indicate to the -# remote broker that it is a bridge not an ordinary client. If successful, this -# means that loop detection will be more effective and that retained messages -# will be propagated correctly. Not all brokers support this feature so it may -# be necessary to set try_private to false if your bridge does not connect -# properly. -#try_private true - -# Some MQTT brokers do not allow retained messages. MQTT v5 gives a mechanism -# for brokers to tell clients that they do not support retained messages, but -# this is not possible for MQTT v3.1.1 or v3.1. If you need to bridge to a -# v3.1.1 or v3.1 broker that does not support retained messages, set the -# bridge_outgoing_retain option to false. This will remove the retain bit on -# all outgoing messages to that bridge, regardless of any other setting. -#bridge_outgoing_retain true - -# If you wish to restrict the size of messages sent to a remote bridge, use the -# bridge_max_packet_size option. This sets the maximum number of bytes for -# the total message, including headers and payload. -# Note that MQTT v5 brokers may provide their own maximum-packet-size property. -# In this case, the smaller of the two limits will be used. -# Set to 0 for "unlimited". -#bridge_max_packet_size 0 - - -# ----------------------------------------------------------------- -# Certificate based SSL/TLS support -# ----------------------------------------------------------------- -# Either bridge_cafile or bridge_capath must be defined to enable TLS support -# for this bridge. -# bridge_cafile defines the path to a file containing the -# Certificate Authority certificates that have signed the remote broker -# certificate. -# bridge_capath defines a directory that will be searched for files containing -# the CA certificates. For bridge_capath to work correctly, the certificate -# files must have ".crt" as the file ending and you must run "openssl rehash -# " each time you add/remove a certificate. -#bridge_cafile -#bridge_capath - - -# If the remote broker has more than one protocol available on its port, e.g. -# MQTT and WebSockets, then use bridge_alpn to configure which protocol is -# requested. Note that WebSockets support for bridges is not yet available. -#bridge_alpn - -# When using certificate based encryption, bridge_insecure disables -# verification of the server hostname in the server certificate. This can be -# useful when testing initial server configurations, but makes it possible for -# a malicious third party to impersonate your server through DNS spoofing, for -# example. Use this option in testing only. If you need to resort to using this -# option in a production environment, your setup is at fault and there is no -# point using encryption. -#bridge_insecure false - -# Path to the PEM encoded client certificate, if required by the remote broker. -#bridge_certfile - -# Path to the PEM encoded client private key, if required by the remote broker. -#bridge_keyfile - -# ----------------------------------------------------------------- -# PSK based SSL/TLS support -# ----------------------------------------------------------------- -# Pre-shared-key encryption provides an alternative to certificate based -# encryption. A bridge can be configured to use PSK with the bridge_identity -# and bridge_psk options. These are the client PSK identity, and pre-shared-key -# in hexadecimal format with no "0x". Only one of certificate and PSK based -# encryption can be used on one -# bridge at once. -#bridge_identity -#bridge_psk - - -# ================================================================= -# External config files -# ================================================================= - -# External configuration files may be included by using the -# include_dir option. This defines a directory that will be searched -# for config files. All files that end in '.conf' will be loaded as -# a configuration file. It is best to have this as the last option -# in the main file. This option will only be processed from the main -# configuration file. The directory specified must not contain the -# main configuration file. -# Files within include_dir will be loaded sorted in case-sensitive -# alphabetical order, with capital letters ordered first. If this option is -# given multiple times, all of the files from the first instance will be -# processed before the next instance. See the man page for examples. -#include_dir \ No newline at end of file From 4844445a8ed35c20d6aac0b1cbda4ea5caee5661 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 10 Jun 2024 15:22:19 +0200 Subject: [PATCH 03/14] feat: test suscribe to actual mqtt broker to verify the request was received --- .../stellio/subscription/service/mqtt/Mqtt.kt | 1 + .../service/mqtt/MqttNotificationData.kt | 14 +- .../service/mqtt/MqttNotificationService.kt | 52 +++++--- .../mqtt/MqttNotificationServiceTest.kt | 123 +++++++++++++++--- .../support/WithMosquittoContainer.kt | 7 +- .../test/resources/mosquitto/mosquitto.conf | 1 - 6 files changed, 154 insertions(+), 44 deletions(-) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt index a258e8a95f..7512b6b5d5 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/Mqtt.kt @@ -11,6 +11,7 @@ object Mqtt { object QualityOfService { const val KEY = "MQTT-QoS" const val AT_MOST_ONCE = 0 + const val EXACTLY_ONCE = 2 } object SCHEME { diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt index 3d6ed21672..d5eb79721b 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationData.kt @@ -4,12 +4,9 @@ import com.egm.stellio.subscription.model.Notification data class MqttNotificationData( val topic: String, - val mqttMessage: MqttMessage, + val message: MqttMessage, val qos: Int, - val brokerUrl: String, - val clientId: String, - val username: String, - val password: String? = null, + val connection: MqttConnectionData ) { data class MqttMessage( @@ -17,3 +14,10 @@ data class MqttNotificationData( val metadata: Map = emptyMap(), ) } + +data class MqttConnectionData( + val brokerUrl: String, + val clientId: String, + val username: String, + val password: String? = null, +) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt index 5290f9a1fb..e101fc6bcc 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt @@ -47,12 +47,14 @@ class MqttNotificationService( val data = MqttNotificationData( topic = uri.path, - brokerUrl = brokerUrl, - clientId = clientId, qos = qos, - mqttMessage = MqttNotificationData.MqttMessage(notification, headers), - username = username, - password = password + message = MqttNotificationData.MqttMessage(notification, headers), + connection = MqttConnectionData( + brokerUrl = brokerUrl, + clientId = clientId, + username = username, + password = password + ) ) try { @@ -62,18 +64,28 @@ class MqttNotificationService( Mqtt.Version.V5 -> callMqttV5(data) else -> callMqttV5(data) } - logger.info("successfull mqtt notification for uri : ${data.brokerUrl} version: $mqttVersion") + logger.info("successfull mqtt notification for uri : $uri version: $mqttVersion") return true } catch (e: MqttExceptionV3) { - logger.error("failed mqttv3 notification for uri : ${data.brokerUrl}", e) + logger.error("failed mqttv3 notification for uri : $uri", e) return false } catch (e: MqttExceptionV5) { - logger.error("failed mqttv5 notification for uri : ${data.brokerUrl}", e) + logger.error("failed mqttv5 notification for uri : $uri", e) return false } } internal suspend fun callMqttV3(data: MqttNotificationData) { + val mqttClient = connectMqttv3(data.connection) + val message = MqttMessage( + serializeObject(data.message).toByteArray() + ) + message.qos = data.qos + mqttClient.publish(data.topic, message) + mqttClient.disconnect() + } + + internal suspend fun connectMqttv3(data: MqttConnectionData): MqttClient { val persistence = MemoryPersistence() val mqttClient = MqttClient(data.brokerUrl, data.clientId, persistence) val connOpts = MqttConnectOptions() @@ -81,28 +93,28 @@ class MqttNotificationService( connOpts.userName = data.username connOpts.password = data.password?.toCharArray() ?: "".toCharArray() mqttClient.connect(connOpts) - val message = MqttMessage( - serializeObject(data.mqttMessage).toByteArray() - ) + return mqttClient + } + + internal suspend fun callMqttV5(data: MqttNotificationData) { + val mqttClient = connectMqttv5(data.connection) + val message = org.eclipse.paho.mqttv5.common.MqttMessage(serializeObject(data.message).toByteArray()) message.qos = data.qos - mqttClient.publish(data.topic, message) + val token = mqttClient.publish(data.topic, message) + token.waitForCompletion() mqttClient.disconnect() + mqttClient.close() } - internal suspend fun callMqttV5(data: MqttNotificationData) { + internal suspend fun connectMqttv5(data: MqttConnectionData): MqttAsyncClient { val persistence = org.eclipse.paho.mqttv5.client.persist.MemoryPersistence() val mqttClient = MqttAsyncClient(data.brokerUrl, data.clientId, persistence) val connOpts = MqttConnectionOptions() connOpts.isCleanStart = true connOpts.userName = data.username connOpts.password = data.password?.toByteArray() - var token: IMqttToken = mqttClient.connect(connOpts) - token.waitForCompletion() - val message = org.eclipse.paho.mqttv5.common.MqttMessage(serializeObject(data.mqttMessage).toByteArray()) - message.qos = data.qos - token = mqttClient.publish(data.topic, message) + val token: IMqttToken = mqttClient.connect(connOpts) token.waitForCompletion() - mqttClient.disconnect() - mqttClient.close() + return mqttClient } } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt index 93640adfea..d5370f8234 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt @@ -1,6 +1,7 @@ package com.egm.stellio.subscription.service.mqtt import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SUBSCRIPTION_TERM +import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.toUri import com.egm.stellio.subscription.model.* import com.egm.stellio.subscription.support.WithMosquittoContainer @@ -8,13 +9,23 @@ import com.ninjasquad.springmockk.SpykBean import io.mockk.coEvery import io.mockk.coVerify import kotlinx.coroutines.test.runTest +import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken import org.eclipse.paho.client.mqttv3.MqttException +import org.eclipse.paho.mqttv5.client.IMqttToken +import org.eclipse.paho.mqttv5.client.MqttDisconnectResponse +import org.eclipse.paho.mqttv5.common.MqttSubscription +import org.eclipse.paho.mqttv5.common.packet.MqttProperties +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow import org.junit.jupiter.api.assertThrows import org.springframework.boot.test.context.SpringBootTest import org.springframework.test.context.ActiveProfiles import java.net.URI +import org.eclipse.paho.client.mqttv3.MqttCallback as MqttCallbackV3 +import org.eclipse.paho.client.mqttv3.MqttMessage as MqttMessageV3 +import org.eclipse.paho.mqttv5.client.MqttCallback as MqttCallbackV5 +import org.eclipse.paho.mqttv5.common.MqttMessage as MqttMessageV5 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [MqttNotificationService::class]) @ActiveProfiles("test") @@ -23,9 +34,8 @@ class MqttNotificationServiceTest : WithMosquittoContainer { @SpykBean private lateinit var mqttNotificationService: MqttNotificationService - private val mqttContainerPort = WithMosquittoContainer.mosquittoContainer.getMappedPort( - Mqtt.SCHEME.MQTT_DEFAULT_PORT - ) + private val mqttContainerPort = WithMosquittoContainer.getBasicPort() + private val mqttSubscriptionV3 = Subscription( type = NGSILD_SUBSCRIPTION_TERM, subscriptionName = "My Subscription", @@ -38,7 +48,7 @@ class MqttNotificationServiceTest : WithMosquittoContainer { accept = Endpoint.AcceptType.JSONLD, notifierInfo = listOf( EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V3), - EndpointInfo(Mqtt.QualityOfService.KEY, Mqtt.QualityOfService.AT_MOST_ONCE.toString()) + EndpointInfo(Mqtt.QualityOfService.KEY, Mqtt.QualityOfService.EXACTLY_ONCE.toString()) ) ) ), @@ -53,19 +63,22 @@ class MqttNotificationServiceTest : WithMosquittoContainer { endpoint = Endpoint( uri = "mqtt://test@localhost:$mqttContainerPort/notification".toUri(), notifierInfo = listOf( - EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V5) + EndpointInfo(Mqtt.Version.KEY, Mqtt.Version.V5), + EndpointInfo(Mqtt.QualityOfService.KEY, Mqtt.QualityOfService.EXACTLY_ONCE.toString()) ) ) ), contexts = emptyList() ) private val validMqttNotificationData = MqttNotificationData( - brokerUrl = "tcp://localhost:$mqttContainerPort", - clientId = "clientId", - username = "test", + connection = MqttConnectionData( + brokerUrl = "tcp://localhost:$mqttContainerPort", + clientId = "clientId", + username = "test", + ), topic = "/notification", qos = 0, - mqttMessage = MqttNotificationData.MqttMessage(getNotificationForSubscription(mqttSubscriptionV3), emptyMap()) + message = MqttNotificationData.MqttMessage(getNotificationForSubscription(mqttSubscriptionV3), emptyMap()) ) private val notification = Notification( @@ -74,12 +87,17 @@ class MqttNotificationServiceTest : WithMosquittoContainer { ) private val invalidUriMqttNotificationData = MqttNotificationData( - brokerUrl = "tcp://badHost:1883", - clientId = "clientId", - username = "test", + connection = MqttConnectionData( + brokerUrl = "tcp://badHost:1883", + clientId = "clientId", + username = "test", + ), topic = "notification", qos = 0, - mqttMessage = MqttNotificationData.MqttMessage(notification, emptyMap()) + message = MqttNotificationData.MqttMessage( + notification, + emptyMap(), + ) ) private fun getNotificationForSubscription(subscription: Subscription) = Notification( @@ -88,7 +106,7 @@ class MqttNotificationServiceTest : WithMosquittoContainer { ) @Test - fun `mqttNotifier should process endpoint uri to get connexion information`() = runTest { + fun `mqttNotifier should process endpoint uri to get connection information`() = runTest { val subscription = mqttSubscriptionV3 coEvery { mqttNotificationService.callMqttV3(any()) } returns Unit assert( @@ -102,10 +120,10 @@ class MqttNotificationServiceTest : WithMosquittoContainer { coVerify { mqttNotificationService.callMqttV3( match { - it.username == validMqttNotificationData.username && - it.password == validMqttNotificationData.password && + it.connection.username == validMqttNotificationData.connection.username && + it.connection.password == validMqttNotificationData.connection.password && it.topic == validMqttNotificationData.topic && - it.brokerUrl == validMqttNotificationData.brokerUrl + it.connection.brokerUrl == validMqttNotificationData.connection.brokerUrl } ) } @@ -152,7 +170,21 @@ class MqttNotificationServiceTest : WithMosquittoContainer { @Test fun `sending mqttV3 notification with good uri should succeed`() = runTest { + // if we give the same clientId the mqtt server close the connection + val testConnectionData = validMqttNotificationData.connection.copy(clientId = "test-broker") + val mqttClient = mqttNotificationService.connectMqttv3(testConnectionData) + val messageReceiver = MqttV3MessageReceiver() + mqttClient.setCallback(messageReceiver) + mqttClient.subscribe(validMqttNotificationData.topic) + assertDoesNotThrow { mqttNotificationService.callMqttV3(validMqttNotificationData) } + Thread.sleep(10) // wait to receive notification in message receiver + assertEquals( + serializeObject(validMqttNotificationData.message), + messageReceiver.lastReceivedMessage + ) + mqttClient.disconnect() + mqttClient.close() } @Test @@ -162,7 +194,20 @@ class MqttNotificationServiceTest : WithMosquittoContainer { @Test fun `sending mqttV5 notification with good uri should succeed`() = runTest { + val testConnectionData = validMqttNotificationData.connection.copy(clientId = "test-broker") + val mqttClient = mqttNotificationService.connectMqttv5(testConnectionData) + val messageReceiver = MqttV5MessageReceiver() + mqttClient.setCallback(messageReceiver) + mqttClient.subscribe(MqttSubscription(validMqttNotificationData.topic)) + assertDoesNotThrow { mqttNotificationService.callMqttV5(validMqttNotificationData) } + + assertEquals( + serializeObject(validMqttNotificationData.message), + messageReceiver.lastReceivedMessage + ) + mqttClient.disconnect() + mqttClient.close() } @Test @@ -173,4 +218,48 @@ class MqttNotificationServiceTest : WithMosquittoContainer { ) } } + + private class MqttV3MessageReceiver : MqttCallbackV3 { + var lastReceivedMessage: String? = null + + override fun messageArrived(topic: String, message: MqttMessageV3) { + lastReceivedMessage = message.payload?.decodeToString() + } + + override fun deliveryComplete(p0: IMqttDeliveryToken?) { + println("delivery complete") + } + + override fun connectionLost(p0: Throwable?) { + println("connection lost") + } + } + + private class MqttV5MessageReceiver : MqttCallbackV5 { + var lastReceivedMessage: String? = null + + override fun messageArrived(topic: String?, message: MqttMessageV5?) { + lastReceivedMessage = message?.payload?.decodeToString() + } + + override fun disconnected(p0: MqttDisconnectResponse?) { + println("connection lost") + } + + override fun mqttErrorOccurred(p0: org.eclipse.paho.mqttv5.common.MqttException?) { + println("mqtt error occured") + } + + override fun deliveryComplete(p0: IMqttToken?) { + println("delivery complete") + } + + override fun connectComplete(p0: Boolean, p1: String?) { + println("connection success") + } + + override fun authPacketArrived(p0: Int, p1: MqttProperties?) { + println("auth") + } + } } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt index 5db96c764a..fcef13bd45 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt @@ -1,5 +1,6 @@ package com.egm.stellio.subscription.support +import com.egm.stellio.subscription.service.mqtt.Mqtt import org.testcontainers.containers.GenericContainer import org.testcontainers.utility.DockerImageName import org.testcontainers.utility.MountableFile @@ -13,13 +14,17 @@ interface WithMosquittoContainer { val mosquittoContainer = GenericContainer(mosquittoImage).apply { withReuse(true) - withExposedPorts(1883) + withExposedPorts(Mqtt.SCHEME.MQTT_DEFAULT_PORT) withCopyFileToContainer( MountableFile.forClasspathResource("/mosquitto/mosquitto.conf"), "/mosquitto/config/mosquitto.conf" ) } + fun getBasicPort() = mosquittoContainer.getMappedPort( + Mqtt.SCHEME.MQTT_DEFAULT_PORT + ) + init { mosquittoContainer.start() } diff --git a/subscription-service/src/test/resources/mosquitto/mosquitto.conf b/subscription-service/src/test/resources/mosquitto/mosquitto.conf index 65adc872cb..2bfe79bd5b 100644 --- a/subscription-service/src/test/resources/mosquitto/mosquitto.conf +++ b/subscription-service/src/test/resources/mosquitto/mosquitto.conf @@ -1,4 +1,3 @@ # see possible config : https://mosquitto.org/man/mosquitto-conf-5.html allow_anonymous true listener 1883 -listener 9001 From 4d39ff15d6827bfdb2aa999ce479e958451941fa Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 10 Jun 2024 15:33:34 +0200 Subject: [PATCH 04/14] feat: test suscribe to actual mqtt broker to verify the request was received --- .../subscription/service/mqtt/MqttNotificationService.kt | 4 ++-- .../src/main/resources/application.properties | 2 +- .../subscription/service/mqtt/MqttNotificationServiceTest.kt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt index e101fc6bcc..fbe2df6c95 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt @@ -19,8 +19,8 @@ import org.eclipse.paho.mqttv5.common.MqttException as MqttExceptionV5 @Service class MqttNotificationService( - @Value("\${mqtt.clientId}") - private val clientId: String = "stellio-context-brokerUrl", + @Value("\${subscription.mqtt.clientId}") + private val clientId: String = "stellio-context-broker", ) { private val logger = LoggerFactory.getLogger(javaClass) diff --git a/subscription-service/src/main/resources/application.properties b/subscription-service/src/main/resources/application.properties index 730f430621..0ae0544bef 100644 --- a/subscription-service/src/main/resources/application.properties +++ b/subscription-service/src/main/resources/application.properties @@ -15,4 +15,4 @@ subscription.entity-service-url=http://localhost:8083 # Stellio url used to form the link to get the contexts associated to a notification subscription.stellio-url=http://localhost:8080 server.port=8084 -mqtt.clientId=stellio-mqtt-client +subscription.mqtt.clientId=stellio-mqtt-client diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt index d5370f8234..84f1815eb6 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt @@ -178,7 +178,7 @@ class MqttNotificationServiceTest : WithMosquittoContainer { mqttClient.subscribe(validMqttNotificationData.topic) assertDoesNotThrow { mqttNotificationService.callMqttV3(validMqttNotificationData) } - Thread.sleep(10) // wait to receive notification in message receiver + Thread.sleep(20) // wait to receive notification in message receiver assertEquals( serializeObject(validMqttNotificationData.message), messageReceiver.lastReceivedMessage From f95717d36c14dafea43d8b0f336acca8da500a81 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 17 Jun 2024 15:20:11 +0200 Subject: [PATCH 05/14] fix: MR comments --- .../egm/stellio/subscription/service/NotificationService.kt | 2 +- .../subscription/service/mqtt/MqttNotificationService.kt | 2 +- subscription-service/src/main/resources/application.properties | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt index b697667254..13417e24b4 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt @@ -87,7 +87,7 @@ class NotificationService( val result = kotlin.runCatching { if (uri.startsWith(Mqtt.SCHEME.MQTT)) { - headerMap["Content-Type"] = mediaType.toString() // could be common with line 99 ? + headerMap[HttpHeaders.CONTENT_TYPE] = mediaType.toString() // could be common with line 99 ? Triple( subscription, notification, diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt index fbe2df6c95..809b08fa45 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt @@ -20,7 +20,7 @@ import org.eclipse.paho.mqttv5.common.MqttException as MqttExceptionV5 @Service class MqttNotificationService( @Value("\${subscription.mqtt.clientId}") - private val clientId: String = "stellio-context-broker", + private val clientId: String, ) { private val logger = LoggerFactory.getLogger(javaClass) diff --git a/subscription-service/src/main/resources/application.properties b/subscription-service/src/main/resources/application.properties index 0ae0544bef..0714c8c05f 100644 --- a/subscription-service/src/main/resources/application.properties +++ b/subscription-service/src/main/resources/application.properties @@ -14,5 +14,6 @@ spring.security.oauth2.client.provider.keycloak.token-uri=https://my.sso/token subscription.entity-service-url=http://localhost:8083 # Stellio url used to form the link to get the contexts associated to a notification subscription.stellio-url=http://localhost:8080 -server.port=8084 subscription.mqtt.clientId=stellio-mqtt-client +server.port=8084 + From cb271de88ec6b00c1a43bb343cccd31d72675444 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Fri, 7 Jun 2024 17:54:00 +0200 Subject: [PATCH 06/14] fix: syntax fixes for mr --- .../service/NotificationService.kt | 2 +- .../service/mqtt/MqttNotificationService.kt | 6 ++--- .../service/NotificationServiceTests.kt | 8 +++---- .../mqtt/MqttNotificationServiceTest.kt | 24 +++++++++---------- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt index 13417e24b4..cd74ef970a 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt @@ -91,7 +91,7 @@ class NotificationService( Triple( subscription, notification, - mqttNotificationService.mqttNotifier( + mqttNotificationService.notify( notification = notification, subscription = subscription, headers = headerMap diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt index 809b08fa45..a84e225882 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationService.kt @@ -1,6 +1,5 @@ package com.egm.stellio.subscription.service.mqtt -import com.egm.stellio.shared.model.BadSchemeException import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.subscription.model.Notification import com.egm.stellio.subscription.model.Subscription @@ -25,7 +24,7 @@ class MqttNotificationService( private val logger = LoggerFactory.getLogger(javaClass) - suspend fun mqttNotifier( + suspend fun notify( subscription: Subscription, notification: Notification, headers: Map @@ -36,12 +35,11 @@ class MqttNotificationService( val username = userInfo.getOrNull(0) ?: "" val password = userInfo.getOrNull(1) val brokerScheme = Mqtt.SCHEME.brokerSchemeMap[uri.scheme] - ?: throw BadSchemeException("${uri.scheme} is not a valid mqtt scheme") val brokerPort = if (uri.port != -1) uri.port else Mqtt.SCHEME.defaultPortMap[uri.scheme] val brokerUrl = "$brokerScheme://${uri.host}:$brokerPort" - val notifierInfo = endpoint.notifierInfo?.map { it.key to it.value }?.toMap() ?: emptyMap() + val notifierInfo = endpoint.notifierInfo?.associate { it.key to it.value } ?: emptyMap() val qos = notifierInfo[Mqtt.QualityOfService.KEY]?.let { Integer.parseInt(it) } ?: Mqtt.QualityOfService.AT_MOST_ONCE diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt index 2b45990969..86296cd4be 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt @@ -579,11 +579,11 @@ class NotificationServiceTests { coEvery { subscriptionService.getContextsLink(any()) } returns buildContextLinkHeader(NGSILD_TEST_CORE_CONTEXT) coEvery { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } returns 1 - coEvery { mqttNotificationService.mqttNotifier(any(), any(), any()) } returns true + coEvery { mqttNotificationService.notify(any(), any(), any()) } returns true notificationService.callSubscriber(subscription, rawEntity.deserializeAsMap()) - coVerify(exactly = 1) { mqttNotificationService.mqttNotifier(any(), any(), any()) } + coVerify(exactly = 1) { mqttNotificationService.notify(any(), any(), any()) } } @Test @@ -601,7 +601,7 @@ class NotificationServiceTests { ) ) - coEvery { mqttNotificationService.mqttNotifier(any(), any(), any()) } returns true + coEvery { mqttNotificationService.notify(any(), any(), any()) } returns true coEvery { subscriptionService.getContextsLink(any()) } returns buildContextLinkHeader(NGSILD_TEST_CORE_CONTEXT) coEvery { subscriptionService.updateSubscriptionNotification(any(), any(), any()) } returns 1 @@ -609,7 +609,7 @@ class NotificationServiceTests { notificationService.callSubscriber(subscription, rawEntity.deserializeAsMap()) coVerify { - mqttNotificationService.mqttNotifier( + mqttNotificationService.notify( any(), any(), match { diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt index 84f1815eb6..1b230f41dd 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/mqtt/MqttNotificationServiceTest.kt @@ -11,6 +11,7 @@ import io.mockk.coVerify import kotlinx.coroutines.test.runTest import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken import org.eclipse.paho.client.mqttv3.MqttException +import org.junit.jupiter.api.Assertions.assertTrue import org.eclipse.paho.mqttv5.client.IMqttToken import org.eclipse.paho.mqttv5.client.MqttDisconnectResponse import org.eclipse.paho.mqttv5.common.MqttSubscription @@ -106,13 +107,13 @@ class MqttNotificationServiceTest : WithMosquittoContainer { ) @Test - fun `mqttNotifier should process endpoint uri to get connection information`() = runTest { + fun `notify should process endpoint uri to get connection information`() = runTest { val subscription = mqttSubscriptionV3 coEvery { mqttNotificationService.callMqttV3(any()) } returns Unit - assert( - mqttNotificationService.mqttNotifier( - subscription, - getNotificationForSubscription(subscription), + assertTrue( + mqttNotificationService.notify( + mqttSubscriptionV3, + getNotificationForSubscription(mqttSubscriptionV3), mapOf() ) ) @@ -130,14 +131,13 @@ class MqttNotificationServiceTest : WithMosquittoContainer { } @Test - fun `mqttNotifier should use notifier info to choose the mqtt version`() = runTest { - val subscription = mqttSubscriptionV3 + fun `notify should use notifier info to choose the mqtt version`() = runTest { coEvery { mqttNotificationService.callMqttV3(any()) } returns Unit coEvery { mqttNotificationService.callMqttV5(any()) } returns Unit - mqttNotificationService.mqttNotifier( - subscription, - getNotificationForSubscription(subscription), + mqttNotificationService.notify( + mqttSubscriptionV3, + getNotificationForSubscription(mqttSubscriptionV3), mapOf() ) coVerify(exactly = 1) { @@ -151,9 +151,9 @@ class MqttNotificationServiceTest : WithMosquittoContainer { ) } - mqttNotificationService.mqttNotifier( + mqttNotificationService.notify( mqttSubscriptionV5, - getNotificationForSubscription(subscription), + getNotificationForSubscription(mqttSubscriptionV5), mapOf() ) coVerify(exactly = 1) { From 4579affd8d7d80b51b066dfb1571c2a8dfce9722 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Fri, 7 Jun 2024 16:07:00 +0200 Subject: [PATCH 07/14] feat: working TemporalPagination no test yet --- .../egm/stellio/search/scope/ScopeService.kt | 23 ++- .../service/AttributeInstanceService.kt | 13 +- .../stellio/search/util/EntitiesQueryUtils.kt | 22 ++- .../search/web/BuildTemporalApiResponse.kt | 143 ++++++++++++++++++ .../egm/stellio/search/web/EntityHandler.kt | 5 +- .../search/web/TemporalEntityHandler.kt | 19 +-- .../search/util/EntitiesQueryUtilsTests.kt | 12 +- .../com/egm/stellio/shared/util/DateUtils.kt | 6 + 8 files changed, 211 insertions(+), 32 deletions(-) create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index 9e40af9070..721dd7f4c2 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -89,7 +89,8 @@ class ScopeService( ): Either> { val temporalQuery = temporalEntitiesQuery.temporalQuery val sqlQueryBuilder = StringBuilder() - + val paginationQuery = temporalEntitiesQuery.entitiesQuery.paginationQuery + val limit = temporalQuery.lastN ?: paginationQuery.limit sqlQueryBuilder.append(composeSearchSelectStatement(temporalEntitiesQuery, origin)) sqlQueryBuilder.append( @@ -106,6 +107,7 @@ class ScopeService( TemporalQuery.Timerel.BETWEEN -> sqlQueryBuilder.append( " AND time > '${temporalQuery.timeAt}' AND time < '${temporalQuery.endTimeAt}'" ) + null -> Unit } @@ -114,9 +116,11 @@ class ScopeService( else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY entity_id") if (temporalQuery.lastN != null) - // in order to get last instances, need to order by time desc - // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start DESC LIMIT ${temporalQuery.lastN}") + // in order to get last instances, need to order by time desc + // final ascending ordering of instances is done in query service + sqlQueryBuilder.append(" ORDER BY start DESC") + else sqlQueryBuilder.append(" ORDER BY start ASC") + sqlQueryBuilder.append(" LIMIT $limit") return databaseClient.sql(sqlQueryBuilder.toString()) .bind("entities_ids", entitiesIds) @@ -149,11 +153,13 @@ class ScopeService( } else "SELECT entity_id, min(time) as start, max(time) as end, $allAggregates " } + temporalEntitiesQuery.temporalQuery.timeproperty == TemporalProperty.OBSERVED_AT -> { """ SELECT entity_id, ARRAY(SELECT jsonb_array_elements_text(value)) as value, time as start """ } + else -> { """ SELECT entity_id, ARRAY(SELECT jsonb_array_elements_text(value)) as value, time as start, sub @@ -245,12 +251,14 @@ class ScopeService( ) ) } + OperationType.APPEND_ATTRIBUTES, OperationType.MERGE_ENTITY -> { val newScopes = (currentScopes ?: emptyList()).toSet().plus(scopes).toList() val newPayload = newScopes.map { mapOf(JsonLdUtils.JSONLD_VALUE to it) } val updatedPayload = currentPayload.replaceScopeValue(newPayload) Pair(newScopes, updatedPayload) } + OperationType.APPEND_ATTRIBUTES_OVERWRITE_ALLOWED, OperationType.MERGE_ENTITY_OVERWRITE_ALLOWED, OperationType.PARTIAL_ATTRIBUTE_UPDATE, @@ -258,6 +266,7 @@ class ScopeService( val updatedPayload = currentPayload.replaceScopeValue(expandedAttributeInstances) Pair(scopes, updatedPayload) } + else -> Pair(null, Json.of("{}")) } @@ -269,9 +278,9 @@ class ScopeService( else TemporalProperty.MODIFIED_AT addHistoryEntry(entityId, it, temporalPropertyToAdd, modifiedAt, sub).bind() if (temporalPropertyToAdd == TemporalProperty.MODIFIED_AT) - // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a - // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt - // sub-Property + // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a + // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt + // sub-Property addHistoryEntry(entityId, it, TemporalProperty.OBSERVED_AT, modifiedAt, sub).bind() updateResult } ?: UpdateResult( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index 235cd5c095..211533db5f 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -9,14 +9,16 @@ import com.egm.stellio.search.model.* import com.egm.stellio.search.model.AggregatedAttributeInstanceResult.AggregateResult import com.egm.stellio.search.util.* import com.egm.stellio.shared.model.* -import com.egm.stellio.shared.util.* +import com.egm.stellio.shared.util.INCONSISTENT_VALUES_IN_AGGREGATION_MESSAGE +import com.egm.stellio.shared.util.attributeOrInstanceNotFoundMessage +import com.egm.stellio.shared.util.ngsiLdDateTime import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.net.URI import java.time.ZonedDateTime -import java.util.UUID +import java.util.* @Service class AttributeInstanceService( @@ -123,6 +125,7 @@ class AttributeInstanceService( ): Either> { val temporalQuery = temporalEntitiesQuery.temporalQuery val sqlQueryBuilder = StringBuilder() + val limit = temporalQuery.lastN ?: temporalEntitiesQuery.entitiesQuery.paginationQuery.limit sqlQueryBuilder.append(composeSearchSelectStatement(temporalQuery, temporalEntityAttributes, origin)) @@ -151,6 +154,7 @@ class AttributeInstanceService( TemporalQuery.Timerel.BETWEEN -> sqlQueryBuilder.append( " AND time > '${temporalQuery.timeAt}' AND time < '${temporalQuery.endTimeAt}'" ) + null -> Unit } @@ -162,7 +166,9 @@ class AttributeInstanceService( if (temporalQuery.lastN != null) // in order to get last instances, need to order by time desc // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start DESC LIMIT ${temporalQuery.lastN}") + sqlQueryBuilder.append(" ORDER BY start DESC") + else sqlQueryBuilder.append(" ORDER BY start ASC") + sqlQueryBuilder.append(" LIMIT $limit") val finalTemporalQuery = composeFinalTemporalQuery(temporalEntityAttributes, sqlQueryBuilder.toString()) @@ -218,6 +224,7 @@ class AttributeInstanceService( } else "SELECT temporal_entity_attribute, min(time) as start, max(time) as end, $allAggregates " } + else -> { val valueColumn = when (temporalEntityAttributes[0].attributeValueType) { TemporalEntityAttribute.AttributeValueType.NUMBER -> "measured_value as value" diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt index 99891b616b..a95c5f3624 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt @@ -6,11 +6,12 @@ import com.egm.stellio.search.model.* import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException +import com.egm.stellio.shared.model.TooManyResultsException import com.egm.stellio.shared.util.* import org.springframework.util.MultiValueMap import org.springframework.util.MultiValueMapAdapter import java.time.ZonedDateTime -import java.util.Optional +import java.util.* fun composeEntitiesQuery( defaultPagination: ApplicationProperties.Pagination, @@ -129,7 +130,8 @@ fun composeTemporalEntitiesQuery( Optional.ofNullable(requestParams.getFirst(QUERY_PARAM_OPTIONS)), OptionsParamValue.AGGREGATED_VALUES ) - val temporalQuery = buildTemporalQuery(requestParams, inQueryEntities, withAggregatedValues).bind() + val maxLastN = defaultPagination.limitMax + val temporalQuery = buildTemporalQuery(requestParams, maxLastN, inQueryEntities, withAggregatedValues).bind() TemporalEntitiesQuery( entitiesQuery = entitiesQuery, @@ -176,7 +178,12 @@ fun composeTemporalEntitiesQueryFromPostRequest( "lastN" to listOf(query.temporalQ?.lastN.toString()), "timeproperty" to listOf(query.temporalQ?.timeproperty) ) - val temporalQuery = buildTemporalQuery(MultiValueMapAdapter(temporalParams), true, withAggregatedValues).bind() + val temporalQuery = buildTemporalQuery( + MultiValueMapAdapter(temporalParams), + defaultPagination.limitMax, + true, + withAggregatedValues, + ).bind() TemporalEntitiesQuery( entitiesQuery = entitiesQuery, @@ -189,8 +196,9 @@ fun composeTemporalEntitiesQueryFromPostRequest( fun buildTemporalQuery( params: MultiValueMap, + maxLastN: Int, inQueryEntities: Boolean = false, - withAggregatedValues: Boolean = false + withAggregatedValues: Boolean = false, ): Either { val timerelParam = params.getFirst("timerel") val timeAtParam = params.getFirst("timeAt") @@ -229,7 +237,11 @@ fun buildTemporalQuery( } val lastN = lastNParam?.toIntOrNull()?.let { - if (it >= 1) it else null + if (it > maxLastN) return TooManyResultsException( + "You asked for the $it last temporal entities, but the supported maximum limit is $maxLastN" + ).left() + else if (it >= 1) it + else null } return TemporalQuery( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt new file mode 100644 index 0000000000..6e9c00cdc8 --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt @@ -0,0 +1,143 @@ +package com.egm.stellio.search.web + +import com.egm.stellio.search.model.TemporalEntitiesQuery +import com.egm.stellio.search.model.TemporalQuery +import com.egm.stellio.shared.model.CompactedEntity +import com.egm.stellio.shared.model.toFinalRepresentation +import com.egm.stellio.shared.util.JsonUtils.serializeObject +import com.egm.stellio.shared.util.buildQueryResponse +import com.egm.stellio.shared.util.parseRepresentations +import com.egm.stellio.shared.util.prepareGetSuccessResponseHeaders +import com.egm.stellio.shared.util.toHttpHeaderFormat +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpStatus +import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity +import org.springframework.util.MultiValueMap +import java.time.ZonedDateTime + +typealias CompactedTemporalAttributes = List>> + +object TemporalApiResponse { + + fun buildListTemporalResponse( + entities: List, + total: Int, + resourceUrl: String, + query: TemporalEntitiesQuery, + requestParams: MultiValueMap, + mediaType: MediaType, + contexts: List + ): ResponseEntity { + val ngsiLdDataRepresentation = parseRepresentations(requestParams, mediaType) + + val successResponse = buildQueryResponse( + entities.toFinalRepresentation(ngsiLdDataRepresentation), + total, + resourceUrl, + query.entitiesQuery.paginationQuery, + requestParams, + mediaType, + contexts + ) + val attributesWhoReachedLimit = getAttributesWhoReachedLimit(entities, query) + + if (attributesWhoReachedLimit.isEmpty()) { + return successResponse + } + + return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).apply { + this.headers( + successResponse.headers + ) + this.header( + HttpHeaders.CONTENT_RANGE, + getTemporalPaginationRange(attributesWhoReachedLimit, query) + ) + }.build() + } + + fun buildEntityTemporalResponse( + entity: CompactedEntity, + query: TemporalEntitiesQuery, + mediaType: MediaType, + requestParams: MultiValueMap, + contexts: List, + ): ResponseEntity { + val ngsiLdDataRepresentation = parseRepresentations(requestParams, mediaType) + + val successResponse = prepareGetSuccessResponseHeaders(mediaType, contexts).body( + serializeObject( + entity.toFinalRepresentation(ngsiLdDataRepresentation) + ) + ) + + val attributesWhoReachedLimit = getAttributesWhoReachedLimit(listOf(entity), query) + + if (attributesWhoReachedLimit.isEmpty()) { + return successResponse + } + + return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT) + .apply { + this.header( + HttpHeaders.CONTENT_RANGE, + getTemporalPaginationRange(attributesWhoReachedLimit, query) + ) + this.headers( + successResponse.headers + ) + }.build() + } + + private fun getAttributesWhoReachedLimit(entities: List, query: TemporalEntitiesQuery): + CompactedTemporalAttributes { + val temporalQuery = query.temporalQuery + val lastN = temporalQuery.lastN + val maxSize = lastN ?: query.entitiesQuery.paginationQuery.limit + return entities.flatMap { + it.values.mapNotNull { + if (it is List<*> && it.size >= maxSize) it as List> else null + } + } + } + + private fun getTemporalPaginationRange( + attributesWhoReachedLimit: CompactedTemporalAttributes, + query: TemporalEntitiesQuery + ): String { + val temporalQuery = query.temporalQuery + val lastN = temporalQuery.lastN + val maxSize = lastN ?: query.entitiesQuery.paginationQuery.limit + val timeProperty = temporalQuery.timeproperty.propertyName + + val attributesTimeRanges = attributesWhoReachedLimit.map { attribute -> attribute.map { it[timeProperty] } } + .map { + ZonedDateTime.parse(it.getOrNull(0) as String) to + ZonedDateTime.parse(it.getOrNull(maxSize - 1) as String) + } + + val range = if (lastN == null) { + val discriminatingTimeRange = attributesTimeRanges.minBy { it.second } + val rangeStart = when (temporalQuery.timerel) { + TemporalQuery.Timerel.AFTER -> temporalQuery.timeAt + TemporalQuery.Timerel.BETWEEN -> temporalQuery.timeAt + else -> discriminatingTimeRange.first + } + + rangeStart to discriminatingTimeRange.second + } else { + val discriminatingTimeRange = attributesTimeRanges.maxBy { it.second } + val rangeStart = when (temporalQuery.timerel) { + TemporalQuery.Timerel.BEFORE -> temporalQuery.timeAt + TemporalQuery.Timerel.BETWEEN -> temporalQuery.endTimeAt + else -> discriminatingTimeRange.first + } + + rangeStart to discriminatingTimeRange.second + } + + val size = lastN?.toString() ?: "*" + return "DateTime ${range.first?.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" + } +} diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/EntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/EntityHandler.kt index 8e4203d622..6b3da823f9 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/EntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/EntityHandler.kt @@ -29,7 +29,7 @@ import org.springframework.util.MultiValueMap import org.springframework.web.bind.annotation.* import reactor.core.publisher.Mono import java.net.URI -import java.util.Optional +import java.util.* @RestController @RequestMapping("/ngsi-ld/v1/entities") @@ -194,7 +194,8 @@ class EntityHandler( val entitiesQuery = composeEntitiesQuery( applicationProperties.pagination, params, - contexts + contexts, + ).bind() .validateMinimalQueryEntitiesParameters().bind() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt index d8b67b2f91..2de567d7cb 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt @@ -5,8 +5,13 @@ import arrow.core.left import arrow.core.raise.either import arrow.core.right import com.egm.stellio.search.authorization.AuthorizationService -import com.egm.stellio.search.service.* +import com.egm.stellio.search.service.AttributeInstanceService +import com.egm.stellio.search.service.EntityPayloadService +import com.egm.stellio.search.service.QueryService +import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.search.util.composeTemporalEntitiesQuery +import com.egm.stellio.search.web.TemporalApiResponse.buildEntityTemporalResponse +import com.egm.stellio.search.web.TemporalApiResponse.buildListTemporalResponse import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.* import com.egm.stellio.shared.util.* @@ -17,7 +22,6 @@ import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.JsonLdUtils.expandAttributes import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdTerm -import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.web.BaseHandler import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus @@ -156,12 +160,11 @@ class TemporalEntityHandler( val compactedEntities = compactEntities(temporalEntities, contexts) - val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) - buildQueryResponse( - compactedEntities.toFinalRepresentation(ngsiLdDataRepresentation), + buildListTemporalResponse( + compactedEntities, total, "/ngsi-ld/v1/temporal/entities", - temporalEntitiesQuery.entitiesQuery.paginationQuery, + temporalEntitiesQuery, params, mediaType, contexts @@ -196,9 +199,7 @@ class TemporalEntityHandler( val compactedEntity = compactEntity(temporalEntity, contexts) - val ngsiLdDataRepresentation = parseRepresentations(params, mediaType) - prepareGetSuccessResponseHeaders(mediaType, contexts) - .body(serializeObject(compactedEntity.toFinalRepresentation(ngsiLdDataRepresentation))) + buildEntityTemporalResponse(compactedEntity, temporalEntitiesQuery, mediaType, params, contexts) }.fold( { it.toErrorResponse() }, { it } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt index 5b3836cac5..6d9e89aada 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt @@ -415,7 +415,7 @@ class EntitiesQueryUtilsTests { queryParams.add("timeAt", "2019-10-17T07:31:39Z") queryParams.add("lastN", "2") - val temporalQuery = buildTemporalQuery(queryParams).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() assertEquals(2, temporalQuery.lastN) } @@ -427,7 +427,7 @@ class EntitiesQueryUtilsTests { queryParams.add("timeAt", "2019-10-17T07:31:39Z") queryParams.add("lastN", "A") - val temporalQuery = buildTemporalQuery(queryParams).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() assertNull(temporalQuery.lastN) } @@ -439,7 +439,7 @@ class EntitiesQueryUtilsTests { queryParams.add("timeAt", "2019-10-17T07:31:39Z") queryParams.add("lastN", "-2") - val temporalQuery = buildTemporalQuery(queryParams).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() assertNull(temporalQuery.lastN) } @@ -448,7 +448,7 @@ class EntitiesQueryUtilsTests { fun `it should treat time and timerel properties as optional in a temporal query`() = runTest { val queryParams = LinkedMultiValueMap() - val temporalQuery = buildTemporalQuery(queryParams).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() assertNull(temporalQuery.timeAt) assertNull(temporalQuery.timerel) @@ -459,7 +459,7 @@ class EntitiesQueryUtilsTests { val queryParams = LinkedMultiValueMap() queryParams.add("timeproperty", "createdAt") - val temporalQuery = buildTemporalQuery(queryParams).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() assertEquals(AttributeInstance.TemporalProperty.CREATED_AT, temporalQuery.timeproperty) } @@ -468,7 +468,7 @@ class EntitiesQueryUtilsTests { fun `it should set timeproperty to observedAt if no value is provided in query parameters`() = runTest { val queryParams = LinkedMultiValueMap() - val temporalQuery = buildTemporalQuery(queryParams).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() assertEquals(AttributeInstance.TemporalProperty.OBSERVED_AT, temporalQuery.timeproperty) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt index b6f56ca241..0c24aac606 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt @@ -4,12 +4,18 @@ import java.time.* import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException import java.time.temporal.ChronoUnit +import java.util.* val formatter: DateTimeFormatter = DateTimeFormatter.ISO_INSTANT +val httpHeaderFormatter = + DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss z", Locale.ENGLISH).withZone(ZoneId.of("GMT")) fun ZonedDateTime.toNgsiLdFormat(): String = formatter.format(this) +fun ZonedDateTime.toHttpHeaderFormat(): String = + httpHeaderFormatter.format(this) + fun ngsiLdDateTime(): ZonedDateTime = Instant.now().truncatedTo(ChronoUnit.MICROS).atZone(ZoneOffset.UTC) From d0ac1e9bc4b142be9abc9938f1d5448f4f9035f5 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 10 Jun 2024 10:14:04 +0200 Subject: [PATCH 08/14] wip: different limit for temporalPagination --- .../egm/stellio/search/model/TemporalQuery.kt | 3 ++- .../egm/stellio/search/scope/ScopeService.kt | 12 ++++----- .../service/AttributeInstanceService.kt | 14 +++++------ .../stellio/search/util/EntitiesQueryUtils.kt | 25 +++++++++++-------- .../search/web/BuildTemporalApiResponse.kt | 10 +++----- .../shared/config/ApplicationProperties.kt | 4 ++- shared/src/main/resources/shared.properties | 2 ++ .../support/WithMosquittoContainer.kt | 2 ++ 8 files changed, 40 insertions(+), 32 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt index 481227b536..2ef5ed0a81 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt @@ -10,7 +10,8 @@ data class TemporalQuery( val endTimeAt: ZonedDateTime? = null, val aggrPeriodDuration: String? = null, val aggrMethods: List? = null, - val lastN: Int? = null, + val isChronological: Boolean? = null, + val limit: Int, val timeproperty: AttributeInstance.TemporalProperty = AttributeInstance.TemporalProperty.OBSERVED_AT ) { enum class Timerel { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index 721dd7f4c2..47305e254a 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -89,8 +89,7 @@ class ScopeService( ): Either> { val temporalQuery = temporalEntitiesQuery.temporalQuery val sqlQueryBuilder = StringBuilder() - val paginationQuery = temporalEntitiesQuery.entitiesQuery.paginationQuery - val limit = temporalQuery.lastN ?: paginationQuery.limit + val limit = temporalQuery.limit sqlQueryBuilder.append(composeSearchSelectStatement(temporalEntitiesQuery, origin)) sqlQueryBuilder.append( @@ -115,11 +114,12 @@ class ScopeService( sqlQueryBuilder.append(" GROUP BY entity_id, start") else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY entity_id") - if (temporalQuery.lastN != null) - // in order to get last instances, need to order by time desc + if (temporalQuery.isChronological == true) + // in order to get first or last instances, need to order by time // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start DESC") - else sqlQueryBuilder.append(" ORDER BY start ASC") + sqlQueryBuilder.append(" ORDER BY start ASC") + else sqlQueryBuilder.append(" ORDER BY start DESC") + sqlQueryBuilder.append(" LIMIT $limit") return databaseClient.sql(sqlQueryBuilder.toString()) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index 211533db5f..795696c5d1 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -125,7 +125,6 @@ class AttributeInstanceService( ): Either> { val temporalQuery = temporalEntitiesQuery.temporalQuery val sqlQueryBuilder = StringBuilder() - val limit = temporalQuery.lastN ?: temporalEntitiesQuery.entitiesQuery.paginationQuery.limit sqlQueryBuilder.append(composeSearchSelectStatement(temporalQuery, temporalEntityAttributes, origin)) @@ -163,12 +162,13 @@ class AttributeInstanceService( else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY temporal_entity_attribute") - if (temporalQuery.lastN != null) - // in order to get last instances, need to order by time desc - // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start DESC") - else sqlQueryBuilder.append(" ORDER BY start ASC") - sqlQueryBuilder.append(" LIMIT $limit") + if (temporalQuery.isChronological == true) + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service + sqlQueryBuilder.append(" ORDER BY start ASC") + else sqlQueryBuilder.append(" ORDER BY start DESC") + + sqlQueryBuilder.append(" LIMIT ${temporalQuery.limit}") val finalTemporalQuery = composeFinalTemporalQuery(temporalEntityAttributes, sqlQueryBuilder.toString()) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt index a95c5f3624..ee99821a86 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt @@ -130,8 +130,8 @@ fun composeTemporalEntitiesQuery( Optional.ofNullable(requestParams.getFirst(QUERY_PARAM_OPTIONS)), OptionsParamValue.AGGREGATED_VALUES ) - val maxLastN = defaultPagination.limitMax - val temporalQuery = buildTemporalQuery(requestParams, maxLastN, inQueryEntities, withAggregatedValues).bind() + val temporalQuery = + buildTemporalQuery(requestParams, defaultPagination, inQueryEntities, withAggregatedValues).bind() TemporalEntitiesQuery( entitiesQuery = entitiesQuery, @@ -180,7 +180,7 @@ fun composeTemporalEntitiesQueryFromPostRequest( ) val temporalQuery = buildTemporalQuery( MultiValueMapAdapter(temporalParams), - defaultPagination.limitMax, + defaultPagination, true, withAggregatedValues, ).bind() @@ -196,7 +196,7 @@ fun composeTemporalEntitiesQueryFromPostRequest( fun buildTemporalQuery( params: MultiValueMap, - maxLastN: Int, + pagination: ApplicationProperties.Pagination, inQueryEntities: Boolean = false, withAggregatedValues: Boolean = false, ): Either { @@ -235,14 +235,16 @@ fun buildTemporalQuery( "'$it' is not a recognized aggregation method for 'aggrMethods' parameter" ).left() } - - val lastN = lastNParam?.toIntOrNull()?.let { - if (it > maxLastN) return TooManyResultsException( - "You asked for the $it last temporal entities, but the supported maximum limit is $maxLastN" + val isChronological = lastNParam == null + val limit = lastNParam?.toIntOrNull()?.let { + if (it > pagination.temporalLimitMax) return TooManyResultsException( + "You asked for the $it last temporal entities, but the supported maximum limit is ${ + pagination.temporalLimitMax + }" ).left() else if (it >= 1) it - else null - } + else pagination.temporalLimitDefault + } ?: pagination.temporalLimitDefault return TemporalQuery( timerel = timerel, @@ -250,7 +252,8 @@ fun buildTemporalQuery( endTimeAt = endTimeAt, aggrPeriodDuration = aggrPeriodDurationParam, aggrMethods = aggregate, - lastN = lastN, + limit = limit, + isChronological = isChronological, timeproperty = timeproperty ).right() } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt index 6e9c00cdc8..a485f087d3 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt @@ -93,11 +93,9 @@ object TemporalApiResponse { private fun getAttributesWhoReachedLimit(entities: List, query: TemporalEntitiesQuery): CompactedTemporalAttributes { val temporalQuery = query.temporalQuery - val lastN = temporalQuery.lastN - val maxSize = lastN ?: query.entitiesQuery.paginationQuery.limit return entities.flatMap { it.values.mapNotNull { - if (it is List<*> && it.size >= maxSize) it as List> else null + if (it is List<*> && it.size >= temporalQuery.limit) it as List> else null } } } @@ -107,8 +105,8 @@ object TemporalApiResponse { query: TemporalEntitiesQuery ): String { val temporalQuery = query.temporalQuery - val lastN = temporalQuery.lastN - val maxSize = lastN ?: query.entitiesQuery.paginationQuery.limit + val lastN = temporalQuery.limit + val maxSize = lastN val timeProperty = temporalQuery.timeproperty.propertyName val attributesTimeRanges = attributesWhoReachedLimit.map { attribute -> attribute.map { it[timeProperty] } } @@ -137,7 +135,7 @@ object TemporalApiResponse { rangeStart to discriminatingTimeRange.second } - val size = lastN?.toString() ?: "*" + val size = lastN.toString() return "DateTime ${range.first?.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" } } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/config/ApplicationProperties.kt b/shared/src/main/kotlin/com/egm/stellio/shared/config/ApplicationProperties.kt index 759f1aa37f..f738bf948c 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/config/ApplicationProperties.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/config/ApplicationProperties.kt @@ -15,7 +15,9 @@ data class ApplicationProperties( data class Pagination( val limitDefault: Int, - val limitMax: Int + val limitMax: Int, + val temporalLimitDefault: Int, + val temporalLimitMax: Int ) data class TenantConfiguration( diff --git a/shared/src/main/resources/shared.properties b/shared/src/main/resources/shared.properties index 84664df75c..ecc5764cad 100644 --- a/shared/src/main/resources/shared.properties +++ b/shared/src/main/resources/shared.properties @@ -8,6 +8,8 @@ application.authentication.enabled = false # Pagination config for query resources endpoints application.pagination.limit-default = 30 application.pagination.limit-max = 100 +application.pagination.temporal-limit-default = 100 +application.pagination.temporal-limit-max = 1000 # default core context used when not provided in the query application.contexts.core = https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context-v1.8.jsonld diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt index fcef13bd45..e661cd7b22 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt @@ -28,5 +28,7 @@ interface WithMosquittoContainer { init { mosquittoContainer.start() } + + fun getMosquittoLogs(): String = mosquittoContainer.logs } } From 64e91acce29c3bf315900796b3418559beea3ecc Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Tue, 11 Jun 2024 18:32:28 +0200 Subject: [PATCH 09/14] feat: fix test for new TemporalQuery and --- .../egm/stellio/search/model/TemporalQuery.kt | 2 +- .../stellio/search/util/EntitiesQueryUtils.kt | 10 +- .../stellio/search/scope/ScopeServiceTests.kt | 20 ++-- .../search/scope/TemporalScopeBuilderTests.kt | 7 +- .../service/AggregatedQueryServiceTests.kt | 4 +- .../service/AttributeInstanceServiceTests.kt | 100 ++++++++++-------- .../search/service/QueryServiceTests.kt | 19 ++-- .../egm/stellio/search/support/TestUtils.kt | 38 +++++++ .../search/util/EntitiesQueryUtilsTests.kt | 54 ++++++---- .../search/util/TemporalEntityBuilderTests.kt | 9 +- .../search/web/TemporalEntityHandlerTests.kt | 5 +- .../TemporalEntityOperationsHandlerTests.kt | 5 +- 12 files changed, 170 insertions(+), 103 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt index 2ef5ed0a81..b6a478e1f6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt @@ -10,7 +10,7 @@ data class TemporalQuery( val endTimeAt: ZonedDateTime? = null, val aggrPeriodDuration: String? = null, val aggrMethods: List? = null, - val isChronological: Boolean? = null, + val isChronological: Boolean, val limit: Int, val timeproperty: AttributeInstance.TemporalProperty = AttributeInstance.TemporalProperty.OBSERVED_AT ) { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt index ee99821a86..a6336ebd1e 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt @@ -235,16 +235,18 @@ fun buildTemporalQuery( "'$it' is not a recognized aggregation method for 'aggrMethods' parameter" ).left() } - val isChronological = lastNParam == null - val limit = lastNParam?.toIntOrNull()?.let { + + val lastN = lastNParam?.toIntOrNull()?.let { if (it > pagination.temporalLimitMax) return TooManyResultsException( "You asked for the $it last temporal entities, but the supported maximum limit is ${ pagination.temporalLimitMax }" ).left() else if (it >= 1) it - else pagination.temporalLimitDefault - } ?: pagination.temporalLimitDefault + else null + } + val limit = lastN ?: pagination.temporalLimitDefault + val isChronological = lastN == null return TemporalQuery( timerel = timerel, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt index 5df8a4cc39..177c30fc66 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt @@ -5,6 +5,7 @@ import com.egm.stellio.search.model.AttributeInstance.TemporalProperty import com.egm.stellio.search.service.EntityPayloadService import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer +import com.egm.stellio.search.support.buildDefaultTestTemporalQuery import com.egm.stellio.search.util.toExpandedAttributeInstance import com.egm.stellio.shared.model.PaginationQuery import com.egm.stellio.shared.model.getScopes @@ -94,7 +95,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { val expandedAttributes = JsonLdUtils.expandAttributes( """ { - "scope": [${inputScopes.joinToString(",") { "\"$it\"" } }] + "scope": [${inputScopes.joinToString(",") { "\"$it\"" }}] } """.trimIndent(), APIC_COMPOUND_CONTEXTS @@ -138,7 +139,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery(timeproperty = TemporalProperty.MODIFIED_AT), + buildDefaultTestTemporalQuery(timeproperty = TemporalProperty.MODIFIED_AT), withTemporalValues = false, withAudit = false, withAggregatedValues = false @@ -164,7 +165,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timeproperty = TemporalProperty.MODIFIED_AT, timerel = TemporalQuery.Timerel.BEFORE, timeAt = ngsiLdDateTime() @@ -194,7 +195,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timeproperty = TemporalProperty.MODIFIED_AT, timerel = TemporalQuery.Timerel.BEFORE, timeAt = ngsiLdDateTime(), @@ -228,7 +229,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timeproperty = TemporalProperty.MODIFIED_AT, timerel = TemporalQuery.Timerel.BEFORE, timeAt = ngsiLdDateTime(), @@ -263,7 +264,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timeproperty = TemporalProperty.MODIFIED_AT, aggrMethods = listOf(TemporalQuery.Aggregate.SUM), aggrPeriodDuration = "PT0S" @@ -295,13 +296,14 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timeproperty = TemporalProperty.MODIFIED_AT, timerel = TemporalQuery.Timerel.BEFORE, timeAt = ngsiLdDateTime(), aggrMethods = listOf(TemporalQuery.Aggregate.SUM), aggrPeriodDuration = "PT1S", - lastN = 1 + limit = 1, + isChronological = false ), withTemporalValues = false, withAudit = false, @@ -338,7 +340,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { paginationQuery = PaginationQuery(limit = 100, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery(), + buildDefaultTestTemporalQuery(), withTemporalValues = false, withAudit = false, withAggregatedValues = false diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/scope/TemporalScopeBuilderTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/scope/TemporalScopeBuilderTests.kt index 03492fdb7e..f736e3abf5 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/scope/TemporalScopeBuilderTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/scope/TemporalScopeBuilderTests.kt @@ -4,6 +4,7 @@ import com.egm.stellio.search.model.AttributeInstance.TemporalProperty import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.support.buildDefaultQueryParams +import com.egm.stellio.search.support.buildDefaultTestTemporalQuery import com.egm.stellio.search.support.gimmeEntityPayload import com.egm.stellio.shared.util.JsonUtils import com.egm.stellio.shared.util.assertJsonPayloadsAreEqual @@ -59,7 +60,7 @@ class TemporalScopeBuilderTests { ) ) ) - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.AFTER, timeAt = Instant.now().atZone(ZoneOffset.UTC).minusHours(1), aggrPeriodDuration = "P1D", @@ -99,7 +100,7 @@ class TemporalScopeBuilderTests { time = ZonedDateTime.parse("2020-03-25T08:30:17.965206Z") ) ) - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( TemporalQuery.Timerel.AFTER, Instant.now().atZone(ZoneOffset.UTC).minusHours(1), ) @@ -139,7 +140,7 @@ class TemporalScopeBuilderTests { timeproperty = TemporalProperty.OBSERVED_AT.propertyName ) ) - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( TemporalQuery.Timerel.AFTER, Instant.now().atZone(ZoneOffset.UTC).minusHours(1), ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/AggregatedQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/AggregatedQueryServiceTests.kt index 6e2f1842b6..d8392b05d3 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/AggregatedQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/AggregatedQueryServiceTests.kt @@ -21,7 +21,7 @@ import org.springframework.test.context.ActiveProfiles import java.time.LocalDate import java.time.OffsetTime import java.time.ZonedDateTime -import java.util.UUID +import java.util.* @SpringBootTest @ActiveProfiles("test") @@ -447,7 +447,7 @@ class AggregatedQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { aggrPeriodDuration: String = "P1D" ): TemporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( + buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.AFTER, timeAt = now.minusHours(1), aggrPeriodDuration = aggrPeriodDuration, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt index 76b97031f6..ab8d7ce106 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt @@ -1,6 +1,7 @@ package com.egm.stellio.search.service import com.egm.stellio.search.model.* +import com.egm.stellio.search.model.TemporalQuery.Timerel import com.egm.stellio.search.support.* import com.egm.stellio.shared.model.ExpandedAttributes import com.egm.stellio.shared.model.ResourceNotFoundException @@ -39,7 +40,7 @@ import org.springframework.test.context.ActiveProfiles import java.time.Instant import java.time.ZoneOffset import java.time.ZonedDateTime -import java.util.UUID +import java.util.* @SpringBootTest @ActiveProfiles("test") @@ -58,7 +59,6 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer private lateinit var r2dbcEntityTemplate: R2dbcEntityTemplate private val now = Instant.now().atZone(ZoneOffset.UTC) - private lateinit var incomingTemporalEntityAttribute: TemporalEntityAttribute private lateinit var outgoingTemporalEntityAttribute: TemporalEntityAttribute private lateinit var jsonTemporalEntityAttribute: TemporalEntityAttribute @@ -152,8 +152,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.create(observation) val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) @@ -175,8 +175,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.create(observation) val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1), timeproperty = AttributeInstance.TemporalProperty.CREATED_AT ) @@ -199,8 +199,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.create(observation) val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1), timeproperty = AttributeInstance.TemporalProperty.CREATED_AT ) @@ -223,8 +223,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.create(observation) val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1), timeproperty = AttributeInstance.TemporalProperty.MODIFIED_AT ) @@ -243,8 +243,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) @@ -261,7 +261,10 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.create(gimmeNumericPropertyAttributeInstance(incomingTemporalEntityAttribute.id)) } - attributeInstanceService.search(gimmeTemporalEntitiesQuery(TemporalQuery()), incomingTemporalEntityAttribute) + attributeInstanceService.search( + gimmeTemporalEntitiesQuery(buildDefaultTestTemporalQuery()), + incomingTemporalEntityAttribute + ) .shouldSucceedWith { assertThat(it) .hasSize(10) @@ -303,7 +306,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } attributeInstanceService.search( - gimmeTemporalEntitiesQuery(TemporalQuery(), withTemporalValues = true), + gimmeTemporalEntitiesQuery(buildDefaultTestTemporalQuery(), withTemporalValues = true), temporalEntityAttribute2 ).shouldSucceedWith { results -> assertThat(results) @@ -327,7 +330,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.create(attributeInstance) } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( + buildDefaultTestTemporalQuery( aggrPeriodDuration = "P30D", aggrMethods = listOf(TemporalQuery.Aggregate.MAX) ), @@ -352,10 +355,11 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1), - lastN = 5 + limit = 5, + isChronological = false ) ) attributeInstanceService.search(temporalEntitiesQuery, incomingTemporalEntityAttribute) @@ -379,12 +383,13 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.BEFORE, + buildDefaultTestTemporalQuery( + timerel = Timerel.BEFORE, timeAt = now, aggrPeriodDuration = "PT1S", aggrMethods = listOf(TemporalQuery.Aggregate.SUM), - lastN = 5 + limit = 5, + isChronological = false ), withAggregatedValues = true ) @@ -415,8 +420,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) @@ -437,8 +442,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) @@ -458,8 +463,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.plusHours(1) ) ) @@ -481,8 +486,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.search( gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ), withTemporalValues = true @@ -629,8 +634,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) as ExpandedAttributes val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = ZonedDateTime.parse("1970-01-01T00:00:00Z") ) ) @@ -667,8 +672,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) as ExpandedAttributes val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = ZonedDateTime.parse("1970-01-01T00:00:00Z") ) ) @@ -706,8 +711,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) as ExpandedAttributes val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = ZonedDateTime.parse("1970-01-01T00:00:00Z") ) ) @@ -750,8 +755,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) as ExpandedAttributes val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = ZonedDateTime.parse("1970-01-01T00:00:00Z") ) ) @@ -791,7 +796,10 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstance.instanceId ).shouldSucceed() - attributeInstanceService.search(gimmeTemporalEntitiesQuery(TemporalQuery()), incomingTemporalEntityAttribute) + attributeInstanceService.search( + gimmeTemporalEntitiesQuery(buildDefaultTestTemporalQuery()), + incomingTemporalEntityAttribute + ) .shouldSucceedWith { assertThat(it) .isEmpty() @@ -860,8 +868,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.deleteInstancesOfEntity(listOf(incomingTemporalEntityAttribute.id)).shouldSucceed() val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) @@ -869,8 +877,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer .shouldSucceedWith { assertThat(it).isEmpty() } val temporalEntitiesAuditQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1), timeproperty = AttributeInstance.TemporalProperty.CREATED_AT ) @@ -896,8 +904,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.deleteAllInstancesOfAttribute(entityId, INCOMING_PROPERTY).shouldSucceed() val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) @@ -923,8 +931,8 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer attributeInstanceService.deleteInstancesOfAttribute(entityId, INCOMING_PROPERTY, null).shouldSucceed() val temporalEntitiesQuery = gimmeTemporalEntitiesQuery( - TemporalQuery( - timerel = TemporalQuery.Timerel.AFTER, + buildDefaultTestTemporalQuery( + timerel = Timerel.AFTER, timeAt = now.minusHours(1) ) ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt index 5c80ec8fc7..34e37ee9b6 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt @@ -5,10 +5,7 @@ import arrow.core.right import com.egm.stellio.search.model.* import com.egm.stellio.search.scope.ScopeInstanceResult import com.egm.stellio.search.scope.ScopeService -import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD -import com.egm.stellio.search.support.EMPTY_PAYLOAD -import com.egm.stellio.search.support.buildDefaultQueryParams -import com.egm.stellio.search.support.gimmeEntityPayload +import com.egm.stellio.search.support.* import com.egm.stellio.shared.model.PaginationQuery import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.* @@ -108,7 +105,7 @@ class QueryServiceTests { queryService.queryTemporalEntity( entityUri, TemporalEntitiesQuery( - temporalQuery = TemporalQuery( + temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.AFTER, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z") ), @@ -160,7 +157,7 @@ class QueryServiceTests { queryService.queryTemporalEntity( entityUri, TemporalEntitiesQuery( - temporalQuery = TemporalQuery( + temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.AFTER, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z") ), @@ -194,7 +191,7 @@ class QueryServiceTests { val origin = queryService.calculateOldestTimestamp( entityUri, TemporalEntitiesQuery( - temporalQuery = TemporalQuery(), + temporalQuery = buildDefaultTestTemporalQuery(), entitiesQuery = EntitiesQuery( paginationQuery = PaginationQuery(limit = 0, offset = 50), contexts = APIC_COMPOUND_CONTEXTS @@ -214,7 +211,7 @@ class QueryServiceTests { val origin = queryService.calculateOldestTimestamp( entityUri, TemporalEntitiesQuery( - temporalQuery = TemporalQuery( + temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.AFTER, timeAt = now ), @@ -245,7 +242,7 @@ class QueryServiceTests { val origin = queryService.calculateOldestTimestamp( entityUri, TemporalEntitiesQuery( - temporalQuery = TemporalQuery(), + temporalQuery = buildDefaultTestTemporalQuery(), entitiesQuery = EntitiesQuery( paginationQuery = PaginationQuery(limit = 0, offset = 50), contexts = APIC_COMPOUND_CONTEXTS @@ -296,7 +293,7 @@ class QueryServiceTests { paginationQuery = PaginationQuery(limit = 2, offset = 2), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.BEFORE, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z") ), @@ -364,7 +361,7 @@ class QueryServiceTests { paginationQuery = PaginationQuery(limit = 2, offset = 2), contexts = APIC_COMPOUND_CONTEXTS ), - TemporalQuery( + buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.BEFORE, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z"), aggrMethods = listOf(TemporalQuery.Aggregate.AVG) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt index 2b381827dd..b76d3774e7 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt @@ -1,6 +1,11 @@ package com.egm.stellio.search.support +import com.egm.stellio.search.model.AttributeInstance import com.egm.stellio.search.model.TemporalEntityAttribute +import com.egm.stellio.search.model.TemporalQuery +import com.egm.stellio.search.model.TemporalQuery.Aggregate +import com.egm.stellio.search.model.TemporalQuery.Timerel +import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.NgsiLdAttribute import com.egm.stellio.shared.model.toNgsiLdAttributes import com.egm.stellio.shared.util.AuthContextModel @@ -22,6 +27,39 @@ import io.r2dbc.postgresql.codec.Json import java.net.URI import java.time.ZonedDateTime +@SuppressWarnings("LongParameterList") +fun buildDefaultTestTemporalQuery( + timerel: Timerel? = null, + timeAt: ZonedDateTime? = null, + endTimeAt: ZonedDateTime? = null, + aggrPeriodDuration: String? = null, + aggrMethods: List? = null, + isChronological: Boolean = true, + limit: Int = 100, + timeproperty: AttributeInstance.TemporalProperty = AttributeInstance.TemporalProperty.OBSERVED_AT +) = TemporalQuery( + timerel = timerel, + timeAt = timeAt, + endTimeAt = endTimeAt, + aggrPeriodDuration = aggrPeriodDuration, + aggrMethods = aggrMethods, + isChronological = isChronological, + limit = limit, + timeproperty = timeproperty +) + +fun buildDefaultPagination( + limitDefault: Int = 50, + limitMax: Int = 100, + temporalLimitDefault: Int = 100, + temporalLimitMax: Int = 1000 +) = ApplicationProperties.Pagination( + limitDefault = limitDefault, + limitMax = limitMax, + temporalLimitDefault = temporalLimitDefault, + temporalLimitMax = temporalLimitMax, +) + fun buildAttributeInstancePayload( value: Any, observedAt: ZonedDateTime, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt index 6d9e89aada..90e0d80039 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt @@ -6,6 +6,8 @@ import com.egm.stellio.search.model.AttributeInstance import com.egm.stellio.search.model.EntitiesQuery import com.egm.stellio.search.model.Query import com.egm.stellio.search.model.TemporalQuery +import com.egm.stellio.search.support.buildDefaultPagination +import com.egm.stellio.search.support.buildDefaultTestTemporalQuery import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException @@ -31,7 +33,7 @@ class EntitiesQueryUtilsTests { fun `it should parse query parameters`() = runTest { val requestParams = gimmeEntitiesQueryParams() val entitiesQuery = composeEntitiesQuery( - ApplicationProperties.Pagination(1, 20), + buildDefaultPagination(1, 20), requestParams, APIC_COMPOUND_CONTEXTS ).shouldSucceedAndResult() @@ -54,7 +56,7 @@ class EntitiesQueryUtilsTests { val requestParams = LinkedMultiValueMap() requestParams.add("q", "speed%3E50%3BfoodName%3D%3Ddietary+fibres") val entitiesQuery = composeEntitiesQuery( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), requestParams, NGSILD_TEST_CORE_CONTEXTS ).shouldSucceedAndResult() @@ -66,7 +68,7 @@ class EntitiesQueryUtilsTests { fun `it should set default values in query parameters`() = runTest { val requestParams = LinkedMultiValueMap() val entitiesQuery = composeEntitiesQuery( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), requestParams, NGSILD_TEST_CORE_CONTEXTS ).shouldSucceedAndResult() @@ -125,7 +127,7 @@ class EntitiesQueryUtilsTests { """.trimIndent() composeEntitiesQueryFromPostRequest( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), query, LinkedMultiValueMap(), APIC_COMPOUND_CONTEXTS @@ -157,7 +159,7 @@ class EntitiesQueryUtilsTests { """.trimIndent() composeEntitiesQueryFromPostRequest( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), query, LinkedMultiValueMap(), APIC_COMPOUND_CONTEXTS @@ -178,7 +180,7 @@ class EntitiesQueryUtilsTests { """.trimIndent() composeEntitiesQueryFromPostRequest( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), query, LinkedMultiValueMap(), APIC_COMPOUND_CONTEXTS @@ -198,7 +200,7 @@ class EntitiesQueryUtilsTests { """.trimIndent() composeEntitiesQueryFromPostRequest( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), query, LinkedMultiValueMap(), APIC_COMPOUND_CONTEXTS @@ -218,7 +220,7 @@ class EntitiesQueryUtilsTests { """.trimIndent() composeEntitiesQueryFromPostRequest( - ApplicationProperties.Pagination(30, 100), + buildDefaultPagination(30, 100), query, LinkedMultiValueMap(), APIC_COMPOUND_CONTEXTS @@ -302,6 +304,8 @@ class EntitiesQueryUtilsTests { val pagination = mockkClass(ApplicationProperties.Pagination::class) every { pagination.limitDefault } returns 30 every { pagination.limitMax } returns 100 + every { pagination.temporalLimitDefault } returns 100 + every { pagination.temporalLimitMax } returns 1000 val temporalEntitiesQuery = composeTemporalEntitiesQuery(pagination, queryParams, APIC_COMPOUND_CONTEXTS).shouldSucceedAndResult() @@ -313,7 +317,7 @@ class EntitiesQueryUtilsTests { assertEquals("$BEEHIVE_TYPE,$APIARY_TYPE", temporalEntitiesQuery.entitiesQuery.typeSelection) assertEquals(setOf(INCOMING_PROPERTY, OUTGOING_PROPERTY), temporalEntitiesQuery.entitiesQuery.attrs) assertEquals( - TemporalQuery( + buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.BETWEEN, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z"), endTimeAt = ZonedDateTime.parse("2019-10-18T07:31:39Z") @@ -335,6 +339,8 @@ class EntitiesQueryUtilsTests { val pagination = mockkClass(ApplicationProperties.Pagination::class) every { pagination.limitDefault } returns 30 every { pagination.limitMax } returns 100 + every { pagination.temporalLimitDefault } returns 100 + every { pagination.temporalLimitMax } returns 1000 val temporalEntitiesQuery = composeTemporalEntitiesQuery(pagination, queryParams, APIC_COMPOUND_CONTEXTS).shouldSucceedAndResult() @@ -362,6 +368,8 @@ class EntitiesQueryUtilsTests { val pagination = mockkClass(ApplicationProperties.Pagination::class) every { pagination.limitDefault } returns 30 every { pagination.limitMax } returns 100 + every { pagination.temporalLimitDefault } returns 100 + every { pagination.temporalLimitMax } returns 1000 val queryParams = LinkedMultiValueMap() queryParams.add("timerel", "after") @@ -380,6 +388,8 @@ class EntitiesQueryUtilsTests { val pagination = mockkClass(ApplicationProperties.Pagination::class) every { pagination.limitDefault } returns 30 every { pagination.limitMax } returns 100 + every { pagination.temporalLimitDefault } returns 100 + every { pagination.temporalLimitMax } returns 1000 val queryParams = LinkedMultiValueMap() queryParams.add("timerel", "after") @@ -398,6 +408,8 @@ class EntitiesQueryUtilsTests { val pagination = mockkClass(ApplicationProperties.Pagination::class) every { pagination.limitDefault } returns 30 every { pagination.limitMax } returns 100 + every { pagination.temporalLimitDefault } returns 100 + every { pagination.temporalLimitMax } returns 1000 val queryParams = LinkedMultiValueMap() queryParams.add("timerel", "after") @@ -415,9 +427,10 @@ class EntitiesQueryUtilsTests { queryParams.add("timeAt", "2019-10-17T07:31:39Z") queryParams.add("lastN", "2") - val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, buildDefaultPagination()).shouldSucceedAndResult() - assertEquals(2, temporalQuery.lastN) + assertEquals(2, temporalQuery.limit) + assertFalse(temporalQuery.isChronological) } @Test @@ -426,10 +439,11 @@ class EntitiesQueryUtilsTests { queryParams.add("timerel", "after") queryParams.add("timeAt", "2019-10-17T07:31:39Z") queryParams.add("lastN", "A") + val pagination = buildDefaultPagination() + val temporalQuery = buildTemporalQuery(queryParams, pagination).shouldSucceedAndResult() - val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() - - assertNull(temporalQuery.lastN) + assertEquals(pagination.temporalLimitDefault, temporalQuery.limit) + assertTrue(temporalQuery.isChronological) } @Test @@ -438,17 +452,19 @@ class EntitiesQueryUtilsTests { queryParams.add("timerel", "after") queryParams.add("timeAt", "2019-10-17T07:31:39Z") queryParams.add("lastN", "-2") + val pagination = buildDefaultPagination() - val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, pagination).shouldSucceedAndResult() - assertNull(temporalQuery.lastN) + assertEquals(pagination.temporalLimitDefault, temporalQuery.limit) + assertTrue(temporalQuery.isChronological) } @Test fun `it should treat time and timerel properties as optional in a temporal query`() = runTest { val queryParams = LinkedMultiValueMap() - val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, buildDefaultPagination()).shouldSucceedAndResult() assertNull(temporalQuery.timeAt) assertNull(temporalQuery.timerel) @@ -459,7 +475,7 @@ class EntitiesQueryUtilsTests { val queryParams = LinkedMultiValueMap() queryParams.add("timeproperty", "createdAt") - val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, buildDefaultPagination()).shouldSucceedAndResult() assertEquals(AttributeInstance.TemporalProperty.CREATED_AT, temporalQuery.timeproperty) } @@ -468,7 +484,7 @@ class EntitiesQueryUtilsTests { fun `it should set timeproperty to observedAt if no value is provided in query parameters`() = runTest { val queryParams = LinkedMultiValueMap() - val temporalQuery = buildTemporalQuery(queryParams, 100).shouldSucceedAndResult() + val temporalQuery = buildTemporalQuery(queryParams, buildDefaultPagination()).shouldSucceedAndResult() assertEquals(AttributeInstance.TemporalProperty.OBSERVED_AT, temporalQuery.timeproperty) } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt index 7933511b41..a782a3f283 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt @@ -5,6 +5,7 @@ import com.egm.stellio.search.model.AggregatedAttributeInstanceResult.AggregateR import com.egm.stellio.search.scope.ScopeInstanceResult import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD import com.egm.stellio.search.support.buildDefaultQueryParams +import com.egm.stellio.search.support.buildDefaultTestTemporalQuery import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_PROPERTY import com.egm.stellio.shared.util.JsonUtils.serializeObject @@ -43,7 +44,7 @@ class TemporalEntityBuilderTests { EntityTemporalResult(entityPayload, emptyList(), attributeAndResultsMap), TemporalEntitiesQuery( entitiesQuery = buildDefaultQueryParams(), - temporalQuery = TemporalQuery(), + temporalQuery = buildDefaultTestTemporalQuery(), withTemporalValues = false, withAudit = false, withAggregatedValues = false @@ -76,7 +77,7 @@ class TemporalEntityBuilderTests { EntityTemporalResult(entityPayload, scopeHistory, attributeAndResultsMap), TemporalEntitiesQuery( entitiesQuery = buildDefaultQueryParams(), - temporalQuery = TemporalQuery(), + temporalQuery = buildDefaultTestTemporalQuery(), withTemporalValues, withAudit, false @@ -101,7 +102,7 @@ class TemporalEntityBuilderTests { entityTemporalResults, TemporalEntitiesQuery( entitiesQuery = buildDefaultQueryParams(), - temporalQuery = TemporalQuery(), + temporalQuery = buildDefaultTestTemporalQuery(), withTemporalValues, withAudit, false @@ -161,7 +162,7 @@ class TemporalEntityBuilderTests { ) ) ) - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( TemporalQuery.Timerel.AFTER, Instant.now().atZone(ZoneOffset.UTC).minusHours(1), null, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index b22e681483..dc6bab3095 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -14,6 +14,7 @@ import com.egm.stellio.search.service.EntityPayloadService import com.egm.stellio.search.service.QueryService import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD +import com.egm.stellio.search.support.buildDefaultTestTemporalQuery import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.* import com.egm.stellio.shared.util.* @@ -39,7 +40,7 @@ import org.springframework.test.web.reactive.server.WebTestClient import org.springframework.web.reactive.function.BodyInserters import java.time.ZoneOffset import java.time.ZonedDateTime -import java.util.UUID +import java.util.* @ActiveProfiles("test") @WebFluxTest(TemporalEntityHandler::class) @@ -758,7 +759,7 @@ class TemporalEntityHandlerTests { @Test fun `it should return a 200 with empty payload if no temporal attribute is found`() { - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.BETWEEN, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z"), endTimeAt = ZonedDateTime.parse("2019-10-18T07:31:39Z") diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt index 8e7884ca6e..22a81652d7 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt @@ -5,6 +5,7 @@ import com.egm.stellio.search.authorization.AuthorizationService import com.egm.stellio.search.config.SearchProperties import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.service.QueryService +import com.egm.stellio.search.support.buildDefaultTestTemporalQuery import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.util.* import com.ninjasquad.springmockk.MockkBean @@ -50,7 +51,7 @@ class TemporalEntityOperationsHandlerTests { @Test fun `it should return a 200 and retrieve requested temporal attributes`() { - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.BETWEEN, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z"), endTimeAt = ZonedDateTime.parse("2019-10-18T07:31:39Z") @@ -99,7 +100,7 @@ class TemporalEntityOperationsHandlerTests { @Test fun `it should return a 200 and the number of results if count is asked for`() { - val temporalQuery = TemporalQuery( + val temporalQuery = buildDefaultTestTemporalQuery( timerel = TemporalQuery.Timerel.BETWEEN, timeAt = ZonedDateTime.parse("2019-10-17T07:31:39Z"), endTimeAt = ZonedDateTime.parse("2019-10-18T07:31:39Z") From 2663ebab7d5b74f0064523f0601e0200eb15c52d Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 12 Jun 2024 09:30:19 +0200 Subject: [PATCH 10/14] feat: fix detekt rules --- .../egm/stellio/search/scope/ScopeService.kt | 10 +++++----- .../service/AttributeInstanceService.kt | 4 ++-- .../stellio/search/util/EntitiesQueryUtils.kt | 1 + .../search/web/BuildTemporalApiResponse.kt | 20 +++++++++---------- .../search/util/TemporalEntityBuilderTests.kt | 1 + 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index 47305e254a..efc0d21ebd 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -115,8 +115,8 @@ class ScopeService( else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY entity_id") if (temporalQuery.isChronological == true) - // in order to get first or last instances, need to order by time - // final ascending ordering of instances is done in query service + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service sqlQueryBuilder.append(" ORDER BY start ASC") else sqlQueryBuilder.append(" ORDER BY start DESC") @@ -278,9 +278,9 @@ class ScopeService( else TemporalProperty.MODIFIED_AT addHistoryEntry(entityId, it, temporalPropertyToAdd, modifiedAt, sub).bind() if (temporalPropertyToAdd == TemporalProperty.MODIFIED_AT) - // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a - // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt - // sub-Property + // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a + // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt + // sub-Property addHistoryEntry(entityId, it, TemporalProperty.OBSERVED_AT, modifiedAt, sub).bind() updateResult } ?: UpdateResult( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index 795696c5d1..abc8931909 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -163,8 +163,8 @@ class AttributeInstanceService( sqlQueryBuilder.append(" GROUP BY temporal_entity_attribute") if (temporalQuery.isChronological == true) - // in order to get first or last instances, need to order by time - // final ascending ordering of instances is done in query service + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service sqlQueryBuilder.append(" ORDER BY start ASC") else sqlQueryBuilder.append(" ORDER BY start DESC") diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt index a6336ebd1e..c68215819b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt @@ -194,6 +194,7 @@ fun composeTemporalEntitiesQueryFromPostRequest( ) } +@SuppressWarnings("ReturnCount") fun buildTemporalQuery( params: MultiValueMap, pagination: ApplicationProperties.Pagination, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt index a485f087d3..09efd8707f 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt @@ -16,10 +16,10 @@ import org.springframework.http.ResponseEntity import org.springframework.util.MultiValueMap import java.time.ZonedDateTime -typealias CompactedTemporalAttributes = List>> +typealias CompactedTemporalAttribute = List> object TemporalApiResponse { - + @SuppressWarnings("LongParameterList") fun buildListTemporalResponse( entities: List, total: Int, @@ -91,31 +91,31 @@ object TemporalApiResponse { } private fun getAttributesWhoReachedLimit(entities: List, query: TemporalEntitiesQuery): - CompactedTemporalAttributes { + List { val temporalQuery = query.temporalQuery return entities.flatMap { it.values.mapNotNull { - if (it is List<*> && it.size >= temporalQuery.limit) it as List> else null + if (it is List<*> && it.size >= temporalQuery.limit) it as CompactedTemporalAttribute else null } } } private fun getTemporalPaginationRange( - attributesWhoReachedLimit: CompactedTemporalAttributes, + attributesWhoReachedLimit: List, query: TemporalEntitiesQuery ): String { val temporalQuery = query.temporalQuery - val lastN = temporalQuery.limit - val maxSize = lastN + val limit = temporalQuery.limit + val isChronological = temporalQuery.isChronological val timeProperty = temporalQuery.timeproperty.propertyName val attributesTimeRanges = attributesWhoReachedLimit.map { attribute -> attribute.map { it[timeProperty] } } .map { ZonedDateTime.parse(it.getOrNull(0) as String) to - ZonedDateTime.parse(it.getOrNull(maxSize - 1) as String) + ZonedDateTime.parse(it.getOrNull(limit - 1) as String) } - val range = if (lastN == null) { + val range = if (isChronological) { val discriminatingTimeRange = attributesTimeRanges.minBy { it.second } val rangeStart = when (temporalQuery.timerel) { TemporalQuery.Timerel.AFTER -> temporalQuery.timeAt @@ -135,7 +135,7 @@ object TemporalApiResponse { rangeStart to discriminatingTimeRange.second } - val size = lastN.toString() + val size = limit.toString() return "DateTime ${range.first?.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt index a782a3f283..4077a2a107 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/util/TemporalEntityBuilderTests.kt @@ -115,6 +115,7 @@ class TemporalEntityBuilderTests { ) } + @SuppressWarnings("LongMethod") @Test fun `it should return a temporal entity with values aggregated`() { val temporalEntityAttribute = TemporalEntityAttribute( From cc4947504740558181fafeccf5bdaeb810611555 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 12 Jun 2024 09:42:16 +0200 Subject: [PATCH 11/14] feat: fix useless equals with non nullable isChronological --- .../com/egm/stellio/search/scope/ScopeService.kt | 12 ++++++------ .../search/service/AttributeInstanceService.kt | 2 +- .../subscription/support/WithMosquittoContainer.kt | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index efc0d21ebd..ed9797ecd8 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -114,9 +114,9 @@ class ScopeService( sqlQueryBuilder.append(" GROUP BY entity_id, start") else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY entity_id") - if (temporalQuery.isChronological == true) - // in order to get first or last instances, need to order by time - // final ascending ordering of instances is done in query service + if (temporalQuery.isChronological) + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service sqlQueryBuilder.append(" ORDER BY start ASC") else sqlQueryBuilder.append(" ORDER BY start DESC") @@ -278,9 +278,9 @@ class ScopeService( else TemporalProperty.MODIFIED_AT addHistoryEntry(entityId, it, temporalPropertyToAdd, modifiedAt, sub).bind() if (temporalPropertyToAdd == TemporalProperty.MODIFIED_AT) - // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a - // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt - // sub-Property + // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a + // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt + // sub-Property addHistoryEntry(entityId, it, TemporalProperty.OBSERVED_AT, modifiedAt, sub).bind() updateResult } ?: UpdateResult( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index abc8931909..e1e28be4f3 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -162,7 +162,7 @@ class AttributeInstanceService( else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY temporal_entity_attribute") - if (temporalQuery.isChronological == true) + if (temporalQuery.isChronological) // in order to get first or last instances, need to order by time // final ascending ordering of instances is done in query service sqlQueryBuilder.append(" ORDER BY start ASC") diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt index e661cd7b22..fcef13bd45 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithMosquittoContainer.kt @@ -28,7 +28,5 @@ interface WithMosquittoContainer { init { mosquittoContainer.start() } - - fun getMosquittoLogs(): String = mosquittoContainer.logs } } From 48cd8167c208db58ef39ee22b66157502432ec87 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 12 Jun 2024 17:07:23 +0200 Subject: [PATCH 12/14] feat: filter entity not in response range --- .../search/web/BuildTemporalApiResponse.kt | 79 +++++++++++++++---- .../stellio/shared/model/CompactedEntity.kt | 4 +- 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt index 09efd8707f..6c0fc9249a 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt @@ -2,6 +2,7 @@ package com.egm.stellio.search.web import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalQuery +import com.egm.stellio.shared.model.CompactedAttribute import com.egm.stellio.shared.model.CompactedEntity import com.egm.stellio.shared.model.toFinalRepresentation import com.egm.stellio.shared.util.JsonUtils.serializeObject @@ -18,6 +19,8 @@ import java.time.ZonedDateTime typealias CompactedTemporalAttribute = List> +typealias Range = Pair + object TemporalApiResponse { @SuppressWarnings("LongParameterList") fun buildListTemporalResponse( @@ -46,15 +49,18 @@ object TemporalApiResponse { return successResponse } + val range = getTemporalPaginationRange(attributesWhoReachedLimit, query) + + val filteredEntities = entities.map { filterEntityInRange(it, range, query.temporalQuery) } return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).apply { this.headers( successResponse.headers ) this.header( HttpHeaders.CONTENT_RANGE, - getTemporalPaginationRange(attributesWhoReachedLimit, query) + getHeaderRange(range, query.temporalQuery) ) - }.build() + }.body(serializeObject(filteredEntities.toFinalRepresentation(ngsiLdDataRepresentation))) } fun buildEntityTemporalResponse( @@ -66,35 +72,42 @@ object TemporalApiResponse { ): ResponseEntity { val ngsiLdDataRepresentation = parseRepresentations(requestParams, mediaType) + val attributesWhoReachedLimit = getAttributesWhoReachedLimit(listOf(entity), query) + val successResponse = prepareGetSuccessResponseHeaders(mediaType, contexts).body( serializeObject( entity.toFinalRepresentation(ngsiLdDataRepresentation) ) ) - val attributesWhoReachedLimit = getAttributesWhoReachedLimit(listOf(entity), query) - if (attributesWhoReachedLimit.isEmpty()) { return successResponse } + val range = getTemporalPaginationRange(attributesWhoReachedLimit, query) + val filteredEntity = filterEntityInRange(entity, range, query.temporalQuery) + return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT) .apply { this.header( HttpHeaders.CONTENT_RANGE, - getTemporalPaginationRange(attributesWhoReachedLimit, query) + getHeaderRange(range, query.temporalQuery) ) this.headers( successResponse.headers ) - }.build() + }.body( + serializeObject( + filteredEntity.toFinalRepresentation(ngsiLdDataRepresentation) + ) + ) } private fun getAttributesWhoReachedLimit(entities: List, query: TemporalEntitiesQuery): List { val temporalQuery = query.temporalQuery - return entities.flatMap { - it.values.mapNotNull { + return entities.flatMap { attr -> + attr.values.mapNotNull { if (it is List<*> && it.size >= temporalQuery.limit) it as CompactedTemporalAttribute else null } } @@ -103,7 +116,7 @@ object TemporalApiResponse { private fun getTemporalPaginationRange( attributesWhoReachedLimit: List, query: TemporalEntitiesQuery - ): String { + ): Range { val temporalQuery = query.temporalQuery val limit = temporalQuery.limit val isChronological = temporalQuery.isChronological @@ -115,11 +128,11 @@ object TemporalApiResponse { ZonedDateTime.parse(it.getOrNull(limit - 1) as String) } - val range = if (isChronological) { + return if (isChronological) { val discriminatingTimeRange = attributesTimeRanges.minBy { it.second } val rangeStart = when (temporalQuery.timerel) { - TemporalQuery.Timerel.AFTER -> temporalQuery.timeAt - TemporalQuery.Timerel.BETWEEN -> temporalQuery.timeAt + TemporalQuery.Timerel.AFTER -> temporalQuery.timeAt ?: discriminatingTimeRange.first + TemporalQuery.Timerel.BETWEEN -> temporalQuery.timeAt ?: discriminatingTimeRange.first else -> discriminatingTimeRange.first } @@ -127,15 +140,49 @@ object TemporalApiResponse { } else { val discriminatingTimeRange = attributesTimeRanges.maxBy { it.second } val rangeStart = when (temporalQuery.timerel) { - TemporalQuery.Timerel.BEFORE -> temporalQuery.timeAt - TemporalQuery.Timerel.BETWEEN -> temporalQuery.endTimeAt + TemporalQuery.Timerel.BEFORE -> temporalQuery.timeAt ?: discriminatingTimeRange.first + TemporalQuery.Timerel.BETWEEN -> temporalQuery.endTimeAt ?: discriminatingTimeRange.first else -> discriminatingTimeRange.first } rangeStart to discriminatingTimeRange.second } + } + + private fun filterEntityInRange(entity: CompactedEntity, range: Range, query: TemporalQuery): CompactedEntity { + return entity.keys.mapNotNull { key -> + when (val attribute = entity[key]) { + is List<*> -> { + val temporalAttribute = attribute as CompactedTemporalAttribute + // faster filter possible because list is already order + val filteredAttribute = temporalAttribute.filter { range.contain(it, query) } + if (filteredAttribute.isNotEmpty()) key to filteredAttribute else null + } + + is Map<*, *> -> { + val compactAttribute: CompactedAttribute = attribute as CompactedAttribute + if (range.contain(compactAttribute, query)) key to attribute + else key to emptyList() + } + + else -> key to (attribute ?: return@mapNotNull null) + } + }.toMap() + } + + private fun Range.contain(time: ZonedDateTime) = + (this.first > time && time > this.second) || (this.first < time && time < this.second) + + private fun Range.contain(attribute: CompactedAttribute, query: TemporalQuery) = this.contain( + attribute.getAttributeDate(query) + ) + + private fun CompactedAttribute.getAttributeDate(query: TemporalQuery) = ZonedDateTime.parse( + this[query.timeproperty.propertyName] as String + ) - val size = limit.toString() - return "DateTime ${range.first?.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" + private fun getHeaderRange(range: Range, temporalQuery: TemporalQuery): String { + val size = if (temporalQuery.isChronological) "*" else temporalQuery.limit + return "DateTime ${range.first.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" } } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt index d9dd982169..a15ad4bdcb 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt @@ -21,9 +21,11 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SYSATTRS_TERMS import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_TERM -import java.util.Locale +import java.util.* typealias CompactedEntity = Map +typealias CompactedAttribute = Map + fun CompactedEntity.toSimplifiedAttributes(): Map = this.mapValues { (_, value) -> From f5295e249fd2ad48c45c652321edbe09992ff8c6 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Mon, 17 Jun 2024 09:55:38 +0200 Subject: [PATCH 13/14] wip --- .../egm/stellio/search/model/TemporalQuery.kt | 11 +- .../egm/stellio/search/scope/ScopeService.kt | 7 +- .../service/AttributeInstanceService.kt | 20 ++- .../stellio/search/service/QueryService.kt | 3 +- .../stellio/search/util/EntitiesQueryUtils.kt | 48 +++--- .../search/web/BuildTemporalApiResponse.kt | 49 +++--- .../egm/stellio/search/support/TestUtils.kt | 6 +- .../search/util/EntitiesQueryUtilsTests.kt | 6 +- .../search/web/TemporalEntityHandlerTests.kt | 20 +-- .../TemporalEntityHandlerpaginationTests.kt | 157 ++++++++++++++++++ .../resources/application-test.properties | 2 + ...nized_temporal_attributes_evolution.jsonld | 102 ++++++++++++ ...ttributes_evolution_temporal_values.jsonld | 37 +++++ .../com/egm/stellio/shared/util/ApiUtils.kt | 4 +- .../com/egm/stellio/shared/util/DateUtils.kt | 2 +- 15 files changed, 391 insertions(+), 83 deletions(-) create mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt create mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld create mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt index b6a478e1f6..31ae6b75cd 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalQuery.kt @@ -4,16 +4,25 @@ import java.time.ZonedDateTime const val WHOLE_TIME_RANGE_DURATION = "PT0S" +const val TIMEREL_PARAM = "timerel" // do i put them in TemporalQuery companion object? +const val TIMEAT_PARAM = "timeAt" +const val ENDTIMEAT_PARAM = "endTimeAt" +const val AGGRPERIODDURATION_PARAM = "aggrPeriodDuration" +const val AGGRMETHODS_PARAM = "aggrMethods" +const val LASTN_PARAM = "lastN" +const val TIMEPROPERTY_PARAM = "timeproperty" + data class TemporalQuery( val timerel: Timerel? = null, val timeAt: ZonedDateTime? = null, val endTimeAt: ZonedDateTime? = null, val aggrPeriodDuration: String? = null, val aggrMethods: List? = null, - val isChronological: Boolean, + val asLastN: Boolean, val limit: Int, val timeproperty: AttributeInstance.TemporalProperty = AttributeInstance.TemporalProperty.OBSERVED_AT ) { + enum class Timerel { BEFORE, AFTER, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index ed9797ecd8..8586ff706f 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -114,11 +114,12 @@ class ScopeService( sqlQueryBuilder.append(" GROUP BY entity_id, start") else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY entity_id") - if (temporalQuery.isChronological) + if (temporalQuery.asLastN) // in order to get first or last instances, need to order by time // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start ASC") - else sqlQueryBuilder.append(" ORDER BY start DESC") + sqlQueryBuilder.append(" ORDER BY start DESC") + else sqlQueryBuilder.append(" ORDER BY start ASC") + sqlQueryBuilder.append(" LIMIT $limit") diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index e1e28be4f3..7d7b501d85 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -7,6 +7,7 @@ import arrow.core.right import arrow.fx.coroutines.parMap import com.egm.stellio.search.model.* import com.egm.stellio.search.model.AggregatedAttributeInstanceResult.AggregateResult +import com.egm.stellio.search.model.TemporalQuery.Timerel import com.egm.stellio.search.util.* import com.egm.stellio.shared.model.* import com.egm.stellio.shared.util.INCONSISTENT_VALUES_IN_AGGREGATION_MESSAGE @@ -111,7 +112,7 @@ class AttributeInstanceService( return create(attributeInstance) } - suspend fun search( + suspend fun search( // que pour les tests? temporalEntitiesQuery: TemporalEntitiesQuery, temporalEntityAttribute: TemporalEntityAttribute, origin: ZonedDateTime? = null @@ -148,9 +149,9 @@ class AttributeInstanceService( ) when (temporalQuery.timerel) { - TemporalQuery.Timerel.BEFORE -> sqlQueryBuilder.append(" AND time < '${temporalQuery.timeAt}'") - TemporalQuery.Timerel.AFTER -> sqlQueryBuilder.append(" AND time > '${temporalQuery.timeAt}'") - TemporalQuery.Timerel.BETWEEN -> sqlQueryBuilder.append( + Timerel.BEFORE -> sqlQueryBuilder.append(" AND time < '${temporalQuery.timeAt}'") + Timerel.AFTER -> sqlQueryBuilder.append(" AND time > '${temporalQuery.timeAt}'") + Timerel.BETWEEN -> sqlQueryBuilder.append( " AND time > '${temporalQuery.timeAt}' AND time < '${temporalQuery.endTimeAt}'" ) @@ -162,11 +163,11 @@ class AttributeInstanceService( else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY temporal_entity_attribute") - if (temporalQuery.isChronological) - // in order to get first or last instances, need to order by time - // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start ASC") - else sqlQueryBuilder.append(" ORDER BY start DESC") + if (temporalQuery.asLastN) + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service + sqlQueryBuilder.append(" ORDER BY start DESC") + else sqlQueryBuilder.append(" ORDER BY start ASC") sqlQueryBuilder.append(" LIMIT ${temporalQuery.limit}") @@ -200,6 +201,7 @@ class AttributeInstanceService( JOIN LATERAL ( $aiLateralQuery ) ai_limited ON true; + """.trimIndent() } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt index 9c57ab62f4..41a69a2a05 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt @@ -185,7 +185,8 @@ class QueryService( .groupBy { it.attributeValueType }.mapValues { - attributeInstanceService.search(temporalEntitiesQuery, it.value, origin).bind() + attributeInstanceService.search(temporalEntitiesQuery, temporalEntityAttributes = it.value, origin) + .bind() } .mapValues { // when retrieved from DB, values of geo-properties are encoded in WKT and won't be automatically diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt index c68215819b..b5e3d98f76 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt @@ -3,6 +3,7 @@ package com.egm.stellio.search.util import arrow.core.* import arrow.core.raise.either import com.egm.stellio.search.model.* +import com.egm.stellio.search.model.TemporalQuery.* import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException @@ -170,13 +171,13 @@ fun composeTemporalEntitiesQueryFromPostRequest( ) val temporalParams = mapOf( - "timerel" to listOf(query.temporalQ?.timerel), - "timeAt" to listOf(query.temporalQ?.timeAt), - "endTimeAt" to listOf(query.temporalQ?.endTimeAt), - "aggrPeriodDuration" to listOf(query.temporalQ?.aggrPeriodDuration), - "aggrMethods" to query.temporalQ?.aggrMethods, - "lastN" to listOf(query.temporalQ?.lastN.toString()), - "timeproperty" to listOf(query.temporalQ?.timeproperty) + TIMEREL_PARAM to listOf(query.temporalQ?.timerel), + TIMEAT_PARAM to listOf(query.temporalQ?.timeAt), + ENDTIMEAT_PARAM to listOf(query.temporalQ?.endTimeAt), + AGGRPERIODDURATION_PARAM to listOf(query.temporalQ?.aggrPeriodDuration), + AGGRMETHODS_PARAM to query.temporalQ?.aggrMethods, + LASTN_PARAM to listOf(query.temporalQ?.lastN.toString()), + TIMEPROPERTY_PARAM to listOf(query.temporalQ?.timeproperty) ) val temporalQuery = buildTemporalQuery( MultiValueMapAdapter(temporalParams), @@ -201,22 +202,19 @@ fun buildTemporalQuery( inQueryEntities: Boolean = false, withAggregatedValues: Boolean = false, ): Either { - val timerelParam = params.getFirst("timerel") - val timeAtParam = params.getFirst("timeAt") - val endTimeAtParam = params.getFirst("endTimeAt") + val timerelParam = params.getFirst(TIMEREL_PARAM) + val timeAtParam = params.getFirst(TIMEAT_PARAM) + val endTimeAtParam = params.getFirst(ENDTIMEAT_PARAM) val aggrPeriodDurationParam = if (withAggregatedValues) - params.getFirst("aggrPeriodDuration") ?: "PT0S" + params.getFirst(AGGRPERIODDURATION_PARAM) ?: WHOLE_TIME_RANGE_DURATION else null - val aggrMethodsParam = params.getFirst("aggrMethods") - val lastNParam = params.getFirst("lastN") - val timeproperty = params.getFirst("timeproperty")?.let { + val aggrMethodsParam = params.getFirst(AGGRMETHODS_PARAM) + val lastNParam = params.getFirst(LASTN_PARAM) + val timeproperty = params.getFirst(TIMEPROPERTY_PARAM)?.let { AttributeInstance.TemporalProperty.forPropertyName(it) } ?: AttributeInstance.TemporalProperty.OBSERVED_AT - if (timerelParam == "between" && endTimeAtParam == null) - return BadRequestDataException("'endTimeAt' request parameter is mandatory if 'timerel' is 'between'").left() - val endTimeAt = endTimeAtParam?.parseTimeParameter("'endTimeAt' parameter is not a valid date") ?.getOrElse { return BadRequestDataException(it).left() @@ -225,12 +223,16 @@ fun buildTemporalQuery( val (timerel, timeAt) = buildTimerelAndTime(timerelParam, timeAtParam, inQueryEntities).getOrElse { return BadRequestDataException(it).left() } + + if (timerel == Timerel.BETWEEN && endTimeAtParam == null) + return BadRequestDataException("'endTimeAt' request parameter is mandatory if 'timerel' is 'between'").left() + if (withAggregatedValues && aggrMethodsParam == null) return BadRequestDataException("'aggrMethods' is mandatory if 'aggregatedValues' option is specified").left() val aggregate = aggrMethodsParam?.split(",")?.map { - if (TemporalQuery.Aggregate.isSupportedAggregate(it)) - TemporalQuery.Aggregate.forMethod(it)!! + if (Aggregate.isSupportedAggregate(it)) + Aggregate.forMethod(it)!! else return BadRequestDataException( "'$it' is not a recognized aggregation method for 'aggrMethods' parameter" @@ -247,7 +249,7 @@ fun buildTemporalQuery( else null } val limit = lastN ?: pagination.temporalLimitDefault - val isChronological = lastN == null + val asLastN = lastN != null return TemporalQuery( timerel = timerel, @@ -256,7 +258,7 @@ fun buildTemporalQuery( aggrPeriodDuration = aggrPeriodDurationParam, aggrMethods = aggregate, limit = limit, - isChronological = isChronological, + asLastN = asLastN, timeproperty = timeproperty ).right() } @@ -265,13 +267,13 @@ fun buildTimerelAndTime( timerelParam: String?, timeAtParam: String?, inQueryEntities: Boolean -): Either> = +): Either> = // when querying a specific temporal entity, timeAt and timerel are optional if (timerelParam == null && timeAtParam == null && !inQueryEntities) { Pair(null, null).right() } else if (timerelParam != null && timeAtParam != null) { val timeRelResult = try { - TemporalQuery.Timerel.valueOf(timerelParam.uppercase()).right() + Timerel.valueOf(timerelParam.uppercase()).right() } catch (e: IllegalArgumentException) { "'timerel' is not valid, it should be one of 'before', 'between', or 'after'".left() } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt index 6c0fc9249a..bf1943fd14 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt @@ -2,6 +2,7 @@ package com.egm.stellio.search.web import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalQuery +import com.egm.stellio.search.model.Timerel import com.egm.stellio.shared.model.CompactedAttribute import com.egm.stellio.shared.model.CompactedEntity import com.egm.stellio.shared.model.toFinalRepresentation @@ -72,14 +73,18 @@ object TemporalApiResponse { ): ResponseEntity { val ngsiLdDataRepresentation = parseRepresentations(requestParams, mediaType) - val attributesWhoReachedLimit = getAttributesWhoReachedLimit(listOf(entity), query) - val successResponse = prepareGetSuccessResponseHeaders(mediaType, contexts).body( serializeObject( entity.toFinalRepresentation(ngsiLdDataRepresentation) ) ) + if (query.temporalQuery.asLastN) { // if lastN > limit it throw an error earlier + return successResponse + } + + val attributesWhoReachedLimit = getAttributesWhoReachedLimit(listOf(entity), query) + if (attributesWhoReachedLimit.isEmpty()) { return successResponse } @@ -119,7 +124,6 @@ object TemporalApiResponse { ): Range { val temporalQuery = query.temporalQuery val limit = temporalQuery.limit - val isChronological = temporalQuery.isChronological val timeProperty = temporalQuery.timeproperty.propertyName val attributesTimeRanges = attributesWhoReachedLimit.map { attribute -> attribute.map { it[timeProperty] } } @@ -128,25 +132,14 @@ object TemporalApiResponse { ZonedDateTime.parse(it.getOrNull(limit - 1) as String) } - return if (isChronological) { - val discriminatingTimeRange = attributesTimeRanges.minBy { it.second } - val rangeStart = when (temporalQuery.timerel) { - TemporalQuery.Timerel.AFTER -> temporalQuery.timeAt ?: discriminatingTimeRange.first - TemporalQuery.Timerel.BETWEEN -> temporalQuery.timeAt ?: discriminatingTimeRange.first - else -> discriminatingTimeRange.first - } - - rangeStart to discriminatingTimeRange.second - } else { - val discriminatingTimeRange = attributesTimeRanges.maxBy { it.second } - val rangeStart = when (temporalQuery.timerel) { - TemporalQuery.Timerel.BEFORE -> temporalQuery.timeAt ?: discriminatingTimeRange.first - TemporalQuery.Timerel.BETWEEN -> temporalQuery.endTimeAt ?: discriminatingTimeRange.first - else -> discriminatingTimeRange.first - } - - rangeStart to discriminatingTimeRange.second + val discriminatingTimeRange = attributesTimeRanges.maxBy { it.second } + val rangeStart = when (temporalQuery.timerel) { + Timerel.BEFORE -> temporalQuery.timeAt ?: discriminatingTimeRange.first + Timerel.BETWEEN -> temporalQuery.endTimeAt ?: discriminatingTimeRange.first + else -> discriminatingTimeRange.first } + + return rangeStart to discriminatingTimeRange.second } private fun filterEntityInRange(entity: CompactedEntity, range: Range, query: TemporalQuery): CompactedEntity { @@ -173,16 +166,16 @@ object TemporalApiResponse { private fun Range.contain(time: ZonedDateTime) = (this.first > time && time > this.second) || (this.first < time && time < this.second) - private fun Range.contain(attribute: CompactedAttribute, query: TemporalQuery) = this.contain( - attribute.getAttributeDate(query) - ) + private fun Range.contain(attribute: CompactedAttribute, query: TemporalQuery) = + attribute.getAttributeDate(query) ?.let { this.contain(it) } ?: true // if no date it should not be filtered - private fun CompactedAttribute.getAttributeDate(query: TemporalQuery) = ZonedDateTime.parse( - this[query.timeproperty.propertyName] as String - ) + private fun CompactedAttribute.getAttributeDate(query: TemporalQuery) = + this[query.timeproperty.propertyName]?.let { + ZonedDateTime.parse(it as String) + } private fun getHeaderRange(range: Range, temporalQuery: TemporalQuery): String { - val size = if (temporalQuery.isChronological) "*" else temporalQuery.limit + val size = "*" return "DateTime ${range.first.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt index b76d3774e7..a639cfe491 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt @@ -4,7 +4,7 @@ import com.egm.stellio.search.model.AttributeInstance import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.model.TemporalQuery.Aggregate -import com.egm.stellio.search.model.TemporalQuery.Timerel +import com.egm.stellio.search.model.Timerel import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.NgsiLdAttribute import com.egm.stellio.shared.model.toNgsiLdAttributes @@ -34,7 +34,7 @@ fun buildDefaultTestTemporalQuery( endTimeAt: ZonedDateTime? = null, aggrPeriodDuration: String? = null, aggrMethods: List? = null, - isChronological: Boolean = true, + asLastN: Boolean = false, limit: Int = 100, timeproperty: AttributeInstance.TemporalProperty = AttributeInstance.TemporalProperty.OBSERVED_AT ) = TemporalQuery( @@ -43,7 +43,7 @@ fun buildDefaultTestTemporalQuery( endTimeAt = endTimeAt, aggrPeriodDuration = aggrPeriodDuration, aggrMethods = aggrMethods, - isChronological = isChronological, + asLastN = asLastN, limit = limit, timeproperty = timeproperty ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt index 90e0d80039..f79ba13dd3 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt @@ -430,7 +430,7 @@ class EntitiesQueryUtilsTests { val temporalQuery = buildTemporalQuery(queryParams, buildDefaultPagination()).shouldSucceedAndResult() assertEquals(2, temporalQuery.limit) - assertFalse(temporalQuery.isChronological) + assertFalse(temporalQuery.asLastN) } @Test @@ -443,7 +443,7 @@ class EntitiesQueryUtilsTests { val temporalQuery = buildTemporalQuery(queryParams, pagination).shouldSucceedAndResult() assertEquals(pagination.temporalLimitDefault, temporalQuery.limit) - assertTrue(temporalQuery.isChronological) + assertTrue(temporalQuery.asLastN) } @Test @@ -457,7 +457,7 @@ class EntitiesQueryUtilsTests { val temporalQuery = buildTemporalQuery(queryParams, pagination).shouldSucceedAndResult() assertEquals(pagination.temporalLimitDefault, temporalQuery.limit) - assertTrue(temporalQuery.isChronological) + assertTrue(temporalQuery.asLastN) } @Test diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index dc6bab3095..bf4844f506 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -45,29 +45,29 @@ import java.util.* @ActiveProfiles("test") @WebFluxTest(TemporalEntityHandler::class) @EnableConfigurationProperties(ApplicationProperties::class, SearchProperties::class) -class TemporalEntityHandlerTests { +open class TemporalEntityHandlerTests { @Autowired - private lateinit var webClient: WebTestClient + protected lateinit var webClient: WebTestClient @MockkBean(relaxed = true) - private lateinit var queryService: QueryService + protected lateinit var queryService: QueryService @MockkBean - private lateinit var entityPayloadService: EntityPayloadService + protected lateinit var entityPayloadService: EntityPayloadService @MockkBean - private lateinit var attributeInstanceService: AttributeInstanceService + protected lateinit var attributeInstanceService: AttributeInstanceService @MockkBean(relaxed = true) - private lateinit var temporalEntityAttributeService: TemporalEntityAttributeService + protected lateinit var temporalEntityAttributeService: TemporalEntityAttributeService @MockkBean - private lateinit var authorizationService: AuthorizationService + protected lateinit var authorizationService: AuthorizationService - private val entityUri = "urn:ngsi-ld:BeeHive:TESTC".toUri() - private val temporalEntityAttributeName = "speed" - private val attributeInstanceId = "urn:ngsi-ld:Instance:01".toUri() + protected val entityUri = "urn:ngsi-ld:BeeHive:TESTC".toUri() + protected val temporalEntityAttributeName = "speed" + protected val attributeInstanceId = "urn:ngsi-ld:Instance:01".toUri() @BeforeAll fun configureWebClientDefaults() { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt new file mode 100644 index 0000000000..374898c6d2 --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt @@ -0,0 +1,157 @@ +package com.egm.stellio.search.web + +import arrow.core.Either +import com.egm.stellio.search.config.SearchProperties +import com.egm.stellio.shared.config.ApplicationProperties +import com.egm.stellio.shared.util.loadAndExpandSampleData +import io.mockk.coEvery +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest +import org.springframework.test.context.ActiveProfiles + +@ActiveProfiles("test") +@WebFluxTest(TemporalEntityHandler::class) +@EnableConfigurationProperties(ApplicationProperties::class, SearchProperties::class) +class TemporalEntityHandlerpaginationTests : TemporalEntityHandlerTests() { + + private suspend fun setupSize5TemporalTest() { + val firstTemporalEntity = loadAndExpandSampleData( + "/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld" + ) + val secondTemporalEntity = loadAndExpandSampleData("beehive.jsonld") + + coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns Either.Right(Pair(listOf(firstTemporalEntity, secondTemporalEntity), 2)) + } + + private suspend fun setupSize10TemporalTestwithTemporalValues() { + val firstTemporalEntity = loadAndExpandSampleData( + "/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld" + ) + val secondTemporalEntity = loadAndExpandSampleData("beehive.jsonld") + + coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns Either.Right(Pair(listOf(firstTemporalEntity, secondTemporalEntity), 2)) + } + + @Test + fun `query temporal entity should return 206 if there is truncated temporal attributes`() = runTest { + setupSize5TemporalTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&timeAt=2019-10-17T07:31:39Z&endTimeAt=2019-10-18T07:31:39Z&" + + "lastN=5&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader().valueMatches("Content-Range", ".*/5") + } + + + @Test + fun `query temporal entity without lastN and timerel before should return 206 if there is truncated attributes`() = + runTest { + setupSize5TemporalTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&timeAt=2019-10-17T07:31:39Z&endTimeAt=2019-10-18T07:31:39Z&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader().valueMatches("Content-Range", ".*/\\*") + } + + @Test + fun `query temporal entity without lastN and with timerel between should return range-start = timeAt`() = + runTest { + setupSize5TemporalTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&timeAt=2019-10-17T07:31:39Z&endTimeAt=2019-10-18T07:31:39Z&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "DateTime Thu, 17 Oct 2019 07:31:39 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" + ) + } + + @Test + fun `query temporal entity without lastN and with timerel after should return range-start = timeAt`() = + runTest { + setupSize5TemporalTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=before&timeAt=2019-10-17T07:31:39Z&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "DateTime Thu, 17 Oct 2019 07:31:39 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" + ) + } + + @Test + fun `query temporal entity with timerel after should return range-start = least recent timestamp`() = + runTest { + setupSize5TemporalTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=before&timeAt=2019-10-17T07:31:39Z&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "DateTime Fri, 24 Jan 2020 13:01:22 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" + ) + } + + @Test + fun `query temporal entity uqdyègèfgq`() = + runTest { + setupSize10TemporalTestwithTemporalValues() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=before&timeAt=2019-10-17T07:31:39Z&" + + "lastN=10&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "DateTime Fri, 24 Jan 2020 13:01:22 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" + ) + } +} + diff --git a/search-service/src/test/resources/application-test.properties b/search-service/src/test/resources/application-test.properties index 8f7bdba917..407cefcc63 100644 --- a/search-service/src/test/resources/application-test.properties +++ b/search-service/src/test/resources/application-test.properties @@ -3,3 +3,5 @@ application.authentication.enabled = true application.contexts.core = http://localhost:8093/jsonld-contexts/ngsi-ld-core-context-v1.8.jsonld application.contexts.authz = http://localhost:8093/jsonld-contexts/authorization.jsonld application.contexts.authz-compound = http://localhost:8093/jsonld-contexts/authorization-compound.jsonld +application.pagination.temporal-limit-default = 5 +application.pagination.temporal-limit-max = 1000 \ No newline at end of file diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld new file mode 100644 index 0000000000..a5863f9a96 --- /dev/null +++ b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld @@ -0,0 +1,102 @@ +{ + "id": "urn:ngsi-ld:BeeHive:TESTC", + "type": "BeeHive", + "incoming": [ + { + "type": "Property", + "value": 1500, + "observedAt": "2020-01-24T13:01:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1501, + "observedAt": "2020-01-24T13:02:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1502, + "observedAt": "2020-01-24T13:03:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1503, + "observedAt": "2020-01-24T13:04:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1504, + "observedAt": "2020-01-24T13:05:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + } + ], + "outgoing": [ + { + "type": "Property", + "value": 100, + "observedAt": "2020-01-24T13:05:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 101, + "observedAt": "2020-01-24T13:06:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 102, + "observedAt": "2020-01-24T13:07:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 103, + "observedAt": "2020-01-24T13:08:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 104, + "observedAt": "2020-01-24T13:09:22.066Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + } + + ], + "@context":[ + "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" + ] +} diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld new file mode 100644 index 0000000000..5bb32627c6 --- /dev/null +++ b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld @@ -0,0 +1,37 @@ +{ + "id": "urn:ngsi-ld:BeeHive:TESTC", + "type": "BeeHive", + "incoming": { + "type": "Property", + "values": [ + [1500.0, "2020-01-24T13:05:22.066Z"], + [1501.0, "2020-01-24T13:06:22.066Z"], + [1502.0, "2020-01-24T13:07:22.066Z"], + [1503.0, "2020-01-24T13:08:22.066Z"], + [1504.0, "2020-01-24T13:09:22.066Z"], + [1505.0, "2020-01-24T13:10:22.066Z"], + [1506.0, "2020-01-24T13:11:22.066Z"], + [1507.0, "2020-01-24T13:12:22.066Z"], + [1508.0, "2020-01-24T13:13:22.066Z"], + [1509.0, "2020-01-24T13:14:22.066Z"] + ] + }, + "outgoing": { + "type": "Property", + "values": [ + [100.0, "2020-01-24T13:01:22.066Z"], + [101.0, "2020-01-24T14:02:22.066Z"], + [102.0, "2020-01-24T14:03:22.066Z"], + [103.0, "2020-01-24T14:04:22.066Z"], + [104.0, "2020-01-24T14:05:22.066Z"], + [105.0, "2020-01-24T14:06:22.066Z"], + [106.0, "2020-01-24T14:07:22.066Z"], + [107.0, "2020-01-24T14:08:22.066Z"], + [108.0, "2020-01-24T14:09:22.066Z"], + [109.0, "2020-01-24T14:10:22.066Z"] + ] + }, + "@context":[ + "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" + ] +} diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt index 7d0c2676bb..7f3363c039 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt @@ -19,7 +19,7 @@ import org.springframework.util.MultiValueMap import reactor.core.publisher.Mono import java.time.ZonedDateTime import java.time.format.DateTimeParseException -import java.util.Optional +import java.util.* import java.util.regex.Pattern const val RESULTS_COUNT_HEADER = "NGSILD-Results-Count" @@ -42,6 +42,8 @@ const val QUERY_PARAM_OPTIONS_SYSATTRS_VALUE: String = "sysAttrs" const val QUERY_PARAM_OPTIONS_KEYVALUES_VALUE: String = "keyValues" const val QUERY_PARAM_OPTIONS_NOOVERWRITE_VALUE: String = "noOverwrite" const val QUERY_PARAM_OPTIONS_OBSERVEDAT_VALUE: String = "observedAt" + + val JSON_LD_MEDIA_TYPE = MediaType.valueOf(JSON_LD_CONTENT_TYPE) val GEO_JSON_MEDIA_TYPE = MediaType.valueOf(GEO_JSON_CONTENT_TYPE) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt index 0c24aac606..ecebf69f51 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/DateUtils.kt @@ -14,7 +14,7 @@ fun ZonedDateTime.toNgsiLdFormat(): String = formatter.format(this) fun ZonedDateTime.toHttpHeaderFormat(): String = - httpHeaderFormatter.format(this) + formatter.format(this) fun ngsiLdDateTime(): ZonedDateTime = Instant.now().truncatedTo(ChronoUnit.MICROS).atZone(ZoneOffset.UTC) From 90c73ee806fac912fb5bba5a4498eb17e8b91577 Mon Sep 17 00:00:00 2001 From: Thomas BOUSSELIN Date: Wed, 19 Jun 2024 14:58:25 +0200 Subject: [PATCH 14/14] fix: test working and temporal values support --- .../egm/stellio/search/scope/ScopeService.kt | 13 +- .../service/AttributeInstanceService.kt | 4 +- .../search/web/BuildTemporalApiResponse.kt | 89 +++++-- .../stellio/search/scope/ScopeServiceTests.kt | 2 +- .../service/AttributeInstanceServiceTests.kt | 4 +- .../egm/stellio/search/support/TestUtils.kt | 3 +- .../search/util/EntitiesQueryUtilsTests.kt | 6 +- .../TemporalEntityHandlerPaginationTests.kt | 222 ++++++++++++++++++ .../search/web/TemporalEntityHandlerTests.kt | 2 +- .../TemporalEntityHandlerpaginationTests.kt | 157 ------------- .../resources/application-test.properties | 2 +- ...nized_temporal_attributes_evolution.jsonld | 20 +- ...ttributes_evolution_temporal_values.jsonld | 27 +++ ..._four_temporal_attributes_evolution.jsonld | 102 ++++++++ ...ttributes_evolution_temporal_values.jsonld | 37 --- .../expected_temporal_value_filtered.json | 1 + ...ected_without_temporal_value_filtered.json | 1 + 17 files changed, 444 insertions(+), 248 deletions(-) create mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerPaginationTests.kt delete mode 100644 search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt create mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld create mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_four_temporal_attributes_evolution.jsonld delete mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld create mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/expected_temporal_value_filtered.json create mode 100644 search-service/src/test/resources/ngsild/temporal/pagination/expected_without_temporal_value_filtered.json diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index 8586ff706f..a4cfc52afc 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -115,12 +115,11 @@ class ScopeService( else if (temporalEntitiesQuery.withAggregatedValues) sqlQueryBuilder.append(" GROUP BY entity_id") if (temporalQuery.asLastN) - // in order to get first or last instances, need to order by time - // final ascending ordering of instances is done in query service - sqlQueryBuilder.append(" ORDER BY start DESC") + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service + sqlQueryBuilder.append(" ORDER BY start DESC") else sqlQueryBuilder.append(" ORDER BY start ASC") - sqlQueryBuilder.append(" LIMIT $limit") return databaseClient.sql(sqlQueryBuilder.toString()) @@ -279,9 +278,9 @@ class ScopeService( else TemporalProperty.MODIFIED_AT addHistoryEntry(entityId, it, temporalPropertyToAdd, modifiedAt, sub).bind() if (temporalPropertyToAdd == TemporalProperty.MODIFIED_AT) - // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a - // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt - // sub-Property + // as stated in 4.5.6: In case the Temporal Representation of the Scope is updated as the result of a + // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt + // sub-Property addHistoryEntry(entityId, it, TemporalProperty.OBSERVED_AT, modifiedAt, sub).bind() updateResult } ?: UpdateResult( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt index 7d7b501d85..b7ea3dd8fd 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/AttributeInstanceService.kt @@ -164,8 +164,8 @@ class AttributeInstanceService( sqlQueryBuilder.append(" GROUP BY temporal_entity_attribute") if (temporalQuery.asLastN) - // in order to get first or last instances, need to order by time - // final ascending ordering of instances is done in query service + // in order to get first or last instances, need to order by time + // final ascending ordering of instances is done in query service sqlQueryBuilder.append(" ORDER BY start DESC") else sqlQueryBuilder.append(" ORDER BY start ASC") diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt index bf1943fd14..360fcc9757 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/BuildTemporalApiResponse.kt @@ -2,7 +2,6 @@ package com.egm.stellio.search.web import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalQuery -import com.egm.stellio.search.model.Timerel import com.egm.stellio.shared.model.CompactedAttribute import com.egm.stellio.shared.model.CompactedEntity import com.egm.stellio.shared.model.toFinalRepresentation @@ -19,6 +18,8 @@ import org.springframework.util.MultiValueMap import java.time.ZonedDateTime typealias CompactedTemporalAttribute = List> +typealias TemporalValue = List +const val TEMPORAL_VALUE_NAME = "values" typealias Range = Pair @@ -44,6 +45,11 @@ object TemporalApiResponse { mediaType, contexts ) + + if (query.temporalQuery.asLastN) { // if lastN > limit it throw an error earlier + return successResponse + } + val attributesWhoReachedLimit = getAttributesWhoReachedLimit(entities, query) if (attributesWhoReachedLimit.isEmpty()) { @@ -52,14 +58,14 @@ object TemporalApiResponse { val range = getTemporalPaginationRange(attributesWhoReachedLimit, query) - val filteredEntities = entities.map { filterEntityInRange(it, range, query.temporalQuery) } + val filteredEntities = entities.map { filterEntityInRange(it, range, query) } return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT).apply { this.headers( successResponse.headers ) this.header( HttpHeaders.CONTENT_RANGE, - getHeaderRange(range, query.temporalQuery) + getHeaderRange(range) ) }.body(serializeObject(filteredEntities.toFinalRepresentation(ngsiLdDataRepresentation))) } @@ -90,13 +96,13 @@ object TemporalApiResponse { } val range = getTemporalPaginationRange(attributesWhoReachedLimit, query) - val filteredEntity = filterEntityInRange(entity, range, query.temporalQuery) + val filteredEntity = filterEntityInRange(entity, range, query) return ResponseEntity.status(HttpStatus.PARTIAL_CONTENT) .apply { this.header( HttpHeaders.CONTENT_RANGE, - getHeaderRange(range, query.temporalQuery) + getHeaderRange(range) ) this.headers( successResponse.headers @@ -108,53 +114,78 @@ object TemporalApiResponse { ) } - private fun getAttributesWhoReachedLimit(entities: List, query: TemporalEntitiesQuery): - List { + private fun getAttributesWhoReachedLimit(entities: List, query: TemporalEntitiesQuery): List { val temporalQuery = query.temporalQuery - return entities.flatMap { attr -> - attr.values.mapNotNull { - if (it is List<*> && it.size >= temporalQuery.limit) it as CompactedTemporalAttribute else null + + return entities.flatMap { entity -> + entity.values.mapNotNull { attr -> + + when (attr) { + is List<*> -> if (attr.size >= temporalQuery.limit) attr else null + is Map<*, *> -> { + val attrSize = attr.toTemporalValuesOrNull(query)?.size ?: 1 + if (attrSize >= temporalQuery.limit) attr else null + } + else -> null + } } } } private fun getTemporalPaginationRange( - attributesWhoReachedLimit: List, + attributesWhoReachedLimit: List, query: TemporalEntitiesQuery ): Range { val temporalQuery = query.temporalQuery val limit = temporalQuery.limit val timeProperty = temporalQuery.timeproperty.propertyName - val attributesTimeRanges = attributesWhoReachedLimit.map { attribute -> attribute.map { it[timeProperty] } } - .map { - ZonedDateTime.parse(it.getOrNull(0) as String) to - ZonedDateTime.parse(it.getOrNull(limit - 1) as String) + val allTimesByAttributes = attributesWhoReachedLimit.mapNotNull { attr -> + when (attr) { + is List<*> -> { + val temporalAttribute = attr as CompactedTemporalAttribute + temporalAttribute.map { it[timeProperty] } + } + is Map<*, *> -> { + attr.toTemporalValuesOrNull(query)?.map { it[1] } + } + else -> null } + } + val attributesTimeRanges = allTimesByAttributes.map { + ZonedDateTime.parse(it.getOrNull(0) as String) to + ZonedDateTime.parse(it.getOrNull(limit - 1) as String) + } - val discriminatingTimeRange = attributesTimeRanges.maxBy { it.second } + val discriminatingTimeRange = attributesTimeRanges.minBy { it.second } val rangeStart = when (temporalQuery.timerel) { - Timerel.BEFORE -> temporalQuery.timeAt ?: discriminatingTimeRange.first - Timerel.BETWEEN -> temporalQuery.endTimeAt ?: discriminatingTimeRange.first + TemporalQuery.Timerel.AFTER -> temporalQuery.timeAt ?: discriminatingTimeRange.first + TemporalQuery.Timerel.BETWEEN -> temporalQuery.timeAt ?: discriminatingTimeRange.first else -> discriminatingTimeRange.first } return rangeStart to discriminatingTimeRange.second } - private fun filterEntityInRange(entity: CompactedEntity, range: Range, query: TemporalQuery): CompactedEntity { + private fun filterEntityInRange( + entity: CompactedEntity, + range: Range, + query: TemporalEntitiesQuery + ): CompactedEntity { return entity.keys.mapNotNull { key -> when (val attribute = entity[key]) { is List<*> -> { val temporalAttribute = attribute as CompactedTemporalAttribute // faster filter possible because list is already order - val filteredAttribute = temporalAttribute.filter { range.contain(it, query) } + val filteredAttribute = temporalAttribute.filter { range.contain(it, query.temporalQuery) } if (filteredAttribute.isNotEmpty()) key to filteredAttribute else null } is Map<*, *> -> { - val compactAttribute: CompactedAttribute = attribute as CompactedAttribute - if (range.contain(compactAttribute, query)) key to attribute + attribute.toTemporalValuesOrNull(query)?.let { + return@mapNotNull key to it.filter { range.contain(it) } + } + if (range.contain(attribute as CompactedAttribute, query.temporalQuery)) key to attribute else key to emptyList() } @@ -164,7 +195,10 @@ object TemporalApiResponse { } private fun Range.contain(time: ZonedDateTime) = - (this.first > time && time > this.second) || (this.first < time && time < this.second) + (this.first >= time && time >= this.second) || (this.first <= time && time <= this.second) + + private fun Range.contain(temporalValue: TemporalValue) = + this.contain(ZonedDateTime.parse(temporalValue[1] as String)) private fun Range.contain(attribute: CompactedAttribute, query: TemporalQuery) = attribute.getAttributeDate(query) ?.let { this.contain(it) } ?: true // if no date it should not be filtered @@ -174,8 +208,13 @@ object TemporalApiResponse { ZonedDateTime.parse(it as String) } - private fun getHeaderRange(range: Range, temporalQuery: TemporalQuery): String { + private fun Map<*, *>.toTemporalValuesOrNull(query: TemporalEntitiesQuery) = + if (query.withTemporalValues && this[TEMPORAL_VALUE_NAME] is List<*>) + this[TEMPORAL_VALUE_NAME] as List + else null + + private fun getHeaderRange(range: Range): String { val size = "*" - return "DateTime ${range.first.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" + return "date-time ${range.first.toHttpHeaderFormat()}-${range.second.toHttpHeaderFormat()}/$size" } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt index 177c30fc66..c45b2d1ef2 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt @@ -303,7 +303,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { aggrMethods = listOf(TemporalQuery.Aggregate.SUM), aggrPeriodDuration = "PT1S", limit = 1, - isChronological = false + asLastN = true ), withTemporalValues = false, withAudit = false, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt index ab8d7ce106..4a358dc11f 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/AttributeInstanceServiceTests.kt @@ -359,7 +359,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer timerel = Timerel.AFTER, timeAt = now.minusHours(1), limit = 5, - isChronological = false + asLastN = true ) ) attributeInstanceService.search(temporalEntitiesQuery, incomingTemporalEntityAttribute) @@ -389,7 +389,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer aggrPeriodDuration = "PT1S", aggrMethods = listOf(TemporalQuery.Aggregate.SUM), limit = 5, - isChronological = false + asLastN = true ), withAggregatedValues = true ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt index a639cfe491..e03f2cb091 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/TestUtils.kt @@ -4,7 +4,6 @@ import com.egm.stellio.search.model.AttributeInstance import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.model.TemporalQuery.Aggregate -import com.egm.stellio.search.model.Timerel import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.NgsiLdAttribute import com.egm.stellio.shared.model.toNgsiLdAttributes @@ -29,7 +28,7 @@ import java.time.ZonedDateTime @SuppressWarnings("LongParameterList") fun buildDefaultTestTemporalQuery( - timerel: Timerel? = null, + timerel: TemporalQuery.Timerel? = null, timeAt: ZonedDateTime? = null, endTimeAt: ZonedDateTime? = null, aggrPeriodDuration: String? = null, diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt index f79ba13dd3..0672034037 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/util/EntitiesQueryUtilsTests.kt @@ -430,7 +430,7 @@ class EntitiesQueryUtilsTests { val temporalQuery = buildTemporalQuery(queryParams, buildDefaultPagination()).shouldSucceedAndResult() assertEquals(2, temporalQuery.limit) - assertFalse(temporalQuery.asLastN) + assertTrue(temporalQuery.asLastN) } @Test @@ -443,7 +443,7 @@ class EntitiesQueryUtilsTests { val temporalQuery = buildTemporalQuery(queryParams, pagination).shouldSucceedAndResult() assertEquals(pagination.temporalLimitDefault, temporalQuery.limit) - assertTrue(temporalQuery.asLastN) + assertFalse(temporalQuery.asLastN) } @Test @@ -457,7 +457,7 @@ class EntitiesQueryUtilsTests { val temporalQuery = buildTemporalQuery(queryParams, pagination).shouldSucceedAndResult() assertEquals(pagination.temporalLimitDefault, temporalQuery.limit) - assertTrue(temporalQuery.asLastN) + assertFalse(temporalQuery.asLastN) } @Test diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerPaginationTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerPaginationTests.kt new file mode 100644 index 0000000000..348bab70de --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerPaginationTests.kt @@ -0,0 +1,222 @@ +package com.egm.stellio.search.web + +import arrow.core.Either +import com.egm.stellio.search.config.SearchProperties +import com.egm.stellio.shared.model.ExpandedEntity +import com.egm.stellio.shared.util.loadAndExpandSampleData +import com.egm.stellio.shared.util.loadSampleData +import io.mockk.coEvery +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest +import org.springframework.test.context.ActiveProfiles +import org.springframework.test.context.TestPropertySource + +@ActiveProfiles("test") +@WebFluxTest(TemporalEntityHandler::class) +@EnableConfigurationProperties(SearchProperties::class) +@TestPropertySource( + properties = + [ + "application.pagination.temporal-limit-default = 5", + "application.pagination.temporal-limit-max = 5" + ] +) +class TemporalEntityHandlerPaginationTests : TemporalEntityHandlerTests() { + + val timeAt = "2019-01-01T00:00:00Z" + private val mostRecentTimestamp = "2020-01-01T00:05:00Z" // from discrimination attribute + private val leastRecentTimestamp = "2020-01-01T00:01:00Z" + + private suspend fun setupTemporalTestWithNonTruncatedValue() { + val firstTemporalEntity = loadAndExpandSampleData( + "/temporal/pagination/beehive_with_four_temporal_attributes_evolution.jsonld" + ) + mockService(firstTemporalEntity) + } + + private suspend fun setupTemporalPaginationTest() { + val firstTemporalEntity = loadAndExpandSampleData( + "/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld" + ) + mockService(firstTemporalEntity) + } + + private suspend fun setupTemporalPaginationTestwithTemporalValues() { + val firstTemporalEntity = loadAndExpandSampleData( + "/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld" + ) + + mockService(firstTemporalEntity) + } + + private suspend fun mockService(firstTemporalEntity: ExpandedEntity) { + val secondTemporalEntity = loadAndExpandSampleData("beehive.jsonld") + coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns Either.Right(Pair(listOf(firstTemporalEntity, secondTemporalEntity), 2)) + } + + @Test + fun `query temporal entity should return 200 if there is no truncated attributes`() = + runTest { + setupTemporalTestWithNonTruncatedValue() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(200) + } + + @Test // no range needed since we use the lastn send by the user + fun `query temporal entity with lastN should return 200 even if its reached limit`() = runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "lastN=5&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(200) + } + + @Test // no range needed since we use the lastn send by the user + fun `query temporal entity with lastN should return 403 if lastN is greater than temporal-max-limit`() = runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "lastN=1000000&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(403) + } + + @Test + fun `query temporal entity without lastN should return 206 if there is truncated attributes`() = + runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader().valueMatches("Content-Range", ".*/\\*") + } + + @Test + fun `query temporal entity with timerel between should return range-start = timeAt`() = + runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&timeAt=$timeAt&endTimeAt=2021-01-01T00:00:00Z&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "date-time $timeAt-$mostRecentTimestamp/*" + ) + } + + @Test + fun `query temporal entity with timerel after should return range-start = timeAt`() = + runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "date-time $timeAt-$mostRecentTimestamp/*" + ) + } + + @Test + fun `query temporal entity with timerel before should return range-start = least recent timestamp`() = + runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=before&timeAt=$timeAt&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "date-time $leastRecentTimestamp-$mostRecentTimestamp/*" + ) + } + + @Test + fun `query temporal entity with temporalValues should return filtered data`() = + runTest { + setupTemporalPaginationTestwithTemporalValues() + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "type=BeeHive&options=temporalValues" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "date-time $timeAt-$mostRecentTimestamp/*" + ).expectBody() + .json(loadSampleData("temporal/pagination/expected_temporal_value_filtered.json")) + } + + @Test + fun `query temporal entity without temporalValues option should return filtered Data`() = + runTest { + setupTemporalPaginationTest() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=after&timeAt=$timeAt&" + + "type=BeeHive" + ) + .exchange() + .expectStatus().isEqualTo(206) + .expectHeader() + .valueEquals( + "Content-Range", + "date-time $timeAt-$mostRecentTimestamp/*" + ).expectBody().json(loadSampleData("temporal/pagination/expected_without_temporal_value_filtered.json")) + } +} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index bf4844f506..255fe34668 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -383,7 +383,7 @@ open class TemporalEntityHandlerTests { buildDefaultMockResponsesForGetEntity() webClient.get() - .uri("/ngsi-ld/v1/temporal/entities/urn:ngsi-ld:Entity:01?timerel=between&timeAt=startTime") + .uri("/ngsi-ld/v1/temporal/entities/urn:ngsi-ld:Entity:01?timerel=between&timeAt=2020-10-29T18:00:00Z") .exchange() .expectStatus().isBadRequest .expectBody().json( diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt deleted file mode 100644 index 374898c6d2..0000000000 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerpaginationTests.kt +++ /dev/null @@ -1,157 +0,0 @@ -package com.egm.stellio.search.web - -import arrow.core.Either -import com.egm.stellio.search.config.SearchProperties -import com.egm.stellio.shared.config.ApplicationProperties -import com.egm.stellio.shared.util.loadAndExpandSampleData -import io.mockk.coEvery -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Test -import org.springframework.boot.context.properties.EnableConfigurationProperties -import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest -import org.springframework.test.context.ActiveProfiles - -@ActiveProfiles("test") -@WebFluxTest(TemporalEntityHandler::class) -@EnableConfigurationProperties(ApplicationProperties::class, SearchProperties::class) -class TemporalEntityHandlerpaginationTests : TemporalEntityHandlerTests() { - - private suspend fun setupSize5TemporalTest() { - val firstTemporalEntity = loadAndExpandSampleData( - "/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld" - ) - val secondTemporalEntity = loadAndExpandSampleData("beehive.jsonld") - - coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } - coEvery { - queryService.queryTemporalEntities(any(), any()) - } returns Either.Right(Pair(listOf(firstTemporalEntity, secondTemporalEntity), 2)) - } - - private suspend fun setupSize10TemporalTestwithTemporalValues() { - val firstTemporalEntity = loadAndExpandSampleData( - "/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld" - ) - val secondTemporalEntity = loadAndExpandSampleData("beehive.jsonld") - - coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } - coEvery { - queryService.queryTemporalEntities(any(), any()) - } returns Either.Right(Pair(listOf(firstTemporalEntity, secondTemporalEntity), 2)) - } - - @Test - fun `query temporal entity should return 206 if there is truncated temporal attributes`() = runTest { - setupSize5TemporalTest() - - webClient.get() - .uri( - "/ngsi-ld/v1/temporal/entities?" + - "timerel=between&timeAt=2019-10-17T07:31:39Z&endTimeAt=2019-10-18T07:31:39Z&" + - "lastN=5&" + - "type=BeeHive" - ) - .exchange() - .expectStatus().isEqualTo(206) - .expectHeader().valueMatches("Content-Range", ".*/5") - } - - - @Test - fun `query temporal entity without lastN and timerel before should return 206 if there is truncated attributes`() = - runTest { - setupSize5TemporalTest() - - webClient.get() - .uri( - "/ngsi-ld/v1/temporal/entities?" + - "timerel=between&timeAt=2019-10-17T07:31:39Z&endTimeAt=2019-10-18T07:31:39Z&" + - "type=BeeHive" - ) - .exchange() - .expectStatus().isEqualTo(206) - .expectHeader().valueMatches("Content-Range", ".*/\\*") - } - - @Test - fun `query temporal entity without lastN and with timerel between should return range-start = timeAt`() = - runTest { - setupSize5TemporalTest() - - webClient.get() - .uri( - "/ngsi-ld/v1/temporal/entities?" + - "timerel=between&timeAt=2019-10-17T07:31:39Z&endTimeAt=2019-10-18T07:31:39Z&" + - "type=BeeHive" - ) - .exchange() - .expectStatus().isEqualTo(206) - .expectHeader() - .valueEquals( - "Content-Range", - "DateTime Thu, 17 Oct 2019 07:31:39 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" - ) - } - - @Test - fun `query temporal entity without lastN and with timerel after should return range-start = timeAt`() = - runTest { - setupSize5TemporalTest() - - webClient.get() - .uri( - "/ngsi-ld/v1/temporal/entities?" + - "timerel=before&timeAt=2019-10-17T07:31:39Z&" + - "type=BeeHive" - ) - .exchange() - .expectStatus().isEqualTo(206) - .expectHeader() - .valueEquals( - "Content-Range", - "DateTime Thu, 17 Oct 2019 07:31:39 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" - ) - } - - @Test - fun `query temporal entity with timerel after should return range-start = least recent timestamp`() = - runTest { - setupSize5TemporalTest() - - webClient.get() - .uri( - "/ngsi-ld/v1/temporal/entities?" + - "timerel=before&timeAt=2019-10-17T07:31:39Z&" + - "type=BeeHive" - ) - .exchange() - .expectStatus().isEqualTo(206) - .expectHeader() - .valueEquals( - "Content-Range", - "DateTime Fri, 24 Jan 2020 13:01:22 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" - ) - } - - @Test - fun `query temporal entity uqdyègèfgq`() = - runTest { - setupSize10TemporalTestwithTemporalValues() - - webClient.get() - .uri( - "/ngsi-ld/v1/temporal/entities?" + - "timerel=before&timeAt=2019-10-17T07:31:39Z&" + - "lastN=10&" + - "type=BeeHive" - ) - .exchange() - .expectStatus().isEqualTo(206) - .expectHeader() - .valueEquals( - "Content-Range", - "DateTime Fri, 24 Jan 2020 13:01:22 GMT-Fri, 24 Jan 2020 13:05:22 GMT/*" - ) - } -} - diff --git a/search-service/src/test/resources/application-test.properties b/search-service/src/test/resources/application-test.properties index 407cefcc63..6ae5cb7638 100644 --- a/search-service/src/test/resources/application-test.properties +++ b/search-service/src/test/resources/application-test.properties @@ -3,5 +3,5 @@ application.authentication.enabled = true application.contexts.core = http://localhost:8093/jsonld-contexts/ngsi-ld-core-context-v1.8.jsonld application.contexts.authz = http://localhost:8093/jsonld-contexts/authorization.jsonld application.contexts.authz-compound = http://localhost:8093/jsonld-contexts/authorization-compound.jsonld -application.pagination.temporal-limit-default = 5 +application.pagination.temporal-limit-default = 100 application.pagination.temporal-limit-max = 1000 \ No newline at end of file diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld index a5863f9a96..cc1db08ba0 100644 --- a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld +++ b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution.jsonld @@ -5,7 +5,7 @@ { "type": "Property", "value": 1500, - "observedAt": "2020-01-24T13:01:22.066Z", + "observedAt": "2020-01-01T00:01:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -14,7 +14,7 @@ { "type": "Property", "value": 1501, - "observedAt": "2020-01-24T13:02:22.066Z", + "observedAt": "2020-01-01T00:02:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -23,7 +23,7 @@ { "type": "Property", "value": 1502, - "observedAt": "2020-01-24T13:03:22.066Z", + "observedAt": "2020-01-01T00:03:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -32,7 +32,7 @@ { "type": "Property", "value": 1503, - "observedAt": "2020-01-24T13:04:22.066Z", + "observedAt": "2020-01-01T00:04:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -41,7 +41,7 @@ { "type": "Property", "value": 1504, - "observedAt": "2020-01-24T13:05:22.066Z", + "observedAt": "2020-01-01T00:05:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -52,7 +52,7 @@ { "type": "Property", "value": 100, - "observedAt": "2020-01-24T13:05:22.066Z", + "observedAt": "2020-01-01T00:04:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -61,7 +61,7 @@ { "type": "Property", "value": 101, - "observedAt": "2020-01-24T13:06:22.066Z", + "observedAt": "2020-01-01T00:05:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -70,7 +70,7 @@ { "type": "Property", "value": 102, - "observedAt": "2020-01-24T13:07:22.066Z", + "observedAt": "2020-01-01T00:06:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -79,7 +79,7 @@ { "type": "Property", "value": 103, - "observedAt": "2020-01-24T13:08:22.066Z", + "observedAt": "2020-01-01T00:07:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" @@ -88,7 +88,7 @@ { "type": "Property", "value": 104, - "observedAt": "2020-01-24T13:09:22.066Z", + "observedAt": "2020-01-01T00:08:00.00Z", "observedBy": { "type": "Relationship", "object": "urn:ngsi-ld:Sensor:IncomingSensor" diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld new file mode 100644 index 0000000000..dfe7e1adb7 --- /dev/null +++ b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_five_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld @@ -0,0 +1,27 @@ +{ + "id": "urn:ngsi-ld:BeeHive:TESTC", + "type": "BeeHive", + "incoming": { + "type": "Property", + "values": [ + [1500.0, "2020-01-01T00:01:00.000Z"], + [1501.0, "2020-01-01T00:02:00.000Z"], + [1502.0, "2020-01-01T00:03:00.000Z"], + [1503.0, "2020-01-01T00:04:00.000Z"], + [1504.0, "2020-01-01T00:05:00.000Z"] + ] + }, + "outgoing": { + "type": "Property", + "values": [ + [100.0, "2020-01-01T00:04:00.000Z"], + [101.0, "2020-01-01T00:05:00.000Z"], + [102.0, "2020-01-01T00:06:00.000Z"], + [103.0, "2020-01-01T00:07:00.000Z"], + [104.0, "2020-01-01T00:08:00.000Z"] + ] + }, + "@context":[ + "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" + ] +} diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_four_temporal_attributes_evolution.jsonld b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_four_temporal_attributes_evolution.jsonld new file mode 100644 index 0000000000..cc1db08ba0 --- /dev/null +++ b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_four_temporal_attributes_evolution.jsonld @@ -0,0 +1,102 @@ +{ + "id": "urn:ngsi-ld:BeeHive:TESTC", + "type": "BeeHive", + "incoming": [ + { + "type": "Property", + "value": 1500, + "observedAt": "2020-01-01T00:01:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1501, + "observedAt": "2020-01-01T00:02:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1502, + "observedAt": "2020-01-01T00:03:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1503, + "observedAt": "2020-01-01T00:04:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 1504, + "observedAt": "2020-01-01T00:05:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + } + ], + "outgoing": [ + { + "type": "Property", + "value": 100, + "observedAt": "2020-01-01T00:04:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 101, + "observedAt": "2020-01-01T00:05:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 102, + "observedAt": "2020-01-01T00:06:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 103, + "observedAt": "2020-01-01T00:07:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + }, + { + "type": "Property", + "value": 104, + "observedAt": "2020-01-01T00:08:00.00Z", + "observedBy": { + "type": "Relationship", + "object": "urn:ngsi-ld:Sensor:IncomingSensor" + } + } + + ], + "@context":[ + "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" + ] +} diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld b/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld deleted file mode 100644 index 5bb32627c6..0000000000 --- a/search-service/src/test/resources/ngsild/temporal/pagination/beehive_with_ten_unsynchronized_temporal_attributes_evolution_temporal_values.jsonld +++ /dev/null @@ -1,37 +0,0 @@ -{ - "id": "urn:ngsi-ld:BeeHive:TESTC", - "type": "BeeHive", - "incoming": { - "type": "Property", - "values": [ - [1500.0, "2020-01-24T13:05:22.066Z"], - [1501.0, "2020-01-24T13:06:22.066Z"], - [1502.0, "2020-01-24T13:07:22.066Z"], - [1503.0, "2020-01-24T13:08:22.066Z"], - [1504.0, "2020-01-24T13:09:22.066Z"], - [1505.0, "2020-01-24T13:10:22.066Z"], - [1506.0, "2020-01-24T13:11:22.066Z"], - [1507.0, "2020-01-24T13:12:22.066Z"], - [1508.0, "2020-01-24T13:13:22.066Z"], - [1509.0, "2020-01-24T13:14:22.066Z"] - ] - }, - "outgoing": { - "type": "Property", - "values": [ - [100.0, "2020-01-24T13:01:22.066Z"], - [101.0, "2020-01-24T14:02:22.066Z"], - [102.0, "2020-01-24T14:03:22.066Z"], - [103.0, "2020-01-24T14:04:22.066Z"], - [104.0, "2020-01-24T14:05:22.066Z"], - [105.0, "2020-01-24T14:06:22.066Z"], - [106.0, "2020-01-24T14:07:22.066Z"], - [107.0, "2020-01-24T14:08:22.066Z"], - [108.0, "2020-01-24T14:09:22.066Z"], - [109.0, "2020-01-24T14:10:22.066Z"] - ] - }, - "@context":[ - "http://localhost:8093/jsonld-contexts/apic-compound.jsonld" - ] -} diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/expected_temporal_value_filtered.json b/search-service/src/test/resources/ngsild/temporal/pagination/expected_temporal_value_filtered.json new file mode 100644 index 0000000000..f19eb99532 --- /dev/null +++ b/search-service/src/test/resources/ngsild/temporal/pagination/expected_temporal_value_filtered.json @@ -0,0 +1 @@ +[{"id":"urn:ngsi-ld:BeeHive:TESTC","type":"https://ontology.eglobalmark.com/apic#BeeHive","https://ontology.eglobalmark.com/apic#incoming":[[1500.0,"2020-01-01T00:01:00.000Z"],[1501.0,"2020-01-01T00:02:00.000Z"],[1502.0,"2020-01-01T00:03:00.000Z"],[1503.0,"2020-01-01T00:04:00.000Z"],[1504.0,"2020-01-01T00:05:00.000Z"]],"https://ontology.eglobalmark.com/apic#outgoing":[[100.0,"2020-01-01T00:04:00.000Z"],[101.0,"2020-01-01T00:05:00.000Z"]],"@context":"http://localhost:8093/jsonld-contexts/ngsi-ld-core-context-v1.8.jsonld"},{"id":"urn:ngsi-ld:BeeHive:TESTC","type":"https://ontology.eglobalmark.com/apic#BeeHive","https://schema.org/name":{"type":"Property","value":"ParisBeehive12"},"https://ontology.eglobalmark.com/apic#dateOfFirstBee":{"type":"Property","value":{"type":"DateTime","@value":"2018-12-04T12:00:00.00Z"}},"https://ontology.eglobalmark.com/apic#incoming":[],"https://ontology.eglobalmark.com/egm#connectsTo":{"type":"Relationship","object":"urn:ngsi-ld:Beekeeper:Pascal"},"@context":"http://localhost:8093/jsonld-contexts/ngsi-ld-core-context-v1.8.jsonld"}] diff --git a/search-service/src/test/resources/ngsild/temporal/pagination/expected_without_temporal_value_filtered.json b/search-service/src/test/resources/ngsild/temporal/pagination/expected_without_temporal_value_filtered.json new file mode 100644 index 0000000000..1a3718e421 --- /dev/null +++ b/search-service/src/test/resources/ngsild/temporal/pagination/expected_without_temporal_value_filtered.json @@ -0,0 +1 @@ +[{"id":"urn:ngsi-ld:BeeHive:TESTC","type":"https://ontology.eglobalmark.com/apic#BeeHive","https://ontology.eglobalmark.com/apic#incoming":[{"type":"Property","value":1500,"observedAt":"2020-01-01T00:01:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}},{"type":"Property","value":1501,"observedAt":"2020-01-01T00:02:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}},{"type":"Property","value":1502,"observedAt":"2020-01-01T00:03:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}},{"type":"Property","value":1503,"observedAt":"2020-01-01T00:04:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}},{"type":"Property","value":1504,"observedAt":"2020-01-01T00:05:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}}],"https://ontology.eglobalmark.com/apic#outgoing":[{"type":"Property","value":100,"observedAt":"2020-01-01T00:04:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}},{"type":"Property","value":101,"observedAt":"2020-01-01T00:05:00.00Z","https://ontology.eglobalmark.com/egm#observedBy":{"type":"Relationship","object":"urn:ngsi-ld:Sensor:IncomingSensor"}}],"@context":"http://localhost:8093/jsonld-contexts/ngsi-ld-core-context-v1.8.jsonld"},{"id":"urn:ngsi-ld:BeeHive:TESTC","type":"https://ontology.eglobalmark.com/apic#BeeHive","https://schema.org/name":{"type":"Property","value":"ParisBeehive12"},"https://ontology.eglobalmark.com/apic#dateOfFirstBee":{"type":"Property","value":{"type":"DateTime","@value":"2018-12-04T12:00:00.00Z"}},"https://ontology.eglobalmark.com/apic#incoming":[],"https://ontology.eglobalmark.com/egm#connectsTo":{"type":"Relationship","object":"urn:ngsi-ld:Beekeeper:Pascal"},"@context":"http://localhost:8093/jsonld-contexts/ngsi-ld-core-context-v1.8.jsonld"}]