From b97b6039dc7c66552dd705e1934193c2ad2d847d Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 19 Aug 2024 15:13:34 +0200 Subject: [PATCH 01/52] #244: Create the Info module * created new module Info * the new modul added to JaCoco and CI routines --- .../scala/za/co/absa/atum/info/FLowInfo.scala | 21 +++++++++++++++++++ .../co/absa/atum/info/PartitioningInfo.scala | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala create mode 100644 info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala diff --git a/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala b/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala new file mode 100644 index 000000000..25c4dc899 --- /dev/null +++ b/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.info + +class FLowInfo { + +} diff --git a/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala b/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala new file mode 100644 index 000000000..1e9901b28 --- /dev/null +++ b/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.info + +class PartitioningInfo { + +} From e6239740a663ba5074de70d6e17dad458481e605 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 19 Aug 2024 15:42:01 +0200 Subject: [PATCH 02/52] * fixed License headers --- info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala | 2 +- info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala b/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala index 25c4dc899..b6daa42bf 100644 --- a/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala +++ b/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala b/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala index 1e9901b28..11608ef83 100644 --- a/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala +++ b/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 5e4eadb31da1e2beb451962f1b9e9a49b91ae014 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sat, 24 Aug 2024 11:44:57 +0200 Subject: [PATCH 03/52] * renamed to _Reader_ --- .../src/main/scala/za/co/absa/atum/info/FLowReader.scala | 2 +- .../main/scala/za/co/absa/atum/info/PartitioningRefactor.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala => reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala (97%) rename info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala => reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala (95%) diff --git a/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala b/reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala similarity index 97% rename from info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala rename to reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala index b6daa42bf..0dd0de6df 100644 --- a/info/src/main/scala/za/co/absa/atum/info/FLowInfo.scala +++ b/reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala @@ -16,6 +16,6 @@ package za.co.absa.atum.info -class FLowInfo { +class FLowReader { } diff --git a/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala b/reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala similarity index 95% rename from info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala rename to reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala index 11608ef83..ddf9391ae 100644 --- a/info/src/main/scala/za/co/absa/atum/info/PartitioningInfo.scala +++ b/reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala @@ -16,6 +16,6 @@ package za.co.absa.atum.info -class PartitioningInfo { +class PartitioningRefactor { } From 2e1e2ea445a1818cf19263212129f7ae8f558e24 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 25 Aug 2024 23:13:34 +0200 Subject: [PATCH 04/52] * README.md update From 738c904c2cb47277f2c9dd553242833902b1bc3f Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 25 Aug 2024 23:51:03 +0200 Subject: [PATCH 05/52] * fix From df8c9bdc1f55c26c7aa736b768642b2cd5e3f042 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 26 Aug 2024 02:03:56 +0200 Subject: [PATCH 06/52] * JaCoCO action update From 5affd82547e7948ed600a4122b5a5f23441a9f20 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 26 Aug 2024 02:25:37 +0200 Subject: [PATCH 07/52] * added dummy code for testing coverage --- .../atum/{info => reader}/PartitioningRefactor.scala | 7 +++++-- .../atum/reader/PartitioningRefactorUnitTests.scala} | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) rename reader/src/main/scala/za/co/absa/atum/{info => reader}/PartitioningRefactor.scala (85%) rename reader/src/{main/scala/za/co/absa/atum/info/FLowReader.scala => test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala} (72%) diff --git a/reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala similarity index 85% rename from reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala rename to reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala index ddf9391ae..6df031e42 100644 --- a/reader/src/main/scala/za/co/absa/atum/info/PartitioningRefactor.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala @@ -14,8 +14,11 @@ * limitations under the License. */ -package za.co.absa.atum.info +package za.co.absa.atum.reader class PartitioningRefactor { - + def foo(): String = { + // just to have some testable content + "bar" + } } diff --git a/reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala similarity index 72% rename from reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala rename to reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala index 0dd0de6df..ec4cabf06 100644 --- a/reader/src/main/scala/za/co/absa/atum/info/FLowReader.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala @@ -14,8 +14,13 @@ * limitations under the License. */ -package za.co.absa.atum.info +package za.co.absa.atum.reader -class FLowReader { +import org.scalatest.funsuite.AnyFunSuiteLike +class PartitioningRefactorUnitTests extends AnyFunSuiteLike { + test("foo") { + val expected = new FlowReader().foo() + assert(expected == "bar") + } } From 0f1e121de088b747da0baf7187c34af70c7d3194 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 26 Aug 2024 19:02:39 +0200 Subject: [PATCH 08/52] * erroneous class renamed * JaCoCo exclusion for model From d773a938fdb0a6acce1b797ad8ed61df37715656 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 26 Aug 2024 19:19:28 +0200 Subject: [PATCH 09/52] * Deleted wrong files --- .../atum/reader/PartitioningRefactor.scala | 24 ----------------- .../PartitioningRefactorUnitTests.scala | 26 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala deleted file mode 100644 index 6df031e42..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningRefactor.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader - -class PartitioningRefactor { - def foo(): String = { - // just to have some testable content - "bar" - } -} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala deleted file mode 100644 index ec4cabf06..000000000 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningRefactorUnitTests.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader - -import org.scalatest.funsuite.AnyFunSuiteLike - -class PartitioningRefactorUnitTests extends AnyFunSuiteLike { - test("foo") { - val expected = new FlowReader().foo() - assert(expected == "bar") - } -} From 0776f9c4970bad41ca796d11d8378a080e60942c Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 10 Sep 2024 15:24:51 +0200 Subject: [PATCH 10/52] #245 Add the ability to query REST endpoints from Reader module * created Provider to query the data from server * support for Future, IO, and ZIO based providers * work in progress --- project/Dependencies.scala | 16 +++-- .../za/co/absa/atum/reader/FlowReader.scala | 6 +- .../absa/atum/reader/PartitioningReader.scala | 6 +- .../za/co/absa/atum/reader/basic/Reader.scala | 21 +++++++ .../reader/exceptions/ReaderException.scala | 19 ++++++ .../reader/exceptions/RequestException.scala | 26 ++++++++ .../provider/AbstractHttpProvider.scala | 61 +++++++++++++++++++ .../absa/atum/reader/provider/Provider.scala | 24 ++++++++ .../reader/provider/future/HttpProvider.scala | 46 ++++++++++++++ .../reader/provider/io/HttpProvider.scala | 32 ++++++++++ .../reader/provider/zio/HttpProvider.scala | 24 ++++++++ 11 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e9783300e..df9fa90c3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -90,9 +90,9 @@ object Dependencies { private def jsonSerdeDependencies: Seq[ModuleID] = { // Circe dependencies - lazy val circeCore = "io.circe" %% "circe-core" % Versions.circeJson - lazy val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson - lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson + val circeCore = "io.circe" %% "circe-core" % Versions.circeJson + val circeParser = "io.circe" %% "circe-parser" % Versions.circeJson + val circeGeneric = "io.circe" %% "circe-generic" % Versions.circeJson Seq( circeCore, @@ -237,8 +237,16 @@ object Dependencies { def readerDependencies(scalaVersion: Version): Seq[ModuleID] = { Seq( + "com.softwaremill.sttp.client3" %% "core" % "3.9.7", + "com.softwaremill.sttp.client3" %% "async-http-client-backend-future" % "3.9.6", + "com.softwaremill.sttp.client3" %% "armeria-backend-cats" % "3.9.8", + "com.softwaremill.sttp.client3" %% "zio" % "3.9.8", + "com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "3.9.8", + "org.typelevel" %% "cats-effect" % "3.3.14", + "dev.zio" %% "zio" % "2.1.4", ) ++ - testDependencies + testDependencies ++ + jsonSerdeDependencies } def databaseDependencies: Seq[ModuleID] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 6c45d504e..09f1b0bf2 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -16,7 +16,11 @@ package za.co.absa.atum.reader -class FlowReader { +import za.co.absa.atum.reader.basic.Reader +import za.co.absa.atum.reader.provider.Provider + +// TODO +class FlowReader[F[_]](override implicit val provider: Provider[F]) extends Reader[F]{ def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index d1153e4b5..263254934 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -16,7 +16,11 @@ package za.co.absa.atum.reader -class PartitioningReader { +import cats.Monad +import za.co.absa.atum.reader.basic.Reader +import za.co.absa.atum.reader.provider.Provider + +class PartitioningReader[F[_]: Monad](partitioning: Partitioning)(override implicit val provider: Provider[F[_]]) extends Reader[F] { def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala new file mode 100644 index 000000000..ba4c65678 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -0,0 +1,21 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import za.co.absa.atum.reader.provider.Provider + +abstract class Reader[F[_]](implicit val provider: Provider[F[_]]) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala new file mode 100644 index 000000000..d668bd39b --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.exceptions + +abstract class ReaderException(message: String) extends Exception(message) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala new file mode 100644 index 000000000..c5130cd59 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.exceptions + +import sttp.model.{RequestMetadata, StatusCode} + +case class RequestException ( + message: String, + responseBody: String, + statusCode: StatusCode, + request: RequestMetadata) + extends ReaderException(message) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala new file mode 100644 index 000000000..7eb811a91 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala @@ -0,0 +1,61 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.provider + +import _root_.io.circe.parser.decode +import _root_.io.circe.Decoder +import com.typesafe.config.Config +import sttp.client3.{Response, SttpBackend, UriContext, basicRequest} +import za.co.absa.atum.reader.exceptions.RequestException + +import scala.util.{Failure, Try} + +/** + * A HttpProvider is a component that is responsible for providing teh data to readers using REST API + * @tparam F + */ +abstract class AbstractHttpProvider[F[_]](val serverUrl: String) extends Provider[F] { + type RequestFunction = SttpBackend[F, Any] => F[Response[Either[String, String]]] + type ResponseMapperFunction[R] = Response[Either[String, String]] => Try[R] + + protected def executeRequest(requestFnc: RequestFunction): F[Response[Either[String, String]]] + protected def mapResponse[R](response: F[Response[Either[String, String]]], mapperFnc: ResponseMapperFunction[R]): F[Try[R]] + + protected def query[R: Decoder](endpointUri: String): F[Try[R]] = { + val endpointToQuery = serverUrl + endpointUri + val request = basicRequest + .get(uri"$endpointToQuery") + val response = executeRequest(request.send(_)) + mapResponse(response, responseMapperFunction[R]) + } + + private def responseMapperFunction[R: Decoder](response: Response[Either[String, String]]): Try[R] = { + response.body match { + case Left(error) => Failure(RequestException(response.statusText, error, response.code, response.request)) + case Right(body) => decode[R](body).toTry + } + } + +} + +object AbstractHttpProvider { + final val UrlKey = "atum.server.url" + + def atumServerUrl(config: Config): String = { + config.getString(UrlKey) + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala new file mode 100644 index 000000000..c6d631787 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.provider + +/** + * A basic class for defining methods that will be providing data to readers. + */ +abstract class Provider[F[_]] { + // here will come abstract methods that are to return data to readers +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala new file mode 100644 index 000000000..e2cd69f61 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.provider.future + +import cats.implicits.catsStdInstancesForFuture +import com.typesafe.config.{Config, ConfigFactory} +import sttp.client3.Response +import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend +import za.co.absa.atum.reader.provider.AbstractHttpProvider + +import scala.concurrent.{ExecutionContext, Future} +import scala.util.Try + + +class HttpProvider(serverUrl: String)(implicit executor: ExecutionContext) extends AbstractHttpProvider[Future](serverUrl) { + + def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { + this(AbstractHttpProvider.atumServerUrl(config ))(executor) + } + + private val asyncHttpClientFutureBackend = AsyncHttpClientFutureBackend() + + override protected def executeRequest(requestFnc: RequestFunction): Future[Response[Either[String, String]]] = { + requestFnc(asyncHttpClientFutureBackend) + } + + override protected def mapResponse[R]( + response: Future[Response[Either[String, String]]], + mapperFnc: ResponseMapperFunction[R]): Future[Try[R]] = { + response.map(mapperFnc) + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala new file mode 100644 index 000000000..d8d1ba9b9 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala @@ -0,0 +1,32 @@ +package za.co.absa.atum.reader.provider.io + +import cats.effect.IO +import com.typesafe.config.{Config, ConfigFactory} +import sttp.client3.Response +import sttp.client3.armeria.cats.ArmeriaCatsBackend +import za.co.absa.atum.reader.provider.AbstractHttpProvider + +import scala.util.Try + +class HttpProvider(serverUrl: String) extends AbstractHttpProvider[IO](serverUrl) { + + def this(config: Config = ConfigFactory.load()) = { + this(AbstractHttpProvider.atumServerUrl(config )) + } + + override protected def executeRequest(requestFnc: RequestFunction): IO[Response[Either[String, String]]] = { + ArmeriaCatsBackend + .resource[IO]() + .use(requestFnc) + } + + override protected def mapResponse[R]( + response: IO[Response[Either[String, String]]], + mapperFnc: ResponseMapperFunction[R]): IO[Try[R]] = { + response.map(mapperFnc) + } +} + +object HttpProvider { + lazy implicit val httpProvider: HttpProvider = new HttpProvider() +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala new file mode 100644 index 000000000..a1885447d --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala @@ -0,0 +1,24 @@ +package za.co.absa.atum.reader.provider.zio + +import com.typesafe.config.{Config, ConfigFactory} +import sttp.client3.Response +import sttp.client3.armeria.zio.ArmeriaZioBackend +import za.co.absa.atum.reader.provider.AbstractHttpProvider +import zio.ZIO + +import scala.util.Try + +class HttpProvider(serverUrl: String) extends AbstractHttpProvider[ZIO](serverUrl) { + + def this(config: Config = ConfigFactory.load()) = { + this(AbstractHttpProvider.atumServerUrl(config )) + } + + + + override protected def executeRequest(requestFnc: RequestFunction): ZIO[Response[Either[String, String]]] = { + ArmeriaZioBackend.usingDefaultClient().map(requestFnc) + } + + override protected def mapResponse[R](response: ZIO[Response[Either[String, String]]], mapperFnc: ResponseMapperFunction[R]): ZIO[Try[R]] = ??? +} //TODO From 38fde1cb069f6a64095ade06b961c2afcaab0f00 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 23 Sep 2024 17:35:02 +0200 Subject: [PATCH 11/52] * Work still in progress --- agent/README.md | 35 ++++++++++++++ .../main/postgres/runs/get_measurements.sql | 47 +++++++++++++++++++ project/Dependencies.scala | 2 +- .../za/co/absa/atum/reader/FlowReader.scala | 5 +- .../absa/atum/reader/PartitioningReader.scala | 5 +- .../za/co/absa/atum/reader/basic/Reader.scala | 4 +- .../absa/atum/reader/provider/Provider.scala | 24 ---------- .../reader/provider/io/HttpProvider.scala | 32 ------------- .../reader/provider/zio/HttpProvider.scala | 24 ---------- .../GenericServerConnection.scala} | 35 +++++++------- .../future/ServerConnection.scala} | 27 +++++------ .../reader/server/io/ServerConnection.scala | 41 ++++++++++++++++ .../reader/server/zio/ServerConnection.scala | 45 ++++++++++++++++++ 13 files changed, 205 insertions(+), 121 deletions(-) create mode 100644 database/src/main/postgres/runs/get_measurements.sql delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala rename reader/src/main/scala/za/co/absa/atum/reader/{provider/AbstractHttpProvider.scala => server/GenericServerConnection.scala} (52%) rename reader/src/main/scala/za/co/absa/atum/reader/{provider/future/HttpProvider.scala => server/future/ServerConnection.scala} (53%) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala diff --git a/agent/README.md b/agent/README.md index 0d520106d..fda82be95 100644 --- a/agent/README.md +++ b/agent/README.md @@ -62,3 +62,38 @@ val sequenceOfMeasures = Seq(RecordCount("columnName"), RecordCount("other colum .format("CSV") .executeMeasures("checkpoint name")(sequenceOfMeasures) ``` + + +## Parent - Child relationship + +get = get partioning id(partioning: JSON): Long + +post = create partitioning(partioning: JSON, parent_partining_id: Long) +- parent definovan, nastavi vztah, a prevede measures z parenta na dite, nastavi flows +- parent neni defnivoan neni co resit, vznikne jen 1 flow + +??? - set parent - child partioning +- pouze prida vztahy ve flows, measures zustavaji stejne + + +### Operace Atum_Agent.get_context(partioning: JSON) +- GET - get_partioning_id (partioning url encoded) +IF 200 + - GET - get_measures(partioning_id) + - GET - get_additioanl_data(partioning_id) +ELSE + - POST - create_partioning(partioning, parent_partining_id = NULL) + +### Operace Atum_Agent.get_sub_context(partioning: JSON) +(zname parent_partioning_id) +- GET - get_partioning_id (partioning url encoded) +IF 200 + - PATCH - /partitionings/{partId}/parents + parent_id - child partioning + vrati 200 pokud opearce uspela + vrati 404 pokud parent nebo partId neexistuje + - + - GET - get_measures(partioning_id) + - GET - get_additioanl_data(partioning_id) +ELSE + - POST - create_partioning(partioning, parent_partining_id) diff --git a/database/src/main/postgres/runs/get_measurements.sql b/database/src/main/postgres/runs/get_measurements.sql new file mode 100644 index 000000000..eabad91e4 --- /dev/null +++ b/database/src/main/postgres/runs/get_measurements.sql @@ -0,0 +1,47 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION postgres.runs.get_measurements( + IN i_parameter TEXT, + OUT status INTEGER, + OUT status_text TEXT +) RETURNS record AS +$$ + ------------------------------------------------------------------------------- +-- +-- Function: postgres.runs.get_measurements([Function_Param_Count]) +-- [Description] +-- +-- Parameters: +-- i_parameter - +-- +-- Returns: +-- status - Status code +-- status_text - Status text +-- +-- Status codes: +-- 10 - OK +-- +------------------------------------------------------------------------------- +DECLARE +BEGIN + +END; +$$ + LANGUAGE plpgsql VOLATILE + SECURITY DEFINER; + +GRANT EXECUTE ON FUNCTION postgres.runs.get_measurements() TO [user]; diff --git a/project/Dependencies.scala b/project/Dependencies.scala index df9fa90c3..17d157bd4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -243,7 +243,7 @@ object Dependencies { "com.softwaremill.sttp.client3" %% "zio" % "3.9.8", "com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "3.9.8", "org.typelevel" %% "cats-effect" % "3.3.14", - "dev.zio" %% "zio" % "2.1.4", + "dev.zio" %% "zio" % "2.1.4" ) ++ testDependencies ++ jsonSerdeDependencies diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 09f1b0bf2..85905e59e 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -17,10 +17,9 @@ package za.co.absa.atum.reader import za.co.absa.atum.reader.basic.Reader -import za.co.absa.atum.reader.provider.Provider +import za.co.absa.atum.reader.server.GenericServerConnection -// TODO -class FlowReader[F[_]](override implicit val provider: Provider[F]) extends Reader[F]{ +class FlowReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F[_]]) extends Reader[F]{ def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index 263254934..23cd00f73 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -16,11 +16,10 @@ package za.co.absa.atum.reader -import cats.Monad import za.co.absa.atum.reader.basic.Reader -import za.co.absa.atum.reader.provider.Provider +import za.co.absa.atum.reader.server.GenericServerConnection -class PartitioningReader[F[_]: Monad](partitioning: Partitioning)(override implicit val provider: Provider[F[_]]) extends Reader[F] { +class PartitioningReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F[_]]) extends Reader[F] { def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index ba4c65678..d84ac3d56 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -16,6 +16,6 @@ package za.co.absa.atum.reader.basic -import za.co.absa.atum.reader.provider.Provider +import za.co.absa.atum.reader.server.GenericServerConnection -abstract class Reader[F[_]](implicit val provider: Provider[F[_]]) +abstract class Reader[F[_]](implicit val serverConnection: GenericServerConnection[F[_]]) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala deleted file mode 100644 index c6d631787..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/provider/Provider.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2024 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.provider - -/** - * A basic class for defining methods that will be providing data to readers. - */ -abstract class Provider[F[_]] { - // here will come abstract methods that are to return data to readers -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala deleted file mode 100644 index d8d1ba9b9..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/provider/io/HttpProvider.scala +++ /dev/null @@ -1,32 +0,0 @@ -package za.co.absa.atum.reader.provider.io - -import cats.effect.IO -import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.Response -import sttp.client3.armeria.cats.ArmeriaCatsBackend -import za.co.absa.atum.reader.provider.AbstractHttpProvider - -import scala.util.Try - -class HttpProvider(serverUrl: String) extends AbstractHttpProvider[IO](serverUrl) { - - def this(config: Config = ConfigFactory.load()) = { - this(AbstractHttpProvider.atumServerUrl(config )) - } - - override protected def executeRequest(requestFnc: RequestFunction): IO[Response[Either[String, String]]] = { - ArmeriaCatsBackend - .resource[IO]() - .use(requestFnc) - } - - override protected def mapResponse[R]( - response: IO[Response[Either[String, String]]], - mapperFnc: ResponseMapperFunction[R]): IO[Try[R]] = { - response.map(mapperFnc) - } -} - -object HttpProvider { - lazy implicit val httpProvider: HttpProvider = new HttpProvider() -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala deleted file mode 100644 index a1885447d..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/provider/zio/HttpProvider.scala +++ /dev/null @@ -1,24 +0,0 @@ -package za.co.absa.atum.reader.provider.zio - -import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.Response -import sttp.client3.armeria.zio.ArmeriaZioBackend -import za.co.absa.atum.reader.provider.AbstractHttpProvider -import zio.ZIO - -import scala.util.Try - -class HttpProvider(serverUrl: String) extends AbstractHttpProvider[ZIO](serverUrl) { - - def this(config: Config = ConfigFactory.load()) = { - this(AbstractHttpProvider.atumServerUrl(config )) - } - - - - override protected def executeRequest(requestFnc: RequestFunction): ZIO[Response[Either[String, String]]] = { - ArmeriaZioBackend.usingDefaultClient().map(requestFnc) - } - - override protected def mapResponse[R](response: ZIO[Response[Either[String, String]]], mapperFnc: ResponseMapperFunction[R]): ZIO[Try[R]] = ??? -} //TODO diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala similarity index 52% rename from reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala rename to reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala index 7eb811a91..e2c98b96c 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/provider/AbstractHttpProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala @@ -14,13 +14,16 @@ * limitations under the License. */ -package za.co.absa.atum.reader.provider +package za.co.absa.atum.reader.server import _root_.io.circe.parser.decode import _root_.io.circe.Decoder +import cats.Monad +import cats.implicits.toFunctorOps import com.typesafe.config.Config -import sttp.client3.{Response, SttpBackend, UriContext, basicRequest} +import sttp.client3.{Identity, RequestT, Response, UriContext, basicRequest} import za.co.absa.atum.reader.exceptions.RequestException +import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse import scala.util.{Failure, Try} @@ -28,33 +31,31 @@ import scala.util.{Failure, Try} * A HttpProvider is a component that is responsible for providing teh data to readers using REST API * @tparam F */ -abstract class AbstractHttpProvider[F[_]](val serverUrl: String) extends Provider[F] { - type RequestFunction = SttpBackend[F, Any] => F[Response[Either[String, String]]] - type ResponseMapperFunction[R] = Response[Either[String, String]] => Try[R] +abstract class GenericServerConnection[F[_]: Monad](val serverUrl: String) { - protected def executeRequest(requestFnc: RequestFunction): F[Response[Either[String, String]]] - protected def mapResponse[R](response: F[Response[Either[String, String]]], mapperFnc: ResponseMapperFunction[R]): F[Try[R]] + protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): F[ReaderResponse] - protected def query[R: Decoder](endpointUri: String): F[Try[R]] = { + def query[R: Decoder](endpointUri: String): F[Try[R]] = { val endpointToQuery = serverUrl + endpointUri val request = basicRequest .get(uri"$endpointToQuery") - val response = executeRequest(request.send(_)) - mapResponse(response, responseMapperFunction[R]) - } - - private def responseMapperFunction[R: Decoder](response: Response[Either[String, String]]): Try[R] = { - response.body match { - case Left(error) => Failure(RequestException(response.statusText, error, response.code, response.request)) - case Right(body) => decode[R](body).toTry + val response = executeRequest(request) + // using map instead of Circe's `asJson` to have own exception from a failed response + response.map { responseData => + responseData.body match { + case Left(error) => Failure(RequestException(responseData.statusText, error, responseData.code, responseData.request)) + case Right(body) => decode[R](body).toTry + } } } } -object AbstractHttpProvider { +object GenericServerConnection { final val UrlKey = "atum.server.url" + type ReaderResponse = Response[Either[String, String]] + def atumServerUrl(config: Config): String = { config.getString(UrlKey) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala similarity index 53% rename from reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala rename to reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala index e2cd69f61..ae3da6934 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/provider/future/HttpProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala @@ -14,33 +14,30 @@ * limitations under the License. */ -package za.co.absa.atum.reader.provider.future +package za.co.absa.atum.reader.server.future -import cats.implicits.catsStdInstancesForFuture import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.Response -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend -import za.co.absa.atum.reader.provider.AbstractHttpProvider import scala.concurrent.{ExecutionContext, Future} -import scala.util.Try +import sttp.client3.{Identity, RequestT, Response} +import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend +import za.co.absa.atum.reader.server.GenericServerConnection +import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse -class HttpProvider(serverUrl: String)(implicit executor: ExecutionContext) extends AbstractHttpProvider[Future](serverUrl) { +class ServerConnection(serverUrl: String)(implicit executor: ExecutionContext) extends GenericServerConnection[Future](serverUrl) { def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { - this(AbstractHttpProvider.atumServerUrl(config ))(executor) + this(GenericServerConnection.atumServerUrl(config ))(executor) } private val asyncHttpClientFutureBackend = AsyncHttpClientFutureBackend() - override protected def executeRequest(requestFnc: RequestFunction): Future[Response[Either[String, String]]] = { - requestFnc(asyncHttpClientFutureBackend) + override protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): Future[ReaderResponse] = { + request.send(asyncHttpClientFutureBackend) } +} - override protected def mapResponse[R]( - response: Future[Response[Either[String, String]]], - mapperFnc: ResponseMapperFunction[R]): Future[Try[R]] = { - response.map(mapperFnc) - } +object ServerConnection { + lazy implicit val serverConnection: ServerConnection = new ServerConnection()(ExecutionContext.Implicits.global) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala new file mode 100644 index 000000000..d337eaee1 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.io + +import cats.effect.IO +import com.typesafe.config.{Config, ConfigFactory} +import sttp.client3.{Identity, RequestT, Response} +import sttp.client3.armeria.cats.ArmeriaCatsBackend +import za.co.absa.atum.reader.server.GenericServerConnection +import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse + +class ServerConnection(serverUrl: String) extends GenericServerConnection[IO](serverUrl) { + + def this(config: Config = ConfigFactory.load()) = { + this(GenericServerConnection.atumServerUrl(config )) + } + + override protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): IO[ReaderResponse] = { + ArmeriaCatsBackend + .resource[IO]() + .use(request.send(_)) + } +} + +object ServerConnection { + lazy implicit val serverConnection: ServerConnection = new ServerConnection() +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala new file mode 100644 index 000000000..abebc83b3 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.zio + +import com.typesafe.config.{Config, ConfigFactory} +import sttp.client3.{Identity, RequestT, Response} +import sttp.client3.armeria.zio.ArmeriaZioBackend +import za.co.absa.atum.reader.server.GenericServerConnection +import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse +import zio.{Task, ZIO, _} + + +class ServerConnection(serverUrl: String) + extends GenericServerConnection[Task](serverUrl) { + + def this(config: Config = ConfigFactory.load()) = { + this(GenericServerConnection.atumServerUrl(config )) + } + + override protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): Task[ReaderResponse] = { + val x = ArmeriaZioBackend.usingDefaultClient().flatMap { backend => + val y: Task[Response[Either[String, String]]] = backend.send(request) + y + } + x + } +} + +object ServerConnection { + lazy implicit val serverConnection: ServerConnection = new ServerConnection() +} From 1ac2233bbbd7ec3ac07794e2b6c107cf2501b497 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 1 Nov 2024 01:24:04 +0100 Subject: [PATCH 12/52] * the first working commit --- README.md | 49 +++++++++++++++ project/Dependencies.scala | 61 +++++++++++++++--- project/Setup.scala | 30 +++++++-- .../za/co/absa/atum/reader/FlowReader.scala | 2 +- .../absa/atum/reader/PartitioningReader.scala | 2 +- .../za/co/absa/atum/reader/basic/Reader.scala | 2 +- .../server/GenericServerConnection.scala | 42 +++++-------- .../future/ArmeriaServerConnection.scala | 53 ++++++++++++++++ .../future/FutureServerConnection.scala | 44 +++++++++++++ .../future/HttpClientServerConnection.scala | 53 ++++++++++++++++ .../server/future/ServerConnection.scala | 43 ------------- .../server/io/ArmeriaServerConnection.scala | 62 +++++++++++++++++++ ...on.scala => ArmeriaServerConnection.scala} | 24 +++---- .../HttpClientServerConnection.scala} | 27 ++++---- .../server/zio/ZioServerConnection.scala | 31 ++++++++++ .../atum/reader/FlowReaderUnitTests.scala | 2 + .../reader/PartitioningReaderUnitTests.scala | 2 + .../zio/ZioServerConnectionUnitTests.scala | 39 ++++++++++++ 18 files changed, 459 insertions(+), 109 deletions(-) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala rename reader/src/main/scala/za/co/absa/atum/reader/server/zio/{ServerConnection.scala => ArmeriaServerConnection.scala} (59%) rename reader/src/main/scala/za/co/absa/atum/reader/server/{io/ServerConnection.scala => zio/HttpClientServerConnection.scala} (54%) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala diff --git a/README.md b/README.md index 46dfc975b..16189aa1f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ - [Measurement](#measurement) - [Checkpoint](#checkpoint) - [Data Flow](#data-flow) + - [Usage](#usage) + - [Reader](#reader-usage) - [How to generate Code coverage report](#how-to-generate-code-coverage-report) - [How to Run in IntelliJ](#how-to-run-in-intellij) - [How to Run Tests](#how-to-run-tests) @@ -156,6 +158,52 @@ We can even say, that `Checkpoint` is a result of particular `Measurements` (ver The journey of a dataset throughout various data transformations and pipelines. It captures the whole journey, even if it involves multiple applications or ETL pipelines. +## Usage + +### Reader usage +Reader module support several asynchronous http clients. The dependencies used for these clients are set as _optional_, +so the user of the module can decide which client to use and include only the necessary dependencies. + +The clients are: +#### Future based `HttpClientServerConnection` +Uses `java.net.http.HttpClient` to send requests to the server, therefore requires no additional dependencies. But works +only with Java 11 or higher. + +#### Future based `ArmeririaServerConnection` +Add +```scala +"com.softwaremill.sttp.client3" %% "armeria-backend" % "[version]" +``` +to your dependencies. + +#### Cats IO based `ArmeririaServerConnection` +Add +```scala +"org.typelevel." %% "cats-effect" % "[version]" +"com.softwaremill.sttp.client3" %% "armeria-backend-cats" % "[version]" // for cats-effect 3.x +// or +"com.softwaremill.sttp.client3" %% "armeria-backend-cats-ce2" % "[version]" // for cats-effect 2.x +``` +" +to your dependencies. + +#### ZIO based `HttpClientServerConnection` +Add +```scala +"com.softwaremill.sttp.client3" %% "zio" % "[version]" // for ZIO 2.x +"com.softwaremill.sttp.client3" %% "zio1" % "[version]" // for ZIO 1.x +``` +to your dependencies. + +#### ZIO based `ArmeririaServerConnection` +Add +```scala +"com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "[version]" // for ZIO 2.x +"com.softwaremill.sttp.client3" %% "armeria-backend-zio1" % "[version]" // for ZIO 1.x +``` +to your dependencies. + + ## How to generate Code coverage report ```sbt @@ -172,6 +220,7 @@ Code coverage wil be generated on path: To make this project runnable via IntelliJ, do the following: - Make sure that your configuration in `server/src/main/resources/reference.conf` is configured according to your needs +- When building within the UI be sure to have the option `-language:higherKinds` on in the compiler options ## How to Run Tests diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 17d157bd4..22f91d4a0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -38,8 +38,8 @@ object Dependencies { val sparkCommons = "0.6.1" - val sttpClient = "3.5.2" - val sttpCirceJson = "3.9.7" + val sttpClient = "3.10.1" + val sttpCirceJson = "3.10.1" val postgresql = "42.6.0" @@ -64,6 +64,8 @@ object Dependencies { val absaCommons = "2.0.0" + val catsEffect = "3.3.14" + def truncateVersion(version: String, parts: Int): String = { version.split("\\.").take(parts).mkString(".") } @@ -236,17 +238,58 @@ object Dependencies { } def readerDependencies(scalaVersion: Version): Seq[ModuleID] = { + val zioOrg = "dev.zio" + val sbtOrg = "com.github.sbt" + val sttpClient3Org = "com.softwaremill.sttp.client3" + val typeLevelOrg = "org.typelevel" + + // STTP core and Circe integration + lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpClient + lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpClient + + // Armeria Future backend + lazy val sttpArmeririaFutureBackend = sttpClient3Org %% "armeria-backend" % Versions.sttpClient % Optional + // Armeria Cats backend + lazy val sttpArmeririaCatsBackend = sttpClient3Org %% "armeria-backend-cats" % Versions.sttpClient % Optional + lazy val catsEffect = typeLevelOrg %% "cats-effect" % Versions.catsEffect % Optional + // Armeria Zio backend + lazy val sttpArmeririaZioBackend = sttpClient3Org %% "armeria-backend-zio" % Versions.sttpClient % Optional + // HttpClient Zio backend + lazy val sttpHttpClientZioBackend = sttpClient3Org %% "zio" % Versions.sttpClient % Optional + + // testing + lazy val zioTest = zioOrg %% "zio-test" % Versions.zio % Test + lazy val zioTestSbt = zioOrg %% "zio-test-sbt" % Versions.zio % Test + lazy val zioTestJunit = zioOrg %% "zio-test-junit" % Versions.zio % Test + lazy val sbtJunitInterface = sbtOrg % "junit-interface" % Versions.sbtJunitInterface % Test + Seq( - "com.softwaremill.sttp.client3" %% "core" % "3.9.7", - "com.softwaremill.sttp.client3" %% "async-http-client-backend-future" % "3.9.6", - "com.softwaremill.sttp.client3" %% "armeria-backend-cats" % "3.9.8", - "com.softwaremill.sttp.client3" %% "zio" % "3.9.8", - "com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "3.9.8", - "org.typelevel" %% "cats-effect" % "3.3.14", - "dev.zio" %% "zio" % "2.1.4" + sttpCore, + sttpCirce, + sttpArmeririaFutureBackend, + sttpArmeririaCatsBackend, + catsEffect, + sttpArmeririaZioBackend, + sttpHttpClientZioBackend, + zioTest, + zioTestSbt, + zioTestJunit, + sbtJunitInterface ) ++ testDependencies ++ jsonSerdeDependencies +// "com.softwaremill.sttp.client3" %% "core" % "3.9.7", +// "com.softwaremill.sttp.client3" %% "async-http-client-backend-future" % "3.9.6", +// "com.softwaremill.sttp.client3" %% "armeria-backend-cats" % "3.9.8", +// "com.softwaremill.sttp.client3" %% "armeria-backend" % "3.9.8", +// "com.softwaremill.sttp.client3" %% "zio" % "3.9.8", +// "com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "3.9.8", +// "org.typelevel" %% "cats-effect" % "3.3.14", +// "dev.zio" %% "zio" % "2.1.4", +// "dev.zio" %% "zio-interop-cats" % "23.1.0.1", +// "dev.zio" %% "zio-macros" % "2.1.4", +// "com.softwaremill.sttp.client3" %% "circe" % Versions.sttpCirceJson + } def databaseDependencies: Seq[ModuleID] = { diff --git a/project/Setup.scala b/project/Setup.scala index 14c3f8927..f1fa9b2ef 100644 --- a/project/Setup.scala +++ b/project/Setup.scala @@ -40,17 +40,37 @@ object Setup { val serverAndDbScalaVersion: Version = scala213 //covers REST server and database modules val clientSupportedScalaVersions: Seq[Version] = Seq(scala212, scala213) - val commonScalacOptions: Seq[String] = Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings") + val commonScalacOptions: Seq[String] = Seq( + "-unchecked", + "-deprecation", + "-feature", + "-Xfatal-warnings" + ) - val serverAndDbJavacOptions: Seq[String] = Seq("-source", "11", "-target", "11", "-Xlint") - val serverAndDbScalacOptions: Seq[String] = Seq("-Ymacro-annotations") + val serverAndDbJavacOptions: Seq[String] = Seq( + "-source", "11", + "-target", "11", + "-Xlint" + ) + val serverAndDbScalacOptions: Seq[String] = Seq( + "-language:higherKinds", + "-Ymacro-annotations" + ) val clientJavacOptions: Seq[String] = Seq("-source", "1.8", "-target", "1.8", "-Xlint") def clientScalacOptions(scalaVersion: Version): Seq[String] = { if (scalaVersion >= scala213) { - Seq("-release", "8", "-Ymacro-annotations") + Seq( + "-release", "8", + "-language:higherKinds", + "-Ymacro-annotations" + ) } else { - Seq("-release", "8", "-target:8") + Seq( + "-release", "8", + "-language:higherKinds", + "-target:8" + ) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 85905e59e..a6be49e5f 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import za.co.absa.atum.reader.basic.Reader import za.co.absa.atum.reader.server.GenericServerConnection -class FlowReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F[_]]) extends Reader[F]{ +class FlowReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F]) extends Reader[F]{ def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index 23cd00f73..7de8d3187 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import za.co.absa.atum.reader.basic.Reader import za.co.absa.atum.reader.server.GenericServerConnection -class PartitioningReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F[_]]) extends Reader[F] { +class PartitioningReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F]) extends Reader[F] { def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index d84ac3d56..db8ef1d65 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -18,4 +18,4 @@ package za.co.absa.atum.reader.basic import za.co.absa.atum.reader.server.GenericServerConnection -abstract class Reader[F[_]](implicit val serverConnection: GenericServerConnection[F[_]]) +abstract class Reader[F[_]](implicit val serverConnection: GenericServerConnection[F]) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala index e2c98b96c..2a1cd1d3b 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala @@ -16,45 +16,37 @@ package za.co.absa.atum.reader.server -import _root_.io.circe.parser.decode import _root_.io.circe.Decoder -import cats.Monad -import cats.implicits.toFunctorOps +import _root_.io.circe.{Error => circeError} import com.typesafe.config.Config -import sttp.client3.{Identity, RequestT, Response, UriContext, basicRequest} -import za.co.absa.atum.reader.exceptions.RequestException -import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse +import sttp.client3.{Identity, RequestT, ResponseException, basicRequest} +import sttp.model.Uri +import sttp.client3.circe._ -import scala.util.{Failure, Try} +import za.co.absa.atum.model.envelopes.ErrorResponse +import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult -/** - * A HttpProvider is a component that is responsible for providing teh data to readers using REST API - * @tparam F - */ -abstract class GenericServerConnection[F[_]: Monad](val serverUrl: String) { +abstract class GenericServerConnection[F[_]](val serverUrl: String) { - protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): F[ReaderResponse] + protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): F[RequestResult[R]] - def query[R: Decoder](endpointUri: String): F[Try[R]] = { + def getQuery[R: Decoder](endpointUri: String, params: Map[String, String] = Map.empty): F[RequestResult[R]] = { val endpointToQuery = serverUrl + endpointUri - val request = basicRequest - .get(uri"$endpointToQuery") - val response = executeRequest(request) - // using map instead of Circe's `asJson` to have own exception from a failed response - response.map { responseData => - responseData.body match { - case Left(error) => Failure(RequestException(responseData.statusText, error, responseData.code, responseData.request)) - case Right(body) => decode[R](body).toTry - } - } + val uri = Uri.unsafeParse(endpointToQuery).addParams(params) + val request: RequestT[Identity, RequestResult[R], Any] = basicRequest + .get(uri) + .response(asJsonEither[ErrorResponse, R]) + executeRequest(request) } + def close(): F[Unit] + } object GenericServerConnection { final val UrlKey = "atum.server.url" - type ReaderResponse = Response[Either[String, String]] + type RequestResult[R] = Either[ResponseException[ErrorResponse, circeError], R] def atumServerUrl(config: Config): String = { config.getString(UrlKey) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala new file mode 100644 index 000000000..1d97a4b55 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.future + +import com.typesafe.config.{Config, ConfigFactory} +import scala.concurrent.{ExecutionContext, Future} +import sttp.client3.armeria.future.ArmeriaFutureBackend +import sttp.client3.SttpBackend + +import za.co.absa.atum.reader.server.GenericServerConnection + +class ArmeriaServerConnection private(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) + extends FutureServerConnection(serverUrl, closeable) { + + def this(serverUrl: String)(implicit executor: ExecutionContext) = { + this(serverUrl, true)(executor) + } + + def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { + this(GenericServerConnection.atumServerUrl(config))(executor) + } + + override protected val backend: SttpBackend[Future, Any] = ArmeriaFutureBackend() + +} + +object ArmeriaServerConnection { + lazy implicit val serverConnection: ArmeriaServerConnection = new ArmeriaServerConnection()(ExecutionContext.Implicits.global) + + def use[R](serverUrl: String)(fnc: ArmeriaServerConnection => Future[R]) + (implicit executor: ExecutionContext): Future[R] = { + val serverConnection = new ArmeriaServerConnection(serverUrl, false) + try { + fnc(serverConnection) + } finally { + serverConnection.backend.close() + } + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala new file mode 100644 index 000000000..a13fa8769 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala @@ -0,0 +1,44 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.future + +import scala.concurrent.{ExecutionContext, Future} +import sttp.client3.{Identity, RequestT, SttpBackend} + +import za.co.absa.atum.reader.server.GenericServerConnection +import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult + + +abstract class FutureServerConnection(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) + extends GenericServerConnection[Future](serverUrl) { + + protected val backend: SttpBackend[Future, Any] + + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Future[RequestResult[R]] = { + request.send(backend).map(_.body) + } + + override def close(): Future[Unit] = { + if (closeable) { + backend.close() + } else { + Future.successful(()) + } + } + +} + diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala new file mode 100644 index 000000000..123c696d2 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.future + +import com.typesafe.config.{Config, ConfigFactory} +import scala.concurrent.{ExecutionContext, Future} +import sttp.client3.{HttpClientFutureBackend, SttpBackend} + +import za.co.absa.atum.reader.server.GenericServerConnection + + +class HttpClientServerConnection private(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) + extends FutureServerConnection(serverUrl, closeable) { + + def this(serverUrl: String)(implicit executor: ExecutionContext) = { + this(serverUrl, true)(executor) + } + + def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { + this(GenericServerConnection.atumServerUrl(config))(executor) + } + + override protected val backend: SttpBackend[Future, Any] = HttpClientFutureBackend() + +} + +object HttpClientServerConnection { + lazy implicit val serverConnection: FutureServerConnection = new HttpClientServerConnection()(ExecutionContext.Implicits.global) + + def use[R](serverUrl: String)(fnc: HttpClientServerConnection => Future[R]) + (implicit executor: ExecutionContext): Future[R] = { + val serverConnection = new HttpClientServerConnection(serverUrl) + try { + fnc(serverConnection) + } finally { + serverConnection.close() + } + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala deleted file mode 100644 index ae3da6934..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ServerConnection.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2024 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.future - -import com.typesafe.config.{Config, ConfigFactory} - -import scala.concurrent.{ExecutionContext, Future} -import sttp.client3.{Identity, RequestT, Response} -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend -import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse - - -class ServerConnection(serverUrl: String)(implicit executor: ExecutionContext) extends GenericServerConnection[Future](serverUrl) { - - def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { - this(GenericServerConnection.atumServerUrl(config ))(executor) - } - - private val asyncHttpClientFutureBackend = AsyncHttpClientFutureBackend() - - override protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): Future[ReaderResponse] = { - request.send(asyncHttpClientFutureBackend) - } -} - -object ServerConnection { - lazy implicit val serverConnection: ServerConnection = new ServerConnection()(ExecutionContext.Implicits.global) -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala new file mode 100644 index 000000000..0a267d1ca --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.io + +import cats.effect.IO +import com.typesafe.config.{Config, ConfigFactory} +import sttp.client3.{Identity, RequestT, SttpBackend} +import sttp.client3.armeria.cats.ArmeriaCatsBackend + +import za.co.absa.atum.reader.server.GenericServerConnection +import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult + + +class ArmeriaServerConnection protected(serverUrl: String, backend: SttpBackend[IO, Any], closeable: Boolean) + extends GenericServerConnection[IO](serverUrl) { + + def this(mserverUrl: String) = { + this(mserverUrl, ArmeriaCatsBackend[IO](), closeable = true) + } + + def this(config: Config = ConfigFactory.load()) = { + this(GenericServerConnection.atumServerUrl(config ), ArmeriaCatsBackend[IO](), closeable = true) + } + + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): IO[RequestResult[R]] = { + request.send(backend).map(_.body) + } + + override def close(): IO[Unit] = { + if (closeable) { + backend.close() + } else { + IO.unit + } + } + +} + +object ArmeriaServerConnection { + lazy implicit val serverConnection: ArmeriaServerConnection = new ArmeriaServerConnection() + + def use[R](serverUrl: String)(fnc: ArmeriaServerConnection => IO[R]): IO[R] = { + ArmeriaCatsBackend.resource[IO]().use{backend => + val serverConnection = new ArmeriaServerConnection(serverUrl, backend, false) + fnc(serverConnection) + } + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala similarity index 59% rename from reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala rename to reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala index abebc83b3..f469000ad 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala @@ -17,29 +17,29 @@ package za.co.absa.atum.reader.server.zio import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT, Response} +import sttp.client3.{Identity, RequestT} import sttp.client3.armeria.zio.ArmeriaZioBackend +import zio.Task + import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse -import zio.{Task, ZIO, _} +import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult -class ServerConnection(serverUrl: String) - extends GenericServerConnection[Task](serverUrl) { +class ArmeriaServerConnection(serverUrl: String) extends ZioServerConnection(serverUrl) { def this(config: Config = ConfigFactory.load()) = { this(GenericServerConnection.atumServerUrl(config )) } - override protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): Task[ReaderResponse] = { - val x = ArmeriaZioBackend.usingDefaultClient().flatMap { backend => - val y: Task[Response[Either[String, String]]] = backend.send(request) - y + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[RequestResult[R]] = { + ArmeriaZioBackend.usingDefaultClient().flatMap { backend => + val response = backend.send(request) + response.map(_.body) } - x } + } -object ServerConnection { - lazy implicit val serverConnection: ServerConnection = new ServerConnection() +object ArmeriaServerConnection { + lazy implicit val serverConnection: ArmeriaServerConnection = new ArmeriaServerConnection() } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala similarity index 54% rename from reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala rename to reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala index d337eaee1..c80ec58ac 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala @@ -14,28 +14,31 @@ * limitations under the License. */ -package za.co.absa.atum.reader.server.io +package za.co.absa.atum.reader.server.zio -import cats.effect.IO import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT, Response} -import sttp.client3.armeria.cats.ArmeriaCatsBackend +import sttp.client3.{Identity, RequestT} +import sttp.client3.httpclient.zio.HttpClientZioBackend import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.ReaderResponse +import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult +import zio.Task -class ServerConnection(serverUrl: String) extends GenericServerConnection[IO](serverUrl) { + +class HttpClientServerConnection(serverUrl: String) extends ZioServerConnection(serverUrl) { def this(config: Config = ConfigFactory.load()) = { this(GenericServerConnection.atumServerUrl(config )) } - override protected def executeRequest(request: RequestT[Identity, Either[String, String], Any]): IO[ReaderResponse] = { - ArmeriaCatsBackend - .resource[IO]() - .use(request.send(_)) + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[RequestResult[R]] = { + HttpClientZioBackend().flatMap { backend => + val response = backend.send(request) + response.map(_.body) + } } + } -object ServerConnection { - lazy implicit val serverConnection: ServerConnection = new ServerConnection() +object HttpClientServerConnection { + lazy implicit val serverConnection: HttpClientServerConnection = new HttpClientServerConnection() } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala new file mode 100644 index 000000000..9e108fd16 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.zio + +import zio.{Exit, Task} + +import za.co.absa.atum.reader.server.GenericServerConnection + +abstract class ZioServerConnection(serverUrl: String) extends GenericServerConnection[Task](serverUrl) { + + override def close(): Task[Unit] = { + Exit.succeed(()) + } + +} + + diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index 3800801a2..bc8de7a84 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -18,6 +18,8 @@ package za.co.absa.atum.reader import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.reader.server.future.ArmeriaServerConnection.serverConnection + class FlowReaderUnitTests extends AnyFunSuiteLike { test("foo") { val expected = new FlowReader().foo() diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index c4221d8c5..6fdd72394 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -18,6 +18,8 @@ package za.co.absa.atum.reader import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.reader.server.future.ArmeriaServerConnection.serverConnection + class PartitioningReaderUnitTests extends AnyFunSuiteLike { test("foo") { val expected = new PartitioningReader().foo() diff --git a/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala new file mode 100644 index 000000000..cdfd529b6 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server.zio + +import sttp.client3.{Identity, RequestT} +import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult +import zio.test.ZIOSpecDefault +import zio._ +import zio.test._ + +object ZioServerConnectionUnitTests extends ZIOSpecDefault { + override def spec: Spec[TestEnvironment with Scope, Any] = { + suite("ZioServerConnection")( + test("close does nothing and succeeds") { + val connection = new ZioServerConnection("foo.bar") { + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[RequestResult[R]] = ??? + } + val expected: Unit = () + for { + result <- connection.close() + } yield assertTrue(result == expected) + } + ) + } +} From e6dcb526b70678e0f30b3ad86891d8bf8e2bb5a0 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 1 Nov 2024 09:56:15 +0100 Subject: [PATCH 13/52] * Removed temporary notes --- agent/README.md | 45 ++++++--------------------------------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/agent/README.md b/agent/README.md index fda82be95..04fe5b67b 100644 --- a/agent/README.md +++ b/agent/README.md @@ -7,26 +7,28 @@ ## Usage -Create multiple `AtumContext` with different control measures to be applied +Create multiple `AtumContext` with different control measures to be applied ### Option 1 ```scala val atumContextInstanceWithRecordCount = AtumContext(processor = processor) + .withMeasureAdded(RecordCount(MockMeasureNames.recordCount1, controlCol = "id")) .withMeasureAdded(RecordCount(MockMeasureNames.recordCount1, measuredColumn = "id")) val atumContextWithSalaryAbsMeasure = atumContextInstanceWithRecordCount + .withMeasureAdded(AbsSumOfValuesOfColumn(controlCol = "salary")) .withMeasureAdded(AbsSumOfValuesOfColumn(measuredColumn = "salary")) ``` -### Option 2 +### Option 2 Use `AtumPartitions` to get an `AtumContext` from the service using the `AtumAgent`. ```scala val atumContext1 = AtumAgent.createAtumContext(atumPartition) ``` #### AtumPartitions -A list of key values that maintains the order of arrival of the items, the `AtumService` -is able to deliver the correct `AtumContext` according to the `AtumPartitions` we give it. +A list of key values that maintains the order of arrival of the items, the `AtumService` +is able to deliver the correct `AtumContext` according to the `AtumPartitions` we give it. ```scala val atumPartitions = AtumPartitions().withPartitions(ListMap("name" -> "partition-name", "country" -> "SA", "gender" -> "female" )) @@ -62,38 +64,3 @@ val sequenceOfMeasures = Seq(RecordCount("columnName"), RecordCount("other colum .format("CSV") .executeMeasures("checkpoint name")(sequenceOfMeasures) ``` - - -## Parent - Child relationship - -get = get partioning id(partioning: JSON): Long - -post = create partitioning(partioning: JSON, parent_partining_id: Long) -- parent definovan, nastavi vztah, a prevede measures z parenta na dite, nastavi flows -- parent neni defnivoan neni co resit, vznikne jen 1 flow - -??? - set parent - child partioning -- pouze prida vztahy ve flows, measures zustavaji stejne - - -### Operace Atum_Agent.get_context(partioning: JSON) -- GET - get_partioning_id (partioning url encoded) -IF 200 - - GET - get_measures(partioning_id) - - GET - get_additioanl_data(partioning_id) -ELSE - - POST - create_partioning(partioning, parent_partining_id = NULL) - -### Operace Atum_Agent.get_sub_context(partioning: JSON) -(zname parent_partioning_id) -- GET - get_partioning_id (partioning url encoded) -IF 200 - - PATCH - /partitionings/{partId}/parents - parent_id - child partioning - vrati 200 pokud opearce uspela - vrati 404 pokud parent nebo partId neexistuje - - - - GET - get_measures(partioning_id) - - GET - get_additioanl_data(partioning_id) -ELSE - - POST - create_partioning(partioning, parent_partining_id) From 6968b0257192667ade38cbf90f7fda7f0eb991c2 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 1 Nov 2024 12:35:09 +0100 Subject: [PATCH 14/52] * introduced `MonadError` into the `GenericServerConnection` * fixed license headers --- .github/workflows/test_filenames_check.yml | 3 ++- .../testing/implicits/StringImplicits.scala | 25 +++++++++++++++++++ .../utils/SerializationUtilsUnitTests.scala | 10 +------- .../za/co/absa/atum/reader/basic/Reader.scala | 2 +- .../reader/exceptions/ReaderException.scala | 2 +- .../reader/exceptions/RequestException.scala | 2 +- .../server/GenericServerConnection.scala | 14 ++++++----- .../future/ArmeriaServerConnection.scala | 2 +- .../future/FutureServerConnection.scala | 14 +++++------ .../future/HttpClientServerConnection.scala | 2 +- .../server/io/ArmeriaServerConnection.scala | 12 ++++----- .../server/zio/ArmeriaServerConnection.scala | 10 +++----- .../zio/HttpClientServerConnection.scala | 9 +++---- .../server/zio/ZioServerConnection.scala | 8 +++--- 14 files changed, 67 insertions(+), 48 deletions(-) create mode 100644 model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index d3e24ee2f..b870c1866 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -39,6 +39,7 @@ jobs: excludes: | server/src/test/scala/za/co/absa/atum/server/api/TestData.scala, server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala, - server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala + server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala, + model/src/test/scala/za/co/absa/atum/model/testing/* verbose-logging: 'false' fail-on-violation: 'true' diff --git a/model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala b/model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala new file mode 100644 index 000000000..1eac82788 --- /dev/null +++ b/model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.testing.implicits + +object StringImplicits { + implicit class StringLinearization(val str: String) extends AnyVal { + def linearize: String = { + str.stripMargin.replace("\r", "").replace("\n", "") + } + } +} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 729392934..e34aa2401 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -21,7 +21,7 @@ import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ -import za.co.absa.atum.model.utils.SerializationUtilsTest.StringLinearization +import za.co.absa.atum.model.testing.implicits.StringImplicits.StringLinearization import java.time.{ZoneId, ZoneOffset, ZonedDateTime} import java.util.UUID @@ -436,11 +436,3 @@ class SerializationUtilsUnitTests extends AnyFlatSpecLike { } } - -object SerializationUtilsTest { - implicit class StringLinearization(val str: String) extends AnyVal { - def linearize: String = { - str.stripMargin.replace("\r", "").replace("\n", "") - } - } -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index db8ef1d65..57c06e923 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala index d668bd39b..5ec9b921b 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala index c5130cd59..af33dbca2 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala index 2a1cd1d3b..32b584a9f 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,16 +19,17 @@ package za.co.absa.atum.reader.server import _root_.io.circe.Decoder import _root_.io.circe.{Error => circeError} import com.typesafe.config.Config -import sttp.client3.{Identity, RequestT, ResponseException, basicRequest} +import sttp.client3.{Identity, RequestT, Response, ResponseException, basicRequest} import sttp.model.Uri import sttp.client3.circe._ - +import sttp.monad.MonadError +import sttp.monad.syntax._ import za.co.absa.atum.model.envelopes.ErrorResponse import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult -abstract class GenericServerConnection[F[_]](val serverUrl: String) { +abstract class GenericServerConnection[F[_]: MonadError](val serverUrl: String) { - protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): F[RequestResult[R]] + protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): F[Response[RequestResult[R]]] def getQuery[R: Decoder](endpointUri: String, params: Map[String, String] = Map.empty): F[RequestResult[R]] = { val endpointToQuery = serverUrl + endpointUri @@ -36,7 +37,8 @@ abstract class GenericServerConnection[F[_]](val serverUrl: String) { val request: RequestT[Identity, RequestResult[R], Any] = basicRequest .get(uri) .response(asJsonEither[ErrorResponse, R]) - executeRequest(request) + val response = executeRequest(request) + response.map(_.body) } def close(): F[Unit] diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala index 1d97a4b55..48f192ff0 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala index a13fa8769..f46770d2d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,20 +16,20 @@ package za.co.absa.atum.reader.server.future -import scala.concurrent.{ExecutionContext, Future} -import sttp.client3.{Identity, RequestT, SttpBackend} +import scala.concurrent.{ExecutionContext, Future} +import sttp.client3.{Identity, RequestT, Response, SttpBackend} +import sttp.monad.FutureMonad import za.co.absa.atum.reader.server.GenericServerConnection import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult - abstract class FutureServerConnection(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) - extends GenericServerConnection[Future](serverUrl) { + extends GenericServerConnection[Future](serverUrl)(new FutureMonad) { protected val backend: SttpBackend[Future, Any] - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Future[RequestResult[R]] = { - request.send(backend).map(_.body) + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Future[Response[RequestResult[R]]] = { + request.send(backend) } override def close(): Future[Unit] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala index 123c696d2..6e09c5d3e 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala index 0a267d1ca..0c0885f46 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,15 +18,15 @@ package za.co.absa.atum.reader.server.io import cats.effect.IO import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT, SttpBackend} +import sttp.client3.{Identity, RequestT, Response, SttpBackend} import sttp.client3.armeria.cats.ArmeriaCatsBackend - +import sttp.client3.impl.cats.CatsMonadAsyncError import za.co.absa.atum.reader.server.GenericServerConnection import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult class ArmeriaServerConnection protected(serverUrl: String, backend: SttpBackend[IO, Any], closeable: Boolean) - extends GenericServerConnection[IO](serverUrl) { + extends GenericServerConnection[IO](serverUrl)(new CatsMonadAsyncError[IO]) { def this(mserverUrl: String) = { this(mserverUrl, ArmeriaCatsBackend[IO](), closeable = true) @@ -36,8 +36,8 @@ class ArmeriaServerConnection protected(serverUrl: String, backend: SttpBackend[ this(GenericServerConnection.atumServerUrl(config ), ArmeriaCatsBackend[IO](), closeable = true) } - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): IO[RequestResult[R]] = { - request.send(backend).map(_.body) + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): IO[Response[RequestResult[R]]] = { + request.send(backend) } override def close(): IO[Unit] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala index f469000ad..1c993b303 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,9 @@ package za.co.absa.atum.reader.server.zio import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT} +import sttp.client3.{Identity, RequestT, Response} import sttp.client3.armeria.zio.ArmeriaZioBackend import zio.Task - import za.co.absa.atum.reader.server.GenericServerConnection import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult @@ -31,10 +30,9 @@ class ArmeriaServerConnection(serverUrl: String) extends ZioServerConnection(ser this(GenericServerConnection.atumServerUrl(config )) } - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[RequestResult[R]] = { + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = { ArmeriaZioBackend.usingDefaultClient().flatMap { backend => - val response = backend.send(request) - response.map(_.body) + backend.send(request) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala index c80ec58ac..6712d5e1f 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package za.co.absa.atum.reader.server.zio import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT} +import sttp.client3.{Identity, RequestT, Response} import sttp.client3.httpclient.zio.HttpClientZioBackend import za.co.absa.atum.reader.server.GenericServerConnection import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult @@ -30,10 +30,9 @@ class HttpClientServerConnection(serverUrl: String) extends ZioServerConnection( this(GenericServerConnection.atumServerUrl(config )) } - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[RequestResult[R]] = { + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = { HttpClientZioBackend().flatMap { backend => - val response = backend.send(request) - response.map(_.body) + backend.send(request) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala index 9e108fd16..cf5b5fe19 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,12 @@ package za.co.absa.atum.reader.server.zio import zio.{Exit, Task} - +import sttp.client3 +import sttp.client3.impl.zio.RIOMonadAsyncError import za.co.absa.atum.reader.server.GenericServerConnection -abstract class ZioServerConnection(serverUrl: String) extends GenericServerConnection[Task](serverUrl) { + +abstract class ZioServerConnection(serverUrl: String) extends GenericServerConnection[Task](serverUrl)(new RIOMonadAsyncError[Any]) { override def close(): Task[Unit] = { Exit.succeed(()) From b9bacefa31c1ce56396e9ce70ddc25cf3104cadb Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 1 Nov 2024 13:15:38 +0100 Subject: [PATCH 15/52] * Fixed UTs --- reader/src/test/resources/reference.conf | 15 +++++++++++++++ .../server/zio/ZioServerConnectionUnitTests.scala | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 reader/src/test/resources/reference.conf diff --git a/reader/src/test/resources/reference.conf b/reader/src/test/resources/reference.conf new file mode 100644 index 000000000..4357da397 --- /dev/null +++ b/reader/src/test/resources/reference.conf @@ -0,0 +1,15 @@ +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The REST API URI of the atum server +atum.server.url="http://localhost:8080" diff --git a/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala index cdfd529b6..4315afb70 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala @@ -16,7 +16,7 @@ package za.co.absa.atum.reader.server.zio -import sttp.client3.{Identity, RequestT} +import sttp.client3.{Identity, RequestT, Response} import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult import zio.test.ZIOSpecDefault import zio._ @@ -27,7 +27,7 @@ object ZioServerConnectionUnitTests extends ZIOSpecDefault { suite("ZioServerConnection")( test("close does nothing and succeeds") { val connection = new ZioServerConnection("foo.bar") { - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[RequestResult[R]] = ??? + override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = ??? } val expected: Unit = () for { From bbb1e7f4e88c674a7f5e70d3387f17567c5ddf7f Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 4 Nov 2024 09:55:40 +0100 Subject: [PATCH 16/52] * trying to get rid of Java 11 dependency --- project/Dependencies.scala | 16 +---- project/VersionAxes.scala | 5 ++ .../future/HttpClientServerConnection.scala | 71 ++++++++++--------- .../zio/HttpClientServerConnection.scala | 51 ++++++------- 4 files changed, 69 insertions(+), 74 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 22f91d4a0..4b822a175 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -255,7 +255,7 @@ object Dependencies { // Armeria Zio backend lazy val sttpArmeririaZioBackend = sttpClient3Org %% "armeria-backend-zio" % Versions.sttpClient % Optional // HttpClient Zio backend - lazy val sttpHttpClientZioBackend = sttpClient3Org %% "zio" % Versions.sttpClient % Optional +// lazy val sttpHttpClientZioBackend = sttpClient3Org %% "zio" % Versions.sttpClient % Optional TODO #298 needs Java 11 cross-build // testing lazy val zioTest = zioOrg %% "zio-test" % Versions.zio % Test @@ -270,7 +270,7 @@ object Dependencies { sttpArmeririaCatsBackend, catsEffect, sttpArmeririaZioBackend, - sttpHttpClientZioBackend, +// sttpHttpClientZioBackend, TODO #298 needs Java 11 cross-build zioTest, zioTestSbt, zioTestJunit, @@ -278,18 +278,6 @@ object Dependencies { ) ++ testDependencies ++ jsonSerdeDependencies -// "com.softwaremill.sttp.client3" %% "core" % "3.9.7", -// "com.softwaremill.sttp.client3" %% "async-http-client-backend-future" % "3.9.6", -// "com.softwaremill.sttp.client3" %% "armeria-backend-cats" % "3.9.8", -// "com.softwaremill.sttp.client3" %% "armeria-backend" % "3.9.8", -// "com.softwaremill.sttp.client3" %% "zio" % "3.9.8", -// "com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "3.9.8", -// "org.typelevel" %% "cats-effect" % "3.3.14", -// "dev.zio" %% "zio" % "2.1.4", -// "dev.zio" %% "zio-interop-cats" % "23.1.0.1", -// "dev.zio" %% "zio-macros" % "2.1.4", -// "com.softwaremill.sttp.client3" %% "circe" % Versions.sttpCirceJson - } def databaseDependencies: Seq[ModuleID] = { diff --git a/project/VersionAxes.scala b/project/VersionAxes.scala index a52aec46c..d55480426 100644 --- a/project/VersionAxes.scala +++ b/project/VersionAxes.scala @@ -32,6 +32,11 @@ object VersionAxes { override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") } + case class JavaVersionAxis(javaVersion: String) extends sbt.VirtualAxis.WeakAxis { + override val directorySuffix = s"-jdk$javaVersion" + override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") + } + private def camelCaseToLowerDashCase(origName: String): String = { origName .replaceAll("([A-Z])", "-$1") diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala index 6e09c5d3e..c05518ce4 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala @@ -16,38 +16,39 @@ package za.co.absa.atum.reader.server.future -import com.typesafe.config.{Config, ConfigFactory} -import scala.concurrent.{ExecutionContext, Future} -import sttp.client3.{HttpClientFutureBackend, SttpBackend} - -import za.co.absa.atum.reader.server.GenericServerConnection - - -class HttpClientServerConnection private(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) - extends FutureServerConnection(serverUrl, closeable) { - - def this(serverUrl: String)(implicit executor: ExecutionContext) = { - this(serverUrl, true)(executor) - } - - def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { - this(GenericServerConnection.atumServerUrl(config))(executor) - } - - override protected val backend: SttpBackend[Future, Any] = HttpClientFutureBackend() - -} - -object HttpClientServerConnection { - lazy implicit val serverConnection: FutureServerConnection = new HttpClientServerConnection()(ExecutionContext.Implicits.global) - - def use[R](serverUrl: String)(fnc: HttpClientServerConnection => Future[R]) - (implicit executor: ExecutionContext): Future[R] = { - val serverConnection = new HttpClientServerConnection(serverUrl) - try { - fnc(serverConnection) - } finally { - serverConnection.close() - } - } -} +// TODO #298 needs Java 11 cross-build +//import com.typesafe.config.{Config, ConfigFactory} +//import scala.concurrent.{ExecutionContext, Future} +//import sttp.client3.{HttpClientFutureBackend, SttpBackend} +// +//import za.co.absa.atum.reader.server.GenericServerConnection +// +// +//class HttpClientServerConnection private(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) +// extends FutureServerConnection(serverUrl, closeable) { +// +// def this(serverUrl: String)(implicit executor: ExecutionContext) = { +// this(serverUrl, true)(executor) +// } +// +// def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { +// this(GenericServerConnection.atumServerUrl(config))(executor) +// } +// +// override protected val backend: SttpBackend[Future, Any] = HttpClientFutureBackend() +// +//} +// +//object HttpClientServerConnection { +// lazy implicit val serverConnection: FutureServerConnection = new HttpClientServerConnection()(ExecutionContext.Implicits.global) +// +// def use[R](serverUrl: String)(fnc: HttpClientServerConnection => Future[R]) +// (implicit executor: ExecutionContext): Future[R] = { +// val serverConnection = new HttpClientServerConnection(serverUrl) +// try { +// fnc(serverConnection) +// } finally { +// serverConnection.close() +// } +// } +//} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala index 6712d5e1f..798fabac8 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala @@ -16,28 +16,29 @@ package za.co.absa.atum.reader.server.zio -import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT, Response} -import sttp.client3.httpclient.zio.HttpClientZioBackend -import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult -import zio.Task - - -class HttpClientServerConnection(serverUrl: String) extends ZioServerConnection(serverUrl) { - - def this(config: Config = ConfigFactory.load()) = { - this(GenericServerConnection.atumServerUrl(config )) - } - - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = { - HttpClientZioBackend().flatMap { backend => - backend.send(request) - } - } - -} - -object HttpClientServerConnection { - lazy implicit val serverConnection: HttpClientServerConnection = new HttpClientServerConnection() -} +//TODO #298 needs Java 11 cross-build +//import com.typesafe.config.{Config, ConfigFactory} +//import sttp.client3.{Identity, RequestT, Response} +//import sttp.client3.httpclient.zio.HttpClientZioBackend +//import za.co.absa.atum.reader.server.GenericServerConnection +//import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult +//import zio.Task +// +// +//class HttpClientServerConnection(serverUrl: String) extends ZioServerConnection(serverUrl) { +// +// def this(config: Config = ConfigFactory.load()) = { +// this(GenericServerConnection.atumServerUrl(config )) +// } +// +// override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = { +// HttpClientZioBackend().flatMap { backend => +// backend.send(request) +// } +// } +// +//} +// +//object HttpClientServerConnection { +// lazy implicit val serverConnection: HttpClientServerConnection = new HttpClientServerConnection() +//} From 33e66287641fda68e30c1b3b3e268e69ac377b4d Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 5 Nov 2024 00:51:34 +0100 Subject: [PATCH 17/52] * Downgraded sttpClient --- project/Dependencies.scala | 2 +- .../co/absa/atum/reader/server/zio/ZioServerConnection.scala | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4b822a175..2e1dedbbd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -38,7 +38,7 @@ object Dependencies { val sparkCommons = "0.6.1" - val sttpClient = "3.10.1" + val sttpClient = "3.6.2" val sttpCirceJson = "3.10.1" val postgresql = "42.6.0" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala index cf5b5fe19..42494d594 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala @@ -16,8 +16,7 @@ package za.co.absa.atum.reader.server.zio -import zio.{Exit, Task} -import sttp.client3 +import zio.{Task, ZIO} import sttp.client3.impl.zio.RIOMonadAsyncError import za.co.absa.atum.reader.server.GenericServerConnection @@ -25,7 +24,7 @@ import za.co.absa.atum.reader.server.GenericServerConnection abstract class ZioServerConnection(serverUrl: String) extends GenericServerConnection[Task](serverUrl)(new RIOMonadAsyncError[Any]) { override def close(): Task[Unit] = { - Exit.succeed(()) + ZIO.unit } } From f7ced56126e353fd37858fe271a027d6d2e14093 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 5 Nov 2024 01:27:59 +0100 Subject: [PATCH 18/52] * further downgrade --- project/Dependencies.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2e1dedbbd..fdc72c1f3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -38,8 +38,8 @@ object Dependencies { val sparkCommons = "0.6.1" - val sttpClient = "3.6.2" - val sttpCirceJson = "3.10.1" + val sttpClient = "3.5.2" //last supported version for Java 8 + val sttpCirceJson = "3.9.7" val postgresql = "42.6.0" From ca2116bdeb96048f6f6338e27aee2d30cc316345 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 6 Nov 2024 12:08:39 +0100 Subject: [PATCH 19/52] * Removed exceptions --- .../reader/exceptions/ReaderException.scala | 19 -------------- .../reader/exceptions/RequestException.scala | 26 ------------------- 2 files changed, 45 deletions(-) delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala deleted file mode 100644 index 5ec9b921b..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.exceptions - -abstract class ReaderException(message: String) extends Exception(message) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala deleted file mode 100644 index af33dbca2..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.exceptions - -import sttp.model.{RequestMetadata, StatusCode} - -case class RequestException ( - message: String, - responseBody: String, - statusCode: StatusCode, - request: RequestMetadata) - extends ReaderException(message) From e5e6f632792036ce31fe6889b9bced698a661078 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 6 Nov 2024 12:55:05 +0100 Subject: [PATCH 20/52] * commented out parts of README.md which are not yet part of the code --- README.md | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 16189aa1f..789d762b0 100644 --- a/README.md +++ b/README.md @@ -165,9 +165,14 @@ Reader module support several asynchronous http clients. The dependencies used f so the user of the module can decide which client to use and include only the necessary dependencies. The clients are: -#### Future based `HttpClientServerConnection` -Uses `java.net.http.HttpClient` to send requests to the server, therefore requires no additional dependencies. But works -only with Java 11 or higher. + +[//]: # (TODO #298 needs Java 11 cross-build) + +[//]: # (#### Future based `HttpClientServerConnection`) + +[//]: # (Uses `java.net.http.HttpClient` to send requests to the server, therefore requires no additional dependencies. But works ) + +[//]: # (only with Java 11 or higher. ) #### Future based `ArmeririaServerConnection` Add @@ -187,13 +192,21 @@ Add " to your dependencies. -#### ZIO based `HttpClientServerConnection` -Add -```scala -"com.softwaremill.sttp.client3" %% "zio" % "[version]" // for ZIO 2.x -"com.softwaremill.sttp.client3" %% "zio1" % "[version]" // for ZIO 1.x -``` -to your dependencies. +[//]: # (TODO #298 needs Java 11 cross-build) + +[//]: # (#### ZIO based `HttpClientServerConnection`) + +[//]: # (Add) + +[//]: # (```scala) + +[//]: # ("com.softwaremill.sttp.client3" %% "zio" % "[version]" // for ZIO 2.x) + +[//]: # ("com.softwaremill.sttp.client3" %% "zio1" % "[version]" // for ZIO 1.x) + +[//]: # (```) + +[//]: # (to your dependencies.) #### ZIO based `ArmeririaServerConnection` Add From fe07272c111e1545e7a2afb3b2fb9f06d12a0225 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 17 Nov 2024 04:47:15 +0100 Subject: [PATCH 21/52] - major rework --- .github/workflows/test_filenames_check.yml | 2 +- .../za/co/absa/atum/agent/AtumAgent.scala | 11 +-- .../za/co/absa/atum/agent/AtumContext.scala | 64 ++++++------- .../absa/atum/agent/AtumAgentUnitTests.scala | 2 +- .../atum/agent/AtumContextUnitTests.scala | 51 ++++++----- .../agent/model/AtumMeasureUnitTests.scala | 3 +- .../atum/agent/model/MeasureUnitTests.scala | 2 +- build.sbt | 4 +- .../main/postgres/runs/get_measurements.sql | 2 +- .../atum/model/envelopes/ErrorResponse.scala | 26 +++++- .../za/co/absa/atum/model/types/basic.scala | 65 ++++++++++++++ .../model/utils/JsonSyntaxExtensions.scala | 8 +- .../envelopes/ErrorResponseUnitTests.scala | 72 +++++++++++++++ ...la => JsonSyntaxExtensionsUnitTests.scala} | 2 +- .../testing/implicits/StringImplicits.scala | 2 +- project/Dependencies.scala | 25 +++--- project/JacocoSetup.scala | 5 +- project/Setup.scala | 7 +- project/VersionAxes.scala | 5 -- .../co/absa/atum/reader/implicits/zio.scala} | 15 +--- .../za/co/absa/atum/reader/FlowReader.scala | 23 ++++- .../absa/atum/reader/PartitioningReader.scala | 20 ++++- .../za/co/absa/atum/reader/basic/Reader.scala | 36 +++++++- .../basic/ReaderWithPartitioningId.scala | 41 +++++++++ .../atum/reader/basic/RequestResult.scala | 38 ++++++++ .../absa/atum/reader/implicits/future.scala | 25 ++++++ .../za/co/absa/atum/reader/implicits/io.scala | 24 +++++ .../server/GenericServerConnection.scala | 56 ------------ .../atum/reader/server/ServerConfig.scala | 29 ++++++ .../future/ArmeriaServerConnection.scala | 53 ----------- .../future/FutureServerConnection.scala | 44 --------- .../future/HttpClientServerConnection.scala | 54 ----------- .../server/io/ArmeriaServerConnection.scala | 62 ------------- .../server/zio/ArmeriaServerConnection.scala | 43 --------- .../zio/HttpClientServerConnection.scala | 44 --------- .../reader/basic/Reader_ZIOUnitTests.scala | 43 +++++++++ .../atum/reader/FlowReaderUnitTests.scala | 19 +++- .../reader/PartitioningReaderUnitTests.scala | 21 ++++- .../ReaderWithPartitioningIdUnitTests.scala | 89 +++++++++++++++++++ .../reader/basic/Reader_CatsIOUnitTests.scala | 57 ++++++++++++ .../reader/basic/Reader_FutureUnitTests.scala | 53 +++++++++++ .../reader/basic/RequestResultUnitTests.scala | 83 +++++++++++++++++ .../reader/server/ServerConfigUnitTests.scala | 31 +++++++ .../zio/ZioServerConnectionUnitTests.scala | 39 -------- 44 files changed, 881 insertions(+), 519 deletions(-) create mode 100644 model/src/main/scala/za/co/absa/atum/model/types/basic.scala create mode 100644 model/src/test/scala/za/co/absa/atum/model/envelopes/ErrorResponseUnitTests.scala rename model/src/test/scala/za/co/absa/atum/model/utils/{SerializationUtilsUnitTests.scala => JsonSyntaxExtensionsUnitTests.scala} (99%) rename model/src/test/scala/za/co/absa/atum/{model => }/testing/implicits/StringImplicits.scala (95%) rename reader/src/main/{scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala => scala-2.13/za/co/absa/atum/reader/implicits/zio.scala} (67%) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala create mode 100644 reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/server/ServerConfigUnitTests.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index b870c1866..ae56d4514 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -40,6 +40,6 @@ jobs: server/src/test/scala/za/co/absa/atum/server/api/TestData.scala, server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala, server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala, - model/src/test/scala/za/co/absa/atum/model/testing/* + model/src/test/scala/za/co/absa/atum/testing/* verbose-logging: 'false' fail-on-violation: 'true' diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index 32e4d9ec8..8e9dba60d 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -17,9 +17,10 @@ package za.co.absa.atum.agent import com.typesafe.config.{Config, ConfigFactory} -import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.dispatcher.{CapturingDispatcher, ConsoleDispatcher, Dispatcher, HttpDispatcher} import za.co.absa.atum.model.dto.{AdditionalDataDTO, AdditionalDataPatchDTO, CheckpointDTO, PartitioningSubmitDTO} +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.model.types.basic.AtumPartitionsOps /** * Entity that communicate with the API, primarily focused on spawning Atum Context(s). @@ -58,7 +59,7 @@ trait AtumAgent { atumPartitions: AtumPartitions, additionalDataPatchDTO: AdditionalDataPatchDTO ): AdditionalDataDTO = { - dispatcher.updateAdditionalData(AtumPartitions.toSeqPartitionDTO(atumPartitions), additionalDataPatchDTO) + dispatcher.updateAdditionalData(atumPartitions.toPartitioningDTO, additionalDataPatchDTO) } /** @@ -75,7 +76,7 @@ trait AtumAgent { */ def getOrCreateAtumContext(atumPartitions: AtumPartitions): AtumContext = { val authorIfNew = AtumAgent.currentUser - val partitioningDTO = PartitioningSubmitDTO(AtumPartitions.toSeqPartitionDTO(atumPartitions), None, authorIfNew) + val partitioningDTO = PartitioningSubmitDTO(atumPartitions.toPartitioningDTO, None, authorIfNew) val atumContextDTO = dispatcher.createPartitioning(partitioningDTO) val atumContext = AtumContext.fromDTO(atumContextDTO, this) @@ -94,8 +95,8 @@ trait AtumAgent { val authorIfNew = AtumAgent.currentUser val newPartitions: AtumPartitions = parentAtumContext.atumPartitions ++ subPartitions - val newPartitionsDTO = AtumPartitions.toSeqPartitionDTO(newPartitions) - val parentPartitionsDTO = Some(AtumPartitions.toSeqPartitionDTO(parentAtumContext.atumPartitions)) + val newPartitionsDTO = newPartitions.toPartitioningDTO + val parentPartitionsDTO = Some(parentAtumContext.atumPartitions.toPartitioningDTO) val partitioningDTO = PartitioningSubmitDTO(newPartitionsDTO, parentPartitionsDTO, authorIfNew) val atumContextDTO = dispatcher.createPartitioning(partitioningDTO) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index 66386b3be..7fc8f8311 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -17,14 +17,13 @@ package za.co.absa.atum.agent import org.apache.spark.sql.DataFrame -import za.co.absa.atum.agent.AtumContext.{AdditionalData, AtumPartitions} import za.co.absa.atum.agent.exception.AtumAgentException.PartitioningUpdateException import za.co.absa.atum.agent.model._ import za.co.absa.atum.model.dto._ +import za.co.absa.atum.model.types.basic.{AdditionalData, AtumPartitions, AtumPartitionsOps, PartitioningDTOOps} import java.time.ZonedDateTime import java.util.UUID -import scala.collection.immutable.ListMap /** * This class provides the methods to measure Spark `Dataframe`. Also allows to add and remove measures. @@ -91,7 +90,7 @@ class AtumContext private[agent] ( name = checkpointName, author = agent.currentUser, measuredByAtumAgent = true, - partitioning = AtumPartitions.toSeqPartitionDTO(atumPartitions), + partitioning = atumPartitions.toPartitioningDTO, processStartTime = startTime, processEndTime = Some(endTime), measurements = measurementDTOs @@ -115,7 +114,7 @@ class AtumContext private[agent] ( id = UUID.randomUUID(), name = checkpointName, author = agent.currentUser, - partitioning = AtumPartitions.toSeqPartitionDTO(atumPartitions), + partitioning = atumPartitions.toPartitioningDTO, processStartTime = dateTimeNow, processEndTime = Some(dateTimeNow), measurements = MeasurementBuilder.buildAndValidateMeasurementsDTO(measurements) @@ -206,36 +205,37 @@ class AtumContext private[agent] ( } object AtumContext { - /** - * Type alias for Atum partitions. - */ - type AtumPartitions = ListMap[String, String] - type AdditionalData = Map[String, Option[String]] - - /** - * Object contains helper methods to work with Atum partitions. - */ - object AtumPartitions { - def apply(elems: (String, String)): AtumPartitions = { - ListMap(elems) - } - - def apply(elems: List[(String, String)]): AtumPartitions = { - ListMap(elems:_*) - } - - private[agent] def toSeqPartitionDTO(atumPartitions: AtumPartitions): PartitioningDTO = { - atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq - } - - private[agent] def fromPartitioning(partitioning: PartitioningDTO): AtumPartitions = { - AtumPartitions(partitioning.map(partition => Tuple2(partition.key, partition.value)).toList) - } - } - +// TODO --- +// /** +// * Type alias for Atum partitions. +// */ +// type AtumPartitions = ListMap[String, String] +// type AdditionalData = Map[String, Option[String]] +// +// /** +// * Object contains helper methods to work with Atum partitions. +// */ +// object AtumPartitions { +// def apply(elems: (String, String)): AtumPartitions = { +// ListMap(elems) +// } +// +// def apply(elems: List[(String, String)]): AtumPartitions = { +// ListMap(elems:_*) +// } +// +// private[agent] def toSeqPartitionDTO(atumPartitions: AtumPartitions): PartitioningDTO = { +// atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq +// } +// +// private[agent] def fromPartitioning(partitioning: PartitioningDTO): AtumPartitions = { +// AtumPartitions(partitioning.map(partition => Tuple2(partition.key, partition.value)).toList) +// } +// } +// private[agent] def fromDTO(atumContextDTO: AtumContextDTO, agent: AtumAgent): AtumContext = { new AtumContext( - AtumPartitions.fromPartitioning(atumContextDTO.partitioning), + atumContextDTO.partitioning.toAtumPartitions, agent, MeasuresBuilder.mapToMeasures(atumContextDTO.measures), atumContextDTO.additionalData diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumAgentUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumAgentUnitTests.scala index 79613e91a..b2cb8c0ca 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumAgentUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumAgentUnitTests.scala @@ -18,8 +18,8 @@ package za.co.absa.atum.agent import com.typesafe.config.{Config, ConfigException, ConfigFactory, ConfigValueFactory} import org.scalatest.funsuite.AnyFunSuiteLike -import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.dispatcher.{CapturingDispatcher, ConsoleDispatcher, HttpDispatcher} +import za.co.absa.atum.model.types.basic.AtumPartitions class AtumAgentUnitTests extends AnyFunSuiteLike { diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala index 75585f485..ba18377d2 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextUnitTests.scala @@ -22,11 +22,11 @@ import org.mockito.ArgumentCaptor import org.mockito.Mockito.{mock, times, verify, when} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{RecordCount, SumOfValuesOfColumn} import za.co.absa.atum.agent.model.{Measure, MeasureResult, MeasurementBuilder, UnknownMeasure} import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.CheckpointDTO +import za.co.absa.atum.model.types.basic._ class AtumContextUnitTests extends AnyFlatSpec with Matchers { @@ -95,12 +95,12 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { val argument = ArgumentCaptor.forClass(classOf[CheckpointDTO]) verify(mockAgent).saveCheckpoint(argument.capture()) - - assert(argument.getValue.name == "testCheckpoint") - assert(argument.getValue.author == authorTest) - assert(argument.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) - assert(argument.getValue.measurements.head.result.mainValue.value == "3") - assert(argument.getValue.measurements.head.result.mainValue.valueType == ResultValueType.LongValue) + val value: CheckpointDTO = argument.getValue + assert(value.name == "testCheckpoint") + assert(value.author == authorTest) + assert(value.partitioning == atumPartitions.toPartitioningDTO) + assert(value.measurements.head.result.mainValue.value == "3") + assert(value.measurements.head.result.mainValue.valueType == ResultValueType.LongValue) } "createCheckpointOnProvidedData" should "create a Checkpoint on provided data" in { @@ -123,13 +123,14 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { val argument = ArgumentCaptor.forClass(classOf[CheckpointDTO]) verify(mockAgent).saveCheckpoint(argument.capture()) - - assert(argument.getValue.name == "name") - assert(argument.getValue.author == authorTest) - assert(!argument.getValue.measuredByAtumAgent) - assert(argument.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) - assert(argument.getValue.processStartTime == argument.getValue.processEndTime.get) - assert(argument.getValue.measurements == MeasurementBuilder.buildAndValidateMeasurementsDTO(measurements)) + val value: CheckpointDTO = argument.getValue + + assert(value.name == "name") + assert(value.author == authorTest) + assert(!value.measuredByAtumAgent) + assert(value.partitioning == atumPartitions.toPartitioningDTO) + assert(value.processStartTime == value.processEndTime.get) + assert(value.measurements == MeasurementBuilder.buildAndValidateMeasurementsDTO(measurements)) } "createCheckpoint" should "take measurements and create a Checkpoint, multiple measure changes" in { @@ -167,12 +168,13 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { val argumentFirst = ArgumentCaptor.forClass(classOf[CheckpointDTO]) verify(mockAgent, times(1)).saveCheckpoint(argumentFirst.capture()) + val valueFirst: CheckpointDTO = argumentFirst.getValue - assert(argumentFirst.getValue.name == "checkPointNameCount") - assert(argumentFirst.getValue.author == authorTest) - assert(argumentFirst.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) - assert(argumentFirst.getValue.measurements.head.result.mainValue.value == "4") - assert(argumentFirst.getValue.measurements.head.result.mainValue.valueType == ResultValueType.LongValue) + assert(valueFirst.name == "checkPointNameCount") + assert(valueFirst.author == authorTest) + assert(valueFirst.partitioning == atumPartitions.toPartitioningDTO) + assert(valueFirst.measurements.head.result.mainValue.value == "4") + assert(valueFirst.measurements.head.result.mainValue.valueType == ResultValueType.LongValue) atumContext.addMeasure(SumOfValuesOfColumn("columnForSum")) when(mockAgent.currentUser).thenReturn(authorTest + "Another") // maybe a process changed the author / current user @@ -180,12 +182,13 @@ class AtumContextUnitTests extends AnyFlatSpec with Matchers { val argumentSecond = ArgumentCaptor.forClass(classOf[CheckpointDTO]) verify(mockAgent, times(2)).saveCheckpoint(argumentSecond.capture()) + val valueSecond: CheckpointDTO = argumentSecond.getValue - assert(argumentSecond.getValue.name == "checkPointNameSum") - assert(argumentSecond.getValue.author == authorTest + "Another") - assert(argumentSecond.getValue.partitioning == AtumPartitions.toSeqPartitionDTO(atumPartitions)) - assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.value == "22.5") - assert(argumentSecond.getValue.measurements.tail.head.result.mainValue.valueType == ResultValueType.BigDecimalValue) + assert(valueSecond.name == "checkPointNameSum") + assert(valueSecond.author == authorTest + "Another") + assert(valueSecond.partitioning == atumPartitions.toPartitioningDTO) + assert(valueSecond.measurements.tail.head.result.mainValue.value == "22.5") + assert(valueSecond.measurements.tail.head.result.mainValue.valueType == ResultValueType.BigDecimalValue) } "addAdditionalData" should "add key/value pair to map for additional data" in { diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala index 7f2278f91..5c3ff2b88 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/AtumMeasureUnitTests.scala @@ -21,9 +21,10 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructT import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent -import za.co.absa.atum.agent.AtumContext.{AtumPartitions, DatasetWrapper} +import za.co.absa.atum.agent.AtumContext.DatasetWrapper import za.co.absa.atum.agent.model.AtumMeasure._ import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.spark.commons.test.SparkTestBase class AtumMeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala index d96d0ac1e..fea11c9f9 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasureUnitTests.scala @@ -19,11 +19,11 @@ package za.co.absa.atum.agent.model import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import za.co.absa.atum.agent.AtumAgent -import za.co.absa.atum.agent.AtumContext.AtumPartitions import za.co.absa.atum.agent.model.AtumMeasure.{AbsSumOfValuesOfColumn, RecordCount, SumOfHashesOfColumn, SumOfValuesOfColumn} import za.co.absa.spark.commons.test.SparkTestBase import za.co.absa.atum.agent.AtumContext._ import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.types.basic.AtumPartitions class MeasureUnitTests extends AnyFlatSpec with Matchers with SparkTestBase { self => diff --git a/build.sbt b/build.sbt index 0c2f6b1ee..2227b5dc9 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ import Dependencies.* import Dependencies.Versions.spark3 import VersionAxes.* -ThisBuild / scalaVersion := Setup.scala213.asString // default version TODO +//ThisBuild / scalaVersion := Setup.scala212.asString // default version TODO ThisBuild / versionScheme := Some("early-semver") @@ -35,6 +35,8 @@ initialize := { //this routine can be used to assert the required Java version } +Test/parallelExecution := false + enablePlugins(FlywayPlugin) flywayUrl := FlywayConfiguration.flywayUrl flywayUser := FlywayConfiguration.flywayUser diff --git a/database/src/main/postgres/runs/get_measurements.sql b/database/src/main/postgres/runs/get_measurements.sql index eabad91e4..4aa3b7434 100644 --- a/database/src/main/postgres/runs/get_measurements.sql +++ b/database/src/main/postgres/runs/get_measurements.sql @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala b/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala index 038018ac2..e531410e5 100644 --- a/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala +++ b/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala @@ -16,20 +16,30 @@ package za.co.absa.atum.model.envelopes +import io.circe.parser.decode import io.circe._ import io.circe.generic.semiauto._ import java.util.UUID object ErrorResponse { - implicit val decodeErrorResponse: Decoder[ErrorResponse] = deriveDecoder + implicit val decodeErrorResponse: Decoder[ErrorResponse] = deriveDecoder //TODo neeeded? implicit val encodeErrorResponse: Encoder[ErrorResponse] = deriveEncoder -} -sealed trait ErrorResponse extends ResponseEnvelope { - def message: String + def basedOnStatusCode(statusCode: Int, jsonString: String): Either[Error, ErrorResponse] = { + statusCode match { + case 400 => decode[BadRequestResponse](jsonString) + case 401 => decode[UnauthorizedErrorResponse](jsonString) + case 404 => decode[NotFoundErrorResponse](jsonString) + case 409 => decode[ConflictErrorResponse](jsonString) + case 500 => decode[InternalServerErrorResponse](jsonString) + case _ => decode[GeneralErrorResponse](jsonString) + } + } } +sealed trait ErrorResponse extends ResponseEnvelope + final case class BadRequestResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse object BadRequestResponse { @@ -71,3 +81,11 @@ object ErrorInDataErrorResponse { implicit val decoderInternalServerErrorResponse: Decoder[ErrorInDataErrorResponse] = deriveDecoder implicit val encoderInternalServerErrorResponse: Encoder[ErrorInDataErrorResponse] = deriveEncoder } + +final case class UnauthorizedErrorResponse(message: String, requestId: UUID = UUID.randomUUID()) extends ErrorResponse + +object UnauthorizedErrorResponse { + implicit val decoderInternalServerErrorResponse: Decoder[UnauthorizedErrorResponse] = deriveDecoder + implicit val encoderInternalServerErrorResponse: Encoder[UnauthorizedErrorResponse] = deriveEncoder +} + diff --git a/model/src/main/scala/za/co/absa/atum/model/types/basic.scala b/model/src/main/scala/za/co/absa/atum/model/types/basic.scala new file mode 100644 index 000000000..00e2c7cd3 --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/types/basic.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.types + +import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningDTO} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax + +import scala.collection.immutable.ListMap + +object basic { + /** + * Type alias for Atum partitions. + */ + type AtumPartitions = ListMap[String, String] + type AdditionalData = Map[String, Option[String]] + + /** + * Object contains helper methods to work with Atum partitions. + */ + object AtumPartitions { + def apply(elems: (String, String)): AtumPartitions = { + ListMap(elems) + } + + def apply(elems: List[(String, String)]): AtumPartitions = { + ListMap(elems:_*) + } + + /*TODO private[agent]*/ def toPartitionDTO(atumPartitions: AtumPartitions): PartitioningDTO = { + atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq + } + + /*TOD private[agent]*/ def fromPartitioningDTO(partitioning: PartitioningDTO): AtumPartitions = { + AtumPartitions(partitioning.map(partition => Tuple2(partition.key, partition.value)).toList) + } + + } + + implicit class AtumPartitionsOps(val atumPartitions: AtumPartitions) extends AnyVal { + def toPartitioningDTO: PartitioningDTO = { + atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq + } + } + + implicit class PartitioningDTOOps(val partitioning: PartitioningDTO) extends AnyVal { + def toAtumPartitions: AtumPartitions = { + AtumPartitions(partitioning.map(partition => Tuple2(partition.key, partition.value)).toList) + } + } + +} diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index c892138e7..e9e49fe81 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -36,12 +36,16 @@ object JsonSyntaxExtensions { implicit class JsonDeserializationSyntax(jsonStr: String) { def as[T: Decoder]: T = { - decode[T](jsonStr) match { + asSafe[T] match { case Right(value) => value - case Left(error) => throw new RuntimeException(s"Failed to decode JSON: $error") + case Left(error) => throw error } } + def asSafe[T: Decoder]: Either[io.circe.Error, T] = { + decode[T](jsonStr) + } + def fromBase64As[T: Decoder]: Either[io.circe.Error, T] = { val decodedBytes = Base64.getDecoder.decode(jsonStr) val decodedString = new String(decodedBytes, "UTF-8") diff --git a/model/src/test/scala/za/co/absa/atum/model/envelopes/ErrorResponseUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/envelopes/ErrorResponseUnitTests.scala new file mode 100644 index 000000000..b14c3de7a --- /dev/null +++ b/model/src/test/scala/za/co/absa/atum/model/envelopes/ErrorResponseUnitTests.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.envelopes + +import io.circe.ParsingFailure +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax + +import java.util.UUID + +class ErrorResponseUnitTests extends AnyFunSuiteLike { + test("ErrorResponse.basedOnStatusCode should return correct error response on `Bad Request`") { + val originalError = BadRequestResponse("Bad Request", UUID.randomUUID()) + val errorResponse = ErrorResponse.basedOnStatusCode(400, originalError.asJsonString) + assert(errorResponse == Right(originalError)) + } + + test("ErrorResponse.basedOnStatusCode should return correct error response on `Unauthorized`") { + val originalError = UnauthorizedErrorResponse("Unauthorized", UUID.randomUUID()) + val errorResponse = ErrorResponse.basedOnStatusCode(401, originalError.asJsonString) + assert(errorResponse == Right(originalError)) + } + + test("ErrorResponse.basedOnStatusCode should return correct error response on `Not Found`") { + val originalError = NotFoundErrorResponse("Not Found", UUID.randomUUID()) + val errorResponse = ErrorResponse.basedOnStatusCode(404, originalError.asJsonString) + assert(errorResponse == Right(originalError)) + } + + test("ErrorResponse.basedOnStatusCode should return correct error response on `Conflict`") { + val originalError = ConflictErrorResponse("Conflict", UUID.randomUUID()) + val errorResponse = ErrorResponse.basedOnStatusCode(409, originalError.asJsonString) + assert(errorResponse == Right(originalError)) + } + + test("ErrorResponse.basedOnStatusCode should return correct error response on `Internal Server Error`") { + val originalError = InternalServerErrorResponse("Internal Server Error", UUID.randomUUID()) + val errorResponse = ErrorResponse.basedOnStatusCode(500, originalError.asJsonString) + assert(errorResponse == Right(originalError)) + } + + test("ErrorResponse.basedOnStatusCode should return GeneralErrorResponse on unknown status code") { + val originalError = GeneralErrorResponse("Heluva", UUID.randomUUID()) + val errorResponse = ErrorResponse.basedOnStatusCode(600, originalError.asJsonString) + assert(errorResponse == Right(originalError)) + } + + test("ErrorResponse.basedOnStatusCode fails on invalid JSON") { + val message = "This is not a JSON" + val errorResponse = ErrorResponse.basedOnStatusCode(400, message) + assert(errorResponse.isLeft) + errorResponse.swap.foreach{e => + // investigate the error + assert(e.isInstanceOf[ParsingFailure]) + } + } + +} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensionsUnitTests.scala similarity index 99% rename from model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala rename to model/src/test/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensionsUnitTests.scala index e34aa2401..6491dc501 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensionsUnitTests.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.model.testing.implicits.StringImplicits.StringLinearizati import java.time.{ZoneId, ZoneOffset, ZonedDateTime} import java.util.UUID -class SerializationUtilsUnitTests extends AnyFlatSpecLike { +class JsonSyntaxExtensionsUnitTests extends AnyFlatSpecLike { // AdditionalDataDTO "asJsonString" should "serialize AdditionalDataDTO into json string" in { diff --git a/model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala b/model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala similarity index 95% rename from model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala rename to model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala index 1eac82788..ec64d2062 100644 --- a/model/src/test/scala/za/co/absa/atum/model/testing/implicits/StringImplicits.scala +++ b/model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/project/Dependencies.scala b/project/Dependencies.scala index fdc72c1f3..ace114abf 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -247,15 +247,12 @@ object Dependencies { lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpClient lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpClient - // Armeria Future backend - lazy val sttpArmeririaFutureBackend = sttpClient3Org %% "armeria-backend" % Versions.sttpClient % Optional - // Armeria Cats backend - lazy val sttpArmeririaCatsBackend = sttpClient3Org %% "armeria-backend-cats" % Versions.sttpClient % Optional + // Cats backend lazy val catsEffect = typeLevelOrg %% "cats-effect" % Versions.catsEffect % Optional - // Armeria Zio backend - lazy val sttpArmeririaZioBackend = sttpClient3Org %% "armeria-backend-zio" % Versions.sttpClient % Optional - // HttpClient Zio backend -// lazy val sttpHttpClientZioBackend = sttpClient3Org %% "zio" % Versions.sttpClient % Optional TODO #298 needs Java 11 cross-build + lazy val sttpCats = sttpClient3Org %% "cats" % Versions.sttpClient % Optional + + // ZIO backend + lazy val sttpZio = sttpClient3Org %% "zio" % Versions.sttpClient % Optional // testing lazy val zioTest = zioOrg %% "zio-test" % Versions.zio % Test @@ -266,15 +263,13 @@ object Dependencies { Seq( sttpCore, sttpCirce, - sttpArmeririaFutureBackend, - sttpArmeririaCatsBackend, + sttpCats, catsEffect, - sttpArmeririaZioBackend, -// sttpHttpClientZioBackend, TODO #298 needs Java 11 cross-build + sttpZio, zioTest, - zioTestSbt, - zioTestJunit, - sbtJunitInterface +// zioTestSbt, +// zioTestJunit, +// sbtJunitInterface ) ++ testDependencies ++ jsonSerdeDependencies diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 635ea276d..40e09bcf2 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -52,7 +52,10 @@ object JacocoSetup { "za.co.absa.atum.server.api.database.DoobieImplicits*", "za.co.absa.atum.server.api.database.TransactorProvider*", "za.co.absa.atum.model.dto.*", - "za.co.absa.atum.model.envelopes.*" + "za.co.absa.atum.model.envelopes.Pagination", + "za.co.absa.atum.model.envelopes.ResponseEnvelope", + "za.co.absa.atum.model.envelopes.StatusResponse", + "za.co.absa.atum.model.envelopes.SuccessResponse" ) } diff --git a/project/Setup.scala b/project/Setup.scala index f1fa9b2ef..52d512204 100644 --- a/project/Setup.scala +++ b/project/Setup.scala @@ -24,7 +24,7 @@ import za.co.absa.commons.version.Version object Setup { - //supported Scala versions + //possible supported Scala versions val scala211: Version = Version.asSemVer("2.11.12") val scala212: Version = Version.asSemVer("2.12.18") val scala213: Version = Version.asSemVer("2.13.11") @@ -38,7 +38,10 @@ object Setup { ) val serverAndDbScalaVersion: Version = scala213 //covers REST server and database modules - val clientSupportedScalaVersions: Seq[Version] = Seq(scala212, scala213) + val clientSupportedScalaVersions: Seq[Version] = Seq( + scala212, + scala213, + ) val commonScalacOptions: Seq[String] = Seq( "-unchecked", diff --git a/project/VersionAxes.scala b/project/VersionAxes.scala index d55480426..a52aec46c 100644 --- a/project/VersionAxes.scala +++ b/project/VersionAxes.scala @@ -32,11 +32,6 @@ object VersionAxes { override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") } - case class JavaVersionAxis(javaVersion: String) extends sbt.VirtualAxis.WeakAxis { - override val directorySuffix = s"-jdk$javaVersion" - override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") - } - private def camelCaseToLowerDashCase(origName: String): String = { origName .replaceAll("([A-Z])", "-$1") diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala b/reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala similarity index 67% rename from reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala rename to reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala index 42494d594..41651397a 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ZioServerConnection.scala +++ b/reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala @@ -14,19 +14,10 @@ * limitations under the License. */ -package za.co.absa.atum.reader.server.zio +package za.co.absa.atum.reader.implicits -import zio.{Task, ZIO} import sttp.client3.impl.zio.RIOMonadAsyncError -import za.co.absa.atum.reader.server.GenericServerConnection - - -abstract class ZioServerConnection(serverUrl: String) extends GenericServerConnection[Task](serverUrl)(new RIOMonadAsyncError[Any]) { - - override def close(): Task[Unit] = { - ZIO.unit - } +object zio { + implicit val ZIOMonad: RIOMonadAsyncError[Any] = new RIOMonadAsyncError[Any] } - - diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index a6be49e5f..073340f78 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -16,12 +16,29 @@ package za.co.absa.atum.reader -import za.co.absa.atum.reader.basic.Reader -import za.co.absa.atum.reader.server.GenericServerConnection +import sttp.client3.SttpBackend +import sttp.monad.MonadError +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.reader.basic.ReaderWithPartitioningId +import za.co.absa.atum.reader.server.ServerConfig + +/** + * This class is a reader that reads data tight to a flow. + * @param mainFlowPartitioning - the partitioning of the main flow; renamed from ancestor's 'flowPartitioning' + * @param serverConfig - tha Atum server configuration + * @param backend - sttp backend, that will be executing the requests + * @param ev - using evidence based approach to ensure that the type F is a MonadError instead of using context + * bounds, as it make the imports easier to follow + * @tparam F - the effect type (e.g. Future, IO, Task, etc.) + */ +class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) + (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) extends ReaderWithPartitioningId[F] { + + override def partitioning: AtumPartitions = mainFlowPartitioning -class FlowReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F]) extends Reader[F]{ def foo(): String = { // just to have some testable content "bar" } + } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index 7de8d3187..2c3782ffc 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -16,10 +16,24 @@ package za.co.absa.atum.reader -import za.co.absa.atum.reader.basic.Reader -import za.co.absa.atum.reader.server.GenericServerConnection +import sttp.client3.SttpBackend +import sttp.monad.MonadError +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.reader.basic.ReaderWithPartitioningId +import za.co.absa.atum.reader.server.ServerConfig -class PartitioningReader[F[_]]()(override implicit val serverConnection: GenericServerConnection[F]) extends Reader[F] { +/** + * + * @param partitioning - the Atum partitions to read the information from + * @param serverConfig - tha Atum server configuration + * @param backend - sttp backend, that will be executing the requests + * @param ev - using evidence based approach to ensure that the type F is a MonadError instead of using context + * bounds, as it make the imports easier to follow + * @tparam F - the effect type (e.g. Future, IO, Task, etc.) + */ +case class PartitioningReader[F[_]](partitioning: AtumPartitions) + (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) + extends ReaderWithPartitioningId[F] { def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index 57c06e923..363bb68bb 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -16,6 +16,38 @@ package za.co.absa.atum.reader.basic -import za.co.absa.atum.reader.server.GenericServerConnection +import io.circe.Decoder +import sttp.client3.{Identity, RequestT, ResponseException, SttpBackend, basicRequest} +import sttp.client3.circe.asJson +import sttp.model.Uri +import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.reader.server.ServerConfig +import za.co.absa.atum.reader.basic.RequestResult._ -abstract class Reader[F[_]](implicit val serverConnection: GenericServerConnection[F]) +/** + * Reader is a base class for reading data from a remote server. + * @param monadError$F$0 - the context bind for the F type; it's MonadError to allow not just map, flatMap but eventually + * also error handling easily on a higher level + * @param serverConfig - the configuration hwo to reach the Atum server + * @param backend - sttp backend to use to send requests + * @tparam F - the monadic effect used to get the data (e.g. Future, IO, Task, etc.) + */ +abstract class Reader[F[_]: MonadError](implicit val serverConfig: ServerConfig, val backend: SttpBackend[F, Any]) { + + protected def getQuery[R: Decoder](endpointUri: String, params: Map[String, String] = Map.empty): F[RequestResult[R]] = { + val endpointToQuery = serverConfig.host + endpointUri + val uri = Uri.unsafeParse(endpointToQuery).addParams(params) + val request: RequestT[Identity, Either[ResponseException[String, CirceError], R], Any] = basicRequest + .get(uri) + .response(asJson[R]) + + val response = backend.send(request) + + response.map(_.toRequestResult) + } +} + +object Reader { + +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala new file mode 100644 index 000000000..e733da82f --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import sttp.client3.SttpBackend +import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.model.dto.PartitioningWithIdDTO +import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.model.types.basic.AtumPartitionsOps +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax +import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.server.ServerConfig + +abstract class ReaderWithPartitioningId[F[_]: MonadError](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any]) + extends Reader[F] { + def partitioning: AtumPartitions + + protected def partitioningId(): F[RequestResult[Long]] = { + val encodedPartitioning = partitioning.toPartitioningDTO.asBase64EncodedJsonString + val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]]("/api/v2/partitionings", Map("partitioning" -> encodedPartitioning)) + queryResult.map{result => + result.map(_.data.id) + } + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala new file mode 100644 index 000000000..76e8cbfa9 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} +import za.co.absa.atum.model.envelopes.ErrorResponse + +object RequestResult { + type CirceError = io.circe.Error + type RequestResult[R] = Either[ResponseException[ErrorResponse, CirceError], R] + + implicit class ResponseOps[R](val response: Response[Either[ResponseException[String, CirceError], R]]) extends AnyVal { + def toRequestResult: RequestResult[R] = { + response.body.left.map { + case he: HttpError[String] => + ErrorResponse.basedOnStatusCode(he.statusCode.code, he.body) match { + case Right(er) => HttpError(er, he.statusCode) + case Left(ce) => DeserializationException(he.body, ce) + } + case de: DeserializationException[CirceError] => de + } + } + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala new file mode 100644 index 000000000..0656bed77 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.implicits + +import sttp.monad.{FutureMonad => SttpFutureMonad} + +import scala.concurrent.ExecutionContext.Implicits.global + +object future { + implicit val FutureMonad: SttpFutureMonad = new SttpFutureMonad +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala new file mode 100644 index 000000000..b43501da0 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.implicits + +import cats.effect.IO +import sttp.client3.impl.cats.CatsMonadAsyncError + +object io { + implicit val CatsIOMonad: CatsMonadAsyncError[IO] = new CatsMonadAsyncError[IO] +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala deleted file mode 100644 index 32b584a9f..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/GenericServerConnection.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server - -import _root_.io.circe.Decoder -import _root_.io.circe.{Error => circeError} -import com.typesafe.config.Config -import sttp.client3.{Identity, RequestT, Response, ResponseException, basicRequest} -import sttp.model.Uri -import sttp.client3.circe._ -import sttp.monad.MonadError -import sttp.monad.syntax._ -import za.co.absa.atum.model.envelopes.ErrorResponse -import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult - -abstract class GenericServerConnection[F[_]: MonadError](val serverUrl: String) { - - protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): F[Response[RequestResult[R]]] - - def getQuery[R: Decoder](endpointUri: String, params: Map[String, String] = Map.empty): F[RequestResult[R]] = { - val endpointToQuery = serverUrl + endpointUri - val uri = Uri.unsafeParse(endpointToQuery).addParams(params) - val request: RequestT[Identity, RequestResult[R], Any] = basicRequest - .get(uri) - .response(asJsonEither[ErrorResponse, R]) - val response = executeRequest(request) - response.map(_.body) - } - - def close(): F[Unit] - -} - -object GenericServerConnection { - final val UrlKey = "atum.server.url" - - type RequestResult[R] = Either[ResponseException[ErrorResponse, circeError], R] - - def atumServerUrl(config: Config): String = { - config.getString(UrlKey) - } -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala new file mode 100644 index 000000000..f38eff892 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server + +import com.typesafe.config.{Config, ConfigFactory} + +case class ServerConfig (host: String) + +object ServerConfig { + final val HostKey = "atum.server.url" + + def fromConfig(config: Config = ConfigFactory.load()): ServerConfig = { + ServerConfig(config.getString(HostKey)) + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala deleted file mode 100644 index 48f192ff0..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/ArmeriaServerConnection.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.future - -import com.typesafe.config.{Config, ConfigFactory} -import scala.concurrent.{ExecutionContext, Future} -import sttp.client3.armeria.future.ArmeriaFutureBackend -import sttp.client3.SttpBackend - -import za.co.absa.atum.reader.server.GenericServerConnection - -class ArmeriaServerConnection private(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) - extends FutureServerConnection(serverUrl, closeable) { - - def this(serverUrl: String)(implicit executor: ExecutionContext) = { - this(serverUrl, true)(executor) - } - - def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { - this(GenericServerConnection.atumServerUrl(config))(executor) - } - - override protected val backend: SttpBackend[Future, Any] = ArmeriaFutureBackend() - -} - -object ArmeriaServerConnection { - lazy implicit val serverConnection: ArmeriaServerConnection = new ArmeriaServerConnection()(ExecutionContext.Implicits.global) - - def use[R](serverUrl: String)(fnc: ArmeriaServerConnection => Future[R]) - (implicit executor: ExecutionContext): Future[R] = { - val serverConnection = new ArmeriaServerConnection(serverUrl, false) - try { - fnc(serverConnection) - } finally { - serverConnection.backend.close() - } - } -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala deleted file mode 100644 index f46770d2d..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/FutureServerConnection.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.future - - -import scala.concurrent.{ExecutionContext, Future} -import sttp.client3.{Identity, RequestT, Response, SttpBackend} -import sttp.monad.FutureMonad -import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult - -abstract class FutureServerConnection(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) - extends GenericServerConnection[Future](serverUrl)(new FutureMonad) { - - protected val backend: SttpBackend[Future, Any] - - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Future[Response[RequestResult[R]]] = { - request.send(backend) - } - - override def close(): Future[Unit] = { - if (closeable) { - backend.close() - } else { - Future.successful(()) - } - } - -} - diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala deleted file mode 100644 index c05518ce4..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/future/HttpClientServerConnection.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.future - -// TODO #298 needs Java 11 cross-build -//import com.typesafe.config.{Config, ConfigFactory} -//import scala.concurrent.{ExecutionContext, Future} -//import sttp.client3.{HttpClientFutureBackend, SttpBackend} -// -//import za.co.absa.atum.reader.server.GenericServerConnection -// -// -//class HttpClientServerConnection private(serverUrl: String, closeable: Boolean)(implicit executor: ExecutionContext) -// extends FutureServerConnection(serverUrl, closeable) { -// -// def this(serverUrl: String)(implicit executor: ExecutionContext) = { -// this(serverUrl, true)(executor) -// } -// -// def this(config: Config = ConfigFactory.load())(implicit executor: ExecutionContext) = { -// this(GenericServerConnection.atumServerUrl(config))(executor) -// } -// -// override protected val backend: SttpBackend[Future, Any] = HttpClientFutureBackend() -// -//} -// -//object HttpClientServerConnection { -// lazy implicit val serverConnection: FutureServerConnection = new HttpClientServerConnection()(ExecutionContext.Implicits.global) -// -// def use[R](serverUrl: String)(fnc: HttpClientServerConnection => Future[R]) -// (implicit executor: ExecutionContext): Future[R] = { -// val serverConnection = new HttpClientServerConnection(serverUrl) -// try { -// fnc(serverConnection) -// } finally { -// serverConnection.close() -// } -// } -//} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala deleted file mode 100644 index 0c0885f46..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/io/ArmeriaServerConnection.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.io - -import cats.effect.IO -import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT, Response, SttpBackend} -import sttp.client3.armeria.cats.ArmeriaCatsBackend -import sttp.client3.impl.cats.CatsMonadAsyncError -import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult - - -class ArmeriaServerConnection protected(serverUrl: String, backend: SttpBackend[IO, Any], closeable: Boolean) - extends GenericServerConnection[IO](serverUrl)(new CatsMonadAsyncError[IO]) { - - def this(mserverUrl: String) = { - this(mserverUrl, ArmeriaCatsBackend[IO](), closeable = true) - } - - def this(config: Config = ConfigFactory.load()) = { - this(GenericServerConnection.atumServerUrl(config ), ArmeriaCatsBackend[IO](), closeable = true) - } - - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): IO[Response[RequestResult[R]]] = { - request.send(backend) - } - - override def close(): IO[Unit] = { - if (closeable) { - backend.close() - } else { - IO.unit - } - } - -} - -object ArmeriaServerConnection { - lazy implicit val serverConnection: ArmeriaServerConnection = new ArmeriaServerConnection() - - def use[R](serverUrl: String)(fnc: ArmeriaServerConnection => IO[R]): IO[R] = { - ArmeriaCatsBackend.resource[IO]().use{backend => - val serverConnection = new ArmeriaServerConnection(serverUrl, backend, false) - fnc(serverConnection) - } - } -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala deleted file mode 100644 index 1c993b303..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/ArmeriaServerConnection.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.zio - -import com.typesafe.config.{Config, ConfigFactory} -import sttp.client3.{Identity, RequestT, Response} -import sttp.client3.armeria.zio.ArmeriaZioBackend -import zio.Task -import za.co.absa.atum.reader.server.GenericServerConnection -import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult - - -class ArmeriaServerConnection(serverUrl: String) extends ZioServerConnection(serverUrl) { - - def this(config: Config = ConfigFactory.load()) = { - this(GenericServerConnection.atumServerUrl(config )) - } - - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = { - ArmeriaZioBackend.usingDefaultClient().flatMap { backend => - backend.send(request) - } - } - -} - -object ArmeriaServerConnection { - lazy implicit val serverConnection: ArmeriaServerConnection = new ArmeriaServerConnection() -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala deleted file mode 100644 index 798fabac8..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/zio/HttpClientServerConnection.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.zio - -//TODO #298 needs Java 11 cross-build -//import com.typesafe.config.{Config, ConfigFactory} -//import sttp.client3.{Identity, RequestT, Response} -//import sttp.client3.httpclient.zio.HttpClientZioBackend -//import za.co.absa.atum.reader.server.GenericServerConnection -//import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult -//import zio.Task -// -// -//class HttpClientServerConnection(serverUrl: String) extends ZioServerConnection(serverUrl) { -// -// def this(config: Config = ConfigFactory.load()) = { -// this(GenericServerConnection.atumServerUrl(config )) -// } -// -// override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = { -// HttpClientZioBackend().flatMap { backend => -// backend.send(request) -// } -// } -// -//} -// -//object HttpClientServerConnection { -// lazy implicit val serverConnection: HttpClientServerConnection = new HttpClientServerConnection() -//} diff --git a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala new file mode 100644 index 000000000..7b4e2dfb6 --- /dev/null +++ b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala @@ -0,0 +1,43 @@ +package za.co.absa.atum.reader.basic + +import io.circe.Decoder +import sttp.capabilities.WebSockets +import sttp.client3.SttpBackend +import sttp.client3.impl.zio.RIOMonadAsyncError +import sttp.client3.testing.SttpBackendStub +import sttp.monad.MonadError +import za.co.absa.atum.model.dto.PartitionDTO +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax +import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.server.ServerConfig +import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertTrue} +import zio.{Scope, Task} + +object Reader_ZIOUnitTests extends ZIOSpecDefault { + private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") + + private class ReaderForTest[F[_]](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) + extends Reader { + override def getQuery[R: Decoder](endpointUri: String, params: Map[String, String]): F[RequestResult[R]] = super.getQuery(endpointUri, params) + } + + override def spec: Spec[TestEnvironment with Scope, Any] = { + suite("Reader_ZIO")( + test("Using ZIO based backend") { + import za.co.absa.atum.reader.implicits.zio.ZIOMonad + + val partitionDTO = PartitionDTO("someKey", "someValue") + + implicit val server: SttpBackendStub[Task, WebSockets] = SttpBackendStub[Task, WebSockets](new RIOMonadAsyncError[Any]) + .whenAnyRequest.thenRespond(partitionDTO.asJsonString) + + val reader = new ReaderForTest + val expected: RequestResult[PartitionDTO] = Right(partitionDTO) + for { + result <- reader.getQuery[PartitionDTO]("test/", Map.empty) + } yield assertTrue(result == expected) + } + ) + } + +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index bc8de7a84..926c90bd8 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -17,12 +17,25 @@ package za.co.absa.atum.reader import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.SttpBackend +import sttp.client3.testing.SttpBackendStub +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.reader.server.ServerConfig +import za.co.absa.atum.reader.implicits.future.FutureMonad -import za.co.absa.atum.reader.server.future.ArmeriaServerConnection.serverConnection +import scala.concurrent.Future class FlowReaderUnitTests extends AnyFunSuiteLike { + private implicit val severConfig: ServerConfig = ServerConfig.fromConfig() + test("foo") { - val expected = new FlowReader().foo() - assert(expected == "bar") + val atumPartitions: AtumPartitions = AtumPartitions(List( + "a" -> "b", + "c" -> "d" + )) + implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture + + val result = new FlowReader(atumPartitions).foo() + assert(result == "bar") } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index 6fdd72394..f785fe604 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -17,12 +17,27 @@ package za.co.absa.atum.reader import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.SttpBackend +import sttp.client3.testing.SttpBackendStub +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.reader.server.ServerConfig +import za.co.absa.atum.reader.implicits.future.FutureMonad + +import scala.concurrent.Future + -import za.co.absa.atum.reader.server.future.ArmeriaServerConnection.serverConnection class PartitioningReaderUnitTests extends AnyFunSuiteLike { + private implicit val severConfig: ServerConfig = ServerConfig.fromConfig() + test("foo") { - val expected = new PartitioningReader().foo() - assert(expected == "bar") + val atumPartitions: AtumPartitions = AtumPartitions(List( + "a" -> "b", + "c" -> "d" + )) + //implicit val monad: FutureMonad = new FutureMonad() + implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture + val result = PartitioningReader(atumPartitions).foo() + assert(result == "bar") } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala new file mode 100644 index 000000000..cba90f5df --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.capabilities +import sttp.client3._ +import sttp.client3.monad.IdMonad +import sttp.client3.testing.SttpBackendStub +import sttp.model._ +import za.co.absa.atum.model.dto.PartitioningWithIdDTO +import za.co.absa.atum.model.envelopes.NotFoundErrorResponse +import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse +import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax +import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.server.ServerConfig + +class ReaderWithPartitioningIdUnitTests extends AnyFunSuiteLike { + private val serverUrl = "http://localhost:8080" + private val atumPartitionsToReply = AtumPartitions("a", "b") + private val atumPartitionsToFailedDecode = AtumPartitions("c", "d") + private val atumPartitionsToNotFound = AtumPartitions(List.empty) + + private implicit val serverConfig: ServerConfig = ServerConfig(serverUrl) + private implicit val monad: IdMonad.type = IdMonad + private implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatches(request => isUriOfAtumPartitions(request.uri, atumPartitionsToReply)) + .thenRespond(SingleSuccessResponse(PartitioningWithIdDTO(1, atumPartitionsToReply.toPartitioningDTO, "Gimli")).asJsonString) + .whenRequestMatches(request => isUriOfAtumPartitions(request.uri, atumPartitionsToFailedDecode)) + .thenRespond("This is not a correct JSON") + .whenRequestMatches(request => isUriOfAtumPartitions(request.uri, atumPartitionsToNotFound)) + .thenRespond(NotFoundErrorResponse("Partitioning not found").asJsonString, StatusCode.NotFound) + + private def isUriOfAtumPartitions(uri: Uri, atumPartitions: AtumPartitions): Boolean = { + val encodedPartitions = atumPartitions.toPartitioningDTO.asBase64EncodedJsonString + val targetUri = uri"$serverUrl/api/v2/partitionings?partitioning=$encodedPartitions" + uri == targetUri + } + + + private case class ReaderWithPartitioningIdForTest[F[_]](partitioning: AtumPartitions) + (implicit serverConfig: ServerConfig) + extends ReaderWithPartitioningId { + override def partitioningId(): Identity[RequestResult[Long]] = super.partitioningId() + } + + + test("Gets the partitioning id") { + val reader = ReaderWithPartitioningIdForTest(atumPartitionsToReply) + val response = reader.partitioningId() + val result: Long = response.getOrElse(throw new Exception("Failed to get partitioning id")) + assert(result == 1) + } + + test("Not found on the partitioning id") { + val reader = ReaderWithPartitioningIdForTest(atumPartitionsToNotFound) + val result = reader.partitioningId() + result match { + case Right(_) => fail("Expected a failure, but OK response received") + case Left(_: DeserializationException[CirceError]) => fail("Expected a not found response, but deserialization error received") + case Left(x: HttpError[_]) => + assert(x.body.isInstanceOf[NotFoundErrorResponse]) + assert(x.statusCode == StatusCode.NotFound) + case _ => fail("Unexpected response") + } + } + + test("Failure to decode response body") { + val reader = ReaderWithPartitioningIdForTest(atumPartitionsToFailedDecode) + val result = reader.partitioningId() + assert(result.isLeft) + result.swap.map(e => assert(e.isInstanceOf[DeserializationException[CirceError]])) + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala new file mode 100644 index 000000000..29051a1d0 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import cats.effect.unsafe.implicits.global +import io.circe.Decoder +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.SttpBackend +import sttp.client3.testing.SttpBackendStub +import sttp.monad.{MonadAsyncError, MonadError} +import za.co.absa.atum.model.dto.PartitionDTO +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax +import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.server.ServerConfig + +class Reader_CatsIOUnitTests extends AnyFunSuiteLike { + private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") + + private class ReaderForTest[F[_]](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) + extends Reader { + override def getQuery[R: Decoder](endpointUri: String, params: Map[String, String]): F[RequestResult[R]] = super.getQuery(endpointUri, params) + } + + test("Using Cats IO based backend") { + import cats.effect.IO + import za.co.absa.atum.reader.implicits.io.CatsIOMonad + + val partitionDTO = PartitionDTO("someKey", "someValue") + implicit val server: SttpBackendStub[IO, Any] = SttpBackendStub[IO, Any](implicitly[MonadAsyncError[IO]]) + .whenAnyRequest.thenRespond(partitionDTO.asJsonString) + + val reader = new ReaderForTest + val query = reader.getQuery[PartitionDTO]("/test", Map.empty) + val result = query.unsafeRunSync() + assert(result == Right(partitionDTO)) + + +// .map { result => +// fail("This test is expected to fail") +// } + } + +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala new file mode 100644 index 000000000..9ac933ebd --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import io.circe.Decoder +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.SttpBackend +import sttp.client3.testing.SttpBackendStub +import sttp.monad.MonadError +import za.co.absa.atum.model.dto.PartitionDTO +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax +import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.server.ServerConfig + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} + +class Reader_FutureUnitTests extends AnyFunSuiteLike { + private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") + + private class ReaderForTest[F[_]](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) + extends Reader { + override def getQuery[R: Decoder](endpointUri: String, params: Map[String, String]): F[RequestResult[R]] = super.getQuery(endpointUri, params) + } + + test("Using Future based backend") { + import za.co.absa.atum.reader.implicits.future.FutureMonad + + val partitionDTO = PartitionDTO("someKey", "someValue") + implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture + .whenAnyRequest.thenRespond(partitionDTO.asJsonString) + + val reader = new ReaderForTest + val resultToBe = reader.getQuery[PartitionDTO]("/test", Map.empty) + val result = Await.result(resultToBe, Duration(3, "second")) + assert(result == Right(partitionDTO)) + } + +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala new file mode 100644 index 000000000..d181154df --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.basic + +import io.circe.ParsingFailure +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} +import sttp.model.StatusCode +import za.co.absa.atum.model.dto.PartitionDTO +import za.co.absa.atum.model.envelopes.NotFoundErrorResponse +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax +import za.co.absa.atum.reader.basic.RequestResult._ + +class RequestResultUnitTests extends AnyFunSuiteLike { + test("Response.toRequestResult keeps the right value") { + val partitionDTO = PartitionDTO("someKey", "someValue") + val body = Right(partitionDTO) + val source: Response[Either[ResponseException[String, CirceError], PartitionDTO]] = Response( + body, + StatusCode.Ok + ) + val result = source.toRequestResult + assert(result == body) + } + + test("Response.toRequestResult keeps the left value if it's a CirceError") { + val circeError: CirceError = ParsingFailure("Just a test error", new Exception) + val deserializationException = DeserializationException("This is not a json", circeError) + val body = Left(deserializationException) + val source: Response[Either[ResponseException[String, CirceError], PartitionDTO]] = Response( + body, + StatusCode.Ok + ) + val result = source.toRequestResult + assert(result == body) + } + + test("Response.toRequestResult decodes NotFound error") { + val error = NotFoundErrorResponse("This is a test") + val errorResponse = error.asJsonString + val httpError = HttpError(errorResponse, StatusCode.NotFound) + val source: Response[Either[ResponseException[String, CirceError], PartitionDTO]] = Response( + Left(httpError), + StatusCode.Ok + ) + val result = source.toRequestResult + val expected: RequestResult[PartitionDTO] = Left(HttpError(error, httpError.statusCode)) + assert(result == expected) + } + + test("Response.toRequestResult fails to decode InternalServerErrorResponse error") { + val responseBody = "This is not a json" + val httpError = HttpError(responseBody, StatusCode.InternalServerError) + val source: Response[Either[ResponseException[String, CirceError], PartitionDTO]] = Response( + Left(httpError), + StatusCode.Ok + ) + val result = source.toRequestResult + + assert(result.isLeft) + result.swap.foreach { e => + // investigate the error + assert(e.isInstanceOf[DeserializationException[_]]) + val ce = e.asInstanceOf[DeserializationException[ParsingFailure]] + assert(ce.body == responseBody) + } + } + +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/server/ServerConfigUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/server/ServerConfigUnitTests.scala new file mode 100644 index 000000000..cc5c1dd5a --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/server/ServerConfigUnitTests.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.server + +import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} +import org.scalatest.funsuite.AnyFunSuiteLike + +class ServerConfigUnitTests extends AnyFunSuiteLike { + + test("test build from config") { + val server = "https://rivendell.middleearth.jrrt" + val config: Config = ConfigFactory.empty() + .withValue(ServerConfig.HostKey, ConfigValueFactory.fromAnyRef(server)) + val serverConfig = ServerConfig.fromConfig(config) + assert(serverConfig.host == server) + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala deleted file mode 100644 index 4315afb70..000000000 --- a/reader/src/test/scala/za/co/absa/atum/reader/server/zio/ZioServerConnectionUnitTests.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2024 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.server.zio - -import sttp.client3.{Identity, RequestT, Response} -import za.co.absa.atum.reader.server.GenericServerConnection.RequestResult -import zio.test.ZIOSpecDefault -import zio._ -import zio.test._ - -object ZioServerConnectionUnitTests extends ZIOSpecDefault { - override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("ZioServerConnection")( - test("close does nothing and succeeds") { - val connection = new ZioServerConnection("foo.bar") { - override protected def executeRequest[R](request: RequestT[Identity, RequestResult[R], Any]): Task[Response[RequestResult[R]]] = ??? - } - val expected: Unit = () - for { - result <- connection.close() - } yield assertTrue(result == expected) - } - ) - } -} From 7656f6f1eb6ec6971b7be2fb301a9f7e7de1258a Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 17 Nov 2024 04:56:50 +0100 Subject: [PATCH 22/52] * doc fix --- .../src/main/scala/za/co/absa/atum/reader/basic/Reader.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index 363bb68bb..7d0d7b61b 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -27,11 +27,11 @@ import za.co.absa.atum.reader.basic.RequestResult._ /** * Reader is a base class for reading data from a remote server. - * @param monadError$F$0 - the context bind for the F type; it's MonadError to allow not just map, flatMap but eventually - * also error handling easily on a higher level * @param serverConfig - the configuration hwo to reach the Atum server * @param backend - sttp backend to use to send requests * @tparam F - the monadic effect used to get the data (e.g. Future, IO, Task, etc.) + * the context bind for the F type is MonadError to allow not just map, flatMap but eventually + * also error handling easily on a higher level */ abstract class Reader[F[_]: MonadError](implicit val serverConfig: ServerConfig, val backend: SttpBackend[F, Any]) { From 7641c0781436b0b7b3e326e0517ab585d73cc675 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 17 Nov 2024 05:58:52 +0100 Subject: [PATCH 23/52] * disabled failing test --- .../za/co/absa/atum/agent/AgentServerCompatibilityTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala index 992aabe12..d720100f1 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AgentServerCompatibilityTests.scala @@ -40,7 +40,7 @@ class AgentServerCompatibilityTests extends DBTestSuite { .add(StructField("columnForSum", DoubleType)) // Need to add service & pg run in CI - test("Agent should be compatible with server") { + ignore("Agent should be compatible with server") { val expectedMeasurement = JsonBString( """{"mainValue": {"value": "4", "valueType": "Long"}, "supportValues": {}}""".stripMargin From bc82a5baab11e6c45ee35652a4dc19a1c7b8b6ac Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 18 Nov 2024 02:49:44 +0100 Subject: [PATCH 24/52] * adjustments --- README.md | 62 ------------------- build.sbt | 1 - .../main/postgres/runs/get_measurements.sql | 47 -------------- .../atum/model/envelopes/ErrorResponse.scala | 3 - .../za/co/absa/atum/model/types/basic.scala | 8 --- project/Dependencies.scala | 8 +-- .../za/co/absa/atum/reader/FlowReader.scala | 5 -- .../atum/reader/FlowReaderUnitTests.scala | 6 +- .../reader/PartitioningReaderUnitTests.scala | 1 - 9 files changed, 4 insertions(+), 137 deletions(-) delete mode 100644 database/src/main/postgres/runs/get_measurements.sql diff --git a/README.md b/README.md index 789d762b0..5672b45df 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ - [Measurement](#measurement) - [Checkpoint](#checkpoint) - [Data Flow](#data-flow) - - [Usage](#usage) - - [Reader](#reader-usage) - [How to generate Code coverage report](#how-to-generate-code-coverage-report) - [How to Run in IntelliJ](#how-to-run-in-intellij) - [How to Run Tests](#how-to-run-tests) @@ -158,66 +156,6 @@ We can even say, that `Checkpoint` is a result of particular `Measurements` (ver The journey of a dataset throughout various data transformations and pipelines. It captures the whole journey, even if it involves multiple applications or ETL pipelines. -## Usage - -### Reader usage -Reader module support several asynchronous http clients. The dependencies used for these clients are set as _optional_, -so the user of the module can decide which client to use and include only the necessary dependencies. - -The clients are: - -[//]: # (TODO #298 needs Java 11 cross-build) - -[//]: # (#### Future based `HttpClientServerConnection`) - -[//]: # (Uses `java.net.http.HttpClient` to send requests to the server, therefore requires no additional dependencies. But works ) - -[//]: # (only with Java 11 or higher. ) - -#### Future based `ArmeririaServerConnection` -Add -```scala -"com.softwaremill.sttp.client3" %% "armeria-backend" % "[version]" -``` -to your dependencies. - -#### Cats IO based `ArmeririaServerConnection` -Add -```scala -"org.typelevel." %% "cats-effect" % "[version]" -"com.softwaremill.sttp.client3" %% "armeria-backend-cats" % "[version]" // for cats-effect 3.x -// or -"com.softwaremill.sttp.client3" %% "armeria-backend-cats-ce2" % "[version]" // for cats-effect 2.x -``` -" -to your dependencies. - -[//]: # (TODO #298 needs Java 11 cross-build) - -[//]: # (#### ZIO based `HttpClientServerConnection`) - -[//]: # (Add) - -[//]: # (```scala) - -[//]: # ("com.softwaremill.sttp.client3" %% "zio" % "[version]" // for ZIO 2.x) - -[//]: # ("com.softwaremill.sttp.client3" %% "zio1" % "[version]" // for ZIO 1.x) - -[//]: # (```) - -[//]: # (to your dependencies.) - -#### ZIO based `ArmeririaServerConnection` -Add -```scala -"com.softwaremill.sttp.client3" %% "armeria-backend-zio" % "[version]" // for ZIO 2.x -"com.softwaremill.sttp.client3" %% "armeria-backend-zio1" % "[version]" // for ZIO 1.x -``` -to your dependencies. - - - ## How to generate Code coverage report ```sbt sbt jacoco diff --git a/build.sbt b/build.sbt index 2227b5dc9..d8d73e521 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,6 @@ import Dependencies.* import Dependencies.Versions.spark3 import VersionAxes.* -//ThisBuild / scalaVersion := Setup.scala212.asString // default version TODO ThisBuild / versionScheme := Some("early-semver") diff --git a/database/src/main/postgres/runs/get_measurements.sql b/database/src/main/postgres/runs/get_measurements.sql deleted file mode 100644 index 4aa3b7434..000000000 --- a/database/src/main/postgres/runs/get_measurements.sql +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -CREATE OR REPLACE FUNCTION postgres.runs.get_measurements( - IN i_parameter TEXT, - OUT status INTEGER, - OUT status_text TEXT -) RETURNS record AS -$$ - ------------------------------------------------------------------------------- --- --- Function: postgres.runs.get_measurements([Function_Param_Count]) --- [Description] --- --- Parameters: --- i_parameter - --- --- Returns: --- status - Status code --- status_text - Status text --- --- Status codes: --- 10 - OK --- -------------------------------------------------------------------------------- -DECLARE -BEGIN - -END; -$$ - LANGUAGE plpgsql VOLATILE - SECURITY DEFINER; - -GRANT EXECUTE ON FUNCTION postgres.runs.get_measurements() TO [user]; diff --git a/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala b/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala index e531410e5..5b881c532 100644 --- a/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala +++ b/model/src/main/scala/za/co/absa/atum/model/envelopes/ErrorResponse.scala @@ -23,9 +23,6 @@ import io.circe.generic.semiauto._ import java.util.UUID object ErrorResponse { - implicit val decodeErrorResponse: Decoder[ErrorResponse] = deriveDecoder //TODo neeeded? - implicit val encodeErrorResponse: Encoder[ErrorResponse] = deriveEncoder - def basedOnStatusCode(statusCode: Int, jsonString: String): Either[Error, ErrorResponse] = { statusCode match { case 400 => decode[BadRequestResponse](jsonString) diff --git a/model/src/main/scala/za/co/absa/atum/model/types/basic.scala b/model/src/main/scala/za/co/absa/atum/model/types/basic.scala index 00e2c7cd3..4c5160105 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/basic.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/basic.scala @@ -40,14 +40,6 @@ object basic { ListMap(elems:_*) } - /*TODO private[agent]*/ def toPartitionDTO(atumPartitions: AtumPartitions): PartitioningDTO = { - atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq - } - - /*TOD private[agent]*/ def fromPartitioningDTO(partitioning: PartitioningDTO): AtumPartitions = { - AtumPartitions(partitioning.map(partition => Tuple2(partition.key, partition.value)).toList) - } - } implicit class AtumPartitionsOps(val atumPartitions: AtumPartitions) extends AnyVal { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ace114abf..47dbf937e 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -256,9 +256,6 @@ object Dependencies { // testing lazy val zioTest = zioOrg %% "zio-test" % Versions.zio % Test - lazy val zioTestSbt = zioOrg %% "zio-test-sbt" % Versions.zio % Test - lazy val zioTestJunit = zioOrg %% "zio-test-junit" % Versions.zio % Test - lazy val sbtJunitInterface = sbtOrg % "junit-interface" % Versions.sbtJunitInterface % Test Seq( sttpCore, @@ -266,10 +263,7 @@ object Dependencies { sttpCats, catsEffect, sttpZio, - zioTest, -// zioTestSbt, -// zioTestJunit, -// sbtJunitInterface + zioTest ) ++ testDependencies ++ jsonSerdeDependencies diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 073340f78..bddd17ebf 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -36,9 +36,4 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) override def partitioning: AtumPartitions = mainFlowPartitioning - def foo(): String = { - // just to have some testable content - "bar" - } - } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index 926c90bd8..b276f3bce 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -28,14 +28,14 @@ import scala.concurrent.Future class FlowReaderUnitTests extends AnyFunSuiteLike { private implicit val severConfig: ServerConfig = ServerConfig.fromConfig() - test("foo") { + test("mainFlowPartitioning is the same as partitioning") { val atumPartitions: AtumPartitions = AtumPartitions(List( "a" -> "b", "c" -> "d" )) implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture - val result = new FlowReader(atumPartitions).foo() - assert(result == "bar") + val result = new FlowReader(atumPartitions).mainFlowPartitioning + assert(result == atumPartitions) } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index f785fe604..1647fa9d3 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -35,7 +35,6 @@ class PartitioningReaderUnitTests extends AnyFunSuiteLike { "a" -> "b", "c" -> "d" )) - //implicit val monad: FutureMonad = new FutureMonad() implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture val result = PartitioningReader(atumPartitions).foo() assert(result == "bar") From 0e7675e228c133c60bcfb92a76d544d8db5ef2f6 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 18 Nov 2024 03:17:41 +0100 Subject: [PATCH 25/52] - further cleaning --- agent/README.md | 4 +--- build.sbt | 1 - project/Dependencies.scala | 12 ++++++------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/agent/README.md b/agent/README.md index 04fe5b67b..ed4247d2a 100644 --- a/agent/README.md +++ b/agent/README.md @@ -12,11 +12,9 @@ Create multiple `AtumContext` with different control measures to be applied ### Option 1 ```scala val atumContextInstanceWithRecordCount = AtumContext(processor = processor) - .withMeasureAdded(RecordCount(MockMeasureNames.recordCount1, controlCol = "id")) - .withMeasureAdded(RecordCount(MockMeasureNames.recordCount1, measuredColumn = "id")) + .withMeasureAdded(RecordCount(MockMeasureNames.recordCount1)) val atumContextWithSalaryAbsMeasure = atumContextInstanceWithRecordCount - .withMeasureAdded(AbsSumOfValuesOfColumn(controlCol = "salary")) .withMeasureAdded(AbsSumOfValuesOfColumn(measuredColumn = "salary")) ``` diff --git a/build.sbt b/build.sbt index d8d73e521..513e17e85 100644 --- a/build.sbt +++ b/build.sbt @@ -34,7 +34,6 @@ initialize := { //this routine can be used to assert the required Java version } -Test/parallelExecution := false enablePlugins(FlywayPlugin) flywayUrl := FlywayConfiguration.flywayUrl diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 47dbf937e..5fe116bd5 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -244,18 +244,18 @@ object Dependencies { val typeLevelOrg = "org.typelevel" // STTP core and Circe integration - lazy val sttpCore = sttpClient3Org %% "core" % Versions.sttpClient - lazy val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpClient + val sttpCore = sttpClient3Org %% "core" % Versions.sttpClient + val sttpCirce = sttpClient3Org %% "circe" % Versions.sttpClient // Cats backend - lazy val catsEffect = typeLevelOrg %% "cats-effect" % Versions.catsEffect % Optional - lazy val sttpCats = sttpClient3Org %% "cats" % Versions.sttpClient % Optional + val catsEffect = typeLevelOrg %% "cats-effect" % Versions.catsEffect % Optional + val sttpCats = sttpClient3Org %% "cats" % Versions.sttpClient % Optional // ZIO backend - lazy val sttpZio = sttpClient3Org %% "zio" % Versions.sttpClient % Optional + val sttpZio = sttpClient3Org %% "zio" % Versions.sttpClient % Optional // testing - lazy val zioTest = zioOrg %% "zio-test" % Versions.zio % Test + val zioTest = zioOrg %% "zio-test" % Versions.zio % Test Seq( sttpCore, From 432716ae8b4ff312df0cac1c85698ef1d0c722f1 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Thu, 21 Nov 2024 23:10:55 +0100 Subject: [PATCH 26/52] * tests progress --- ...ensionsUnitTests.scala => SerializationUtilsUnitTests.scala} | 2 +- project/JacocoSetup.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename model/src/test/scala/za/co/absa/atum/model/utils/{JsonSyntaxExtensionsUnitTests.scala => SerializationUtilsUnitTests.scala} (99%) diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensionsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala similarity index 99% rename from model/src/test/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensionsUnitTests.scala rename to model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala index 6491dc501..e34aa2401 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensionsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala @@ -26,7 +26,7 @@ import za.co.absa.atum.model.testing.implicits.StringImplicits.StringLinearizati import java.time.{ZoneId, ZoneOffset, ZonedDateTime} import java.util.UUID -class JsonSyntaxExtensionsUnitTests extends AnyFlatSpecLike { +class SerializationUtilsUnitTests extends AnyFlatSpecLike { // AdditionalDataDTO "asJsonString" should "serialize AdditionalDataDTO into json string" in { diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 40e09bcf2..77d21b8c2 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -51,7 +51,7 @@ object JacocoSetup { "za.co.absa.atum.server.Constants*", "za.co.absa.atum.server.api.database.DoobieImplicits*", "za.co.absa.atum.server.api.database.TransactorProvider*", - "za.co.absa.atum.model.dto.*", +//TDO "za.co.absa.atum.model.dto.*", "za.co.absa.atum.model.envelopes.Pagination", "za.co.absa.atum.model.envelopes.ResponseEnvelope", "za.co.absa.atum.model.envelopes.StatusResponse", From 11b0a16c64c3b46dc99b91301b36ffd07e9773e0 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 22 Nov 2024 15:09:18 +0100 Subject: [PATCH 27/52] * several UTs added --- build.sbt | 1 + .../za/co/absa/atum/model/types/basic.scala | 3 +- .../model/utils/JsonSyntaxExtensions.scala | 4 +- .../SerializationUtilsUnitTests.scala | 5 +- .../model/types/AtumPartitionsUnitTests.scala | 85 ++++++++++++++++ .../JsonDeserializationSyntaxUnitTests.scala | 98 +++++++++++++++++++ .../JsonSerializationSyntaxUnitTests.scala | 57 +++++++++++ project/JacocoSetup.scala | 1 - .../reader/basic/Reader_ZIOUnitTests.scala | 72 ++++++++------ 9 files changed, 290 insertions(+), 36 deletions(-) rename model/src/test/scala/za/co/absa/atum/model/{utils => dto}/SerializationUtilsUnitTests.scala (99%) create mode 100644 model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala create mode 100644 model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala create mode 100644 model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala diff --git a/build.sbt b/build.sbt index 513e17e85..c02b9bf47 100644 --- a/build.sbt +++ b/build.sbt @@ -20,6 +20,7 @@ import Dependencies.* import Dependencies.Versions.spark3 import VersionAxes.* +ThisBuild / scalaVersion := Setup.scala213.asString ThisBuild / versionScheme := Some("early-semver") diff --git a/model/src/main/scala/za/co/absa/atum/model/types/basic.scala b/model/src/main/scala/za/co/absa/atum/model/types/basic.scala index 4c5160105..2631f243c 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/basic.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/basic.scala @@ -16,10 +16,9 @@ package za.co.absa.atum.model.types +import scala.collection.immutable.ListMap import za.co.absa.atum.model.dto.{PartitionDTO, PartitioningDTO} -import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import scala.collection.immutable.ListMap object basic { /** diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index e9e49fe81..7b1c17e7a 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.model.utils import io.circe.parser.decode import io.circe.syntax._ -import io.circe.{Decoder, Encoder, parser} +import io.circe.{Decoder, Encoder} import java.util.Base64 @@ -49,7 +49,7 @@ object JsonSyntaxExtensions { def fromBase64As[T: Decoder]: Either[io.circe.Error, T] = { val decodedBytes = Base64.getDecoder.decode(jsonStr) val decodedString = new String(decodedBytes, "UTF-8") - parser.decode[T](decodedString) + decode[T](decodedString) } } diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala similarity index 99% rename from model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala rename to model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala index e34aa2401..4c3704779 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala @@ -14,14 +14,13 @@ * limitations under the License. */ -package za.co.absa.atum.model.utils +package za.co.absa.atum.model.dto import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue -import za.co.absa.atum.model.dto._ -import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import za.co.absa.atum.model.testing.implicits.StringImplicits.StringLinearization +import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ import java.time.{ZoneId, ZoneOffset, ZonedDateTime} import java.util.UUID diff --git a/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala new file mode 100644 index 000000000..11a529d02 --- /dev/null +++ b/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala @@ -0,0 +1,85 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.types + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.model.dto.PartitionDTO +import za.co.absa.atum.model.types.basic.AtumPartitions + +import scala.collection.immutable.ListMap + + + +class AtumPartitionsUnitTests extends AnyFunSuiteLike { + test("Creating AtumPartitions from one pair of key-value") { + val expected = ListMap("a" -> "b") + val result = AtumPartitions(("a", "b")) + assert(result == expected) + } + + test("Creating AtumPartitions from multiple key-value pairs") { + val expected = ListMap( + "a" -> "b", + "e" -> "Hello", + "c" -> "d" + ) + val result = AtumPartitions(List( + ("a", "b"), + ("e", "Hello"), + ("c", "d") + )) + assert(result == expected) + assert(result.head == ("a", "b")) + } + + test("Conversion to PartitioningDTO returns expected result") { + import za.co.absa.atum.model.types.basic.AtumPartitionsOps + + val atumPartitions = AtumPartitions(List( + ("a", "b"), + ("e", "Hello"), + ("c", "d") + )) + + val expected = Seq( + PartitionDTO("a", "b"), + PartitionDTO("e", "Hello"), + PartitionDTO("c", "d") + ) + + assert(atumPartitions.toPartitioningDTO == expected) + } + + test("Creating AtumPartitions from PartitioningDTO") { + import za.co.absa.atum.model.types.basic.PartitioningDTOOps + + val partitionDTO = Seq( + PartitionDTO("a", "b"), + PartitionDTO("e", "Hello"), + PartitionDTO("c", "d") + ) + + val expected = AtumPartitions(List( + ("a", "b"), + ("e", "Hello"), + ("c", "d") + )) + + assert(partitionDTO.toAtumPartitions == expected) + } + +} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala new file mode 100644 index 000000000..4ca1ac9df --- /dev/null +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala @@ -0,0 +1,98 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.utils + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.model.dto.{FlowDTO, PartitionDTO} +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonDeserializationSyntax + +class JsonDeserializationSyntaxUnitTests extends AnyFunSuiteLike { + test("Decode object from Json with defined Option field") { + val source = + """{ + | "id": 1, + | "name": "Test flow", + | "description": "Having description", + | "fromPattern": false + |}""".stripMargin + val result = source.as[FlowDTO] + val expected = FlowDTO( + id = 1, + name = "Test flow", + description = Some("Having description"), + fromPattern = false + ) + assert(result == expected) + } + + test("Decode object from Json with Option field undefined") { + val source = + """{ + | "id": 1, + | "name": "Test flow", + | "fromPattern": true + |}""".stripMargin + val result = source.as[FlowDTO] + val expected = FlowDTO( + id = 1, + name = "Test flow", + description = None, + fromPattern = true + ) + assert(result == expected) + } + + test("Fail when input is not Json") { + val source = "This is not a Json!" + intercept[io.circe.Error] { + source.as[FlowDTO] + } + } + + test("Fail when given wrong class") { + val source = + """{ + | "id": 1, + | "name": "Test flow", + | "description": "Having description", + | "fromPattern": false + |}""".stripMargin + intercept[io.circe.Error] { + source.as[PartitionDTO] + } + } + + + test("Decode object from Base64 string") { + val source = "eyJpZCI6MSwibmFtZSI6IlRlc3QgZmxvdyIsImRlc2NyaXB0aW9uIjpudWxsLCJmcm9tUGF0dGVybiI6ZmFsc2V9" + val result = source.fromBase64As[FlowDTO] + val expected = FlowDTO( + id = 1, + name = "Test flow", + description = None, + fromPattern = false + ) + assert(result == Right(expected)) + } + + test("Failing decode if not Base64 string") { + val source = "" + val result = source.fromBase64As[FlowDTO] + assert(result.isLeft) + } + +} diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala new file mode 100644 index 000000000..830f4e9b7 --- /dev/null +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala @@ -0,0 +1,57 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.utils + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.model.dto.FlowDTO +import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax + +class JsonSerializationSyntaxUnitTests extends AnyFunSuiteLike { + test("Converting to Json with option field defined") { + val expected = """{"id":1,"name":"Test flow","description":"Having description","fromPattern":false}""" + val result = FlowDTO( + id = 1, + name = "Test flow", + description = Some("Having description"), + fromPattern = false + ).asJsonString + assert(result == expected) + } + + test("Converting to Json with option field undefined") { + val expected = """{"id":1,"name":"Test flow","description":null,"fromPattern":true}""" + val result = FlowDTO( + id = 1, + name = "Test flow", + description = None, + fromPattern = true + ).asJsonString + assert(result == expected) + } + + test("Converting to Base64") { + val expected = "eyJpZCI6MSwibmFtZSI6IlRlc3QgZmxvdyIsImRlc2NyaXB0aW9uIjpudWxsLCJmcm9tUGF0dGVybiI6ZmFsc2V9" + val result = FlowDTO( + id = 1, + name = "Test flow", + description = None, + fromPattern = false + ).asBase64EncodedJsonString + assert(result == expected) + } + +} diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 77d21b8c2..4afcc950a 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -51,7 +51,6 @@ object JacocoSetup { "za.co.absa.atum.server.Constants*", "za.co.absa.atum.server.api.database.DoobieImplicits*", "za.co.absa.atum.server.api.database.TransactorProvider*", -//TDO "za.co.absa.atum.model.dto.*", "za.co.absa.atum.model.envelopes.Pagination", "za.co.absa.atum.model.envelopes.ResponseEnvelope", "za.co.absa.atum.model.envelopes.StatusResponse", diff --git a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala index 7b4e2dfb6..181bc9695 100644 --- a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala +++ b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.reader.basic import io.circe.Decoder @@ -13,31 +29,31 @@ import za.co.absa.atum.reader.server.ServerConfig import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertTrue} import zio.{Scope, Task} -object Reader_ZIOUnitTests extends ZIOSpecDefault { - private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") - - private class ReaderForTest[F[_]](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) - extends Reader { - override def getQuery[R: Decoder](endpointUri: String, params: Map[String, String]): F[RequestResult[R]] = super.getQuery(endpointUri, params) - } - - override def spec: Spec[TestEnvironment with Scope, Any] = { - suite("Reader_ZIO")( - test("Using ZIO based backend") { - import za.co.absa.atum.reader.implicits.zio.ZIOMonad - - val partitionDTO = PartitionDTO("someKey", "someValue") - - implicit val server: SttpBackendStub[Task, WebSockets] = SttpBackendStub[Task, WebSockets](new RIOMonadAsyncError[Any]) - .whenAnyRequest.thenRespond(partitionDTO.asJsonString) - - val reader = new ReaderForTest - val expected: RequestResult[PartitionDTO] = Right(partitionDTO) - for { - result <- reader.getQuery[PartitionDTO]("test/", Map.empty) - } yield assertTrue(result == expected) - } - ) - } - -} +//object Reader_ZIOUnitTests extends ZIOSpecDefault { +// private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") +// +// private class ReaderForTest[F[_]](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) +// extends Reader { +// override def getQuery[R: Decoder](endpointUri: String, params: Map[String, String]): F[RequestResult[R]] = super.getQuery(endpointUri, params) +// } +// +// override def spec: Spec[TestEnvironment with Scope, Any] = { +// suite("Reader_ZIO")( +// test("Using ZIO based backend") { +// import za.co.absa.atum.reader.implicits.zio.ZIOMonad +// +// val partitionDTO = PartitionDTO("someKey", "someValue") +// +// implicit val server: SttpBackendStub[Task, WebSockets] = SttpBackendStub[Task, WebSockets](new RIOMonadAsyncError[Any]) +// .whenAnyRequest.thenRespond(partitionDTO.asJsonString) +// +// val reader = new ReaderForTest +// val expected: RequestResult[PartitionDTO] = Right(partitionDTO) +// for { +// result <- reader.getQuery[PartitionDTO]("test/", Map.empty) +// } yield assertTrue(result == expected) +// } +// ) +// } +// +//} From e07dffbdabf38bcabebe60c0da1e53ff7a97824f Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 24 Nov 2024 02:47:51 +0100 Subject: [PATCH 28/52] * last improvements before PR ready --- .github/workflows/jacoco_report.yml | 16 +++++----- .../za/co/absa/atum/agent/AtumContext.scala | 29 +------------------ .../reader/basic/Reader_ZIOUnitTests.scala | 3 ++ 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index 0f3157b95..a076b4e37 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -109,14 +109,14 @@ jobs: - name: Get the Coverage info if: steps.jacocorun.outcome == 'success' run: | - echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" - echo "Total agent module coverage ${{ steps.jacoco-reader.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-reader.outputs.coverage-changed-files }}" - echo "Total model module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-model.outputs.coverage-changed-files }}" - echo "Total server module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" + echo "Total 'agent' module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" + echo "Changed files of 'agent' module coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" + echo "Total 'reader' module coverage ${{ steps.jacoco-reader.outputs.coverage-overall }}" + echo "Changed files of 'reader' module coverage ${{ steps.jacoco-reader.outputs.coverage-changed-files }}" + echo "Total 'model' module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}" + echo "Changed files of 'model' module coverage ${{ steps.jacoco-model.outputs.coverage-changed-files }}" + echo "Total 'server' module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" + echo "Changed files of'server' module coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - name: Fail PR if changed files coverage is less than ${{ env.coverage-changed-files }}% if: steps.jacocorun.outcome == 'success' uses: actions/github-script@v6 diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index 7fc8f8311..f5f9a8591 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -205,34 +205,7 @@ class AtumContext private[agent] ( } object AtumContext { -// TODO --- -// /** -// * Type alias for Atum partitions. -// */ -// type AtumPartitions = ListMap[String, String] -// type AdditionalData = Map[String, Option[String]] -// -// /** -// * Object contains helper methods to work with Atum partitions. -// */ -// object AtumPartitions { -// def apply(elems: (String, String)): AtumPartitions = { -// ListMap(elems) -// } -// -// def apply(elems: List[(String, String)]): AtumPartitions = { -// ListMap(elems:_*) -// } -// -// private[agent] def toSeqPartitionDTO(atumPartitions: AtumPartitions): PartitioningDTO = { -// atumPartitions.map { case (key, value) => PartitionDTO(key, value) }.toSeq -// } -// -// private[agent] def fromPartitioning(partitioning: PartitioningDTO): AtumPartitions = { -// AtumPartitions(partitioning.map(partition => Tuple2(partition.key, partition.value)).toList) -// } -// } -// + private[agent] def fromDTO(atumContextDTO: AtumContextDTO, agent: AtumAgent): AtumContext = { new AtumContext( atumContextDTO.partitioning.toAtumPartitions, diff --git a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala index 181bc9695..98de30598 100644 --- a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala +++ b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala @@ -29,6 +29,9 @@ import za.co.absa.atum.reader.server.ServerConfig import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertTrue} import zio.{Scope, Task} +// This test is disabled as is breaks on JaCoCo execution +// Once the problem is figured out or how to cirmvent it, this can be re-enabled +// //object Reader_ZIOUnitTests extends ZIOSpecDefault { // private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") // From 3955a5072ebc856ab141260a0a5c116f0520b498 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 25 Nov 2024 10:57:10 +0100 Subject: [PATCH 29/52] * description to class `ServerConfig` --- .../scala/za/co/absa/atum/reader/server/ServerConfig.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala b/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala index f38eff892..92e90eb8e 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/server/ServerConfig.scala @@ -18,6 +18,10 @@ package za.co.absa.atum.reader.server import com.typesafe.config.{Config, ConfigFactory} +/** + * A case class representing the configuration to connect to an Atum server instance. + * @param host The URL of the Atum server instance. + */ case class ServerConfig (host: String) object ServerConfig { From b287a666a8157a477513f7c28cf524a37c9d5d34 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 25 Nov 2024 10:59:22 +0100 Subject: [PATCH 30/52] * removed empty line --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 01b98684d..839e11b56 100644 --- a/build.sbt +++ b/build.sbt @@ -41,7 +41,6 @@ initialize := { } } - enablePlugins(FlywayPlugin) flywayUrl := FlywayConfiguration.flywayUrl flywayUser := FlywayConfiguration.flywayUser From d04d23b522f6b6c5308a127e4dfe8aacaf4e6402 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 27 Nov 2024 22:05:51 +0100 Subject: [PATCH 31/52] * addressed PR comments --- .../model/utils/JsonSyntaxExtensions.scala | 2 +- .../dto/SerializationUtilsUnitTests.scala | 2 +- .../testing/implicits/StringImplicits.scala | 2 +- project/Dependencies.scala | 11 +--- .../co/absa/atum/reader/implicits/zio.scala | 23 ------- .../za/co/absa/atum/reader/FlowReader.scala | 5 +- .../absa/atum/reader/PartitioningReader.scala | 4 +- ...gId.scala => PartitioningIdProvider.scala} | 7 +-- .../za/co/absa/atum/reader/basic/Reader.scala | 4 -- .../absa/atum/reader/implicits/future.scala | 2 +- .../za/co/absa/atum/reader/implicits/io.scala | 2 +- .../reader/basic/Reader_ZIOUnitTests.scala | 62 ------------------- .../atum/reader/FlowReaderUnitTests.scala | 2 +- .../reader/PartitioningReaderUnitTests.scala | 2 +- ... => PartitioningIdProviderUnitTests.scala} | 10 +-- .../reader/basic/Reader_CatsIOUnitTests.scala | 7 +-- .../reader/basic/Reader_FutureUnitTests.scala | 2 +- 17 files changed, 23 insertions(+), 126 deletions(-) delete mode 100644 reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala rename reader/src/main/scala/za/co/absa/atum/reader/basic/{ReaderWithPartitioningId.scala => PartitioningIdProvider.scala} (83%) delete mode 100644 reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala rename reader/src/test/scala/za/co/absa/atum/reader/basic/{ReaderWithPartitioningIdUnitTests.scala => PartitioningIdProviderUnitTests.scala} (91%) diff --git a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala index 7b1c17e7a..3f7b7457f 100644 --- a/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala +++ b/model/src/main/scala/za/co/absa/atum/model/utils/JsonSyntaxExtensions.scala @@ -42,7 +42,7 @@ object JsonSyntaxExtensions { } } - def asSafe[T: Decoder]: Either[io.circe.Error, T] = { + private def asSafe[T: Decoder]: Either[io.circe.Error, T] = { decode[T](jsonStr) } diff --git a/model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala index 4c3704779..045cb7e3b 100644 --- a/model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/dto/SerializationUtilsUnitTests.scala @@ -19,8 +19,8 @@ package za.co.absa.atum.model.dto import org.scalatest.flatspec.AnyFlatSpecLike import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue -import za.co.absa.atum.model.testing.implicits.StringImplicits.StringLinearization import za.co.absa.atum.model.utils.JsonSyntaxExtensions._ +import za.co.absa.atum.testing.implicits.StringImplicits.StringLinearization import java.time.{ZoneId, ZoneOffset, ZonedDateTime} import java.util.UUID diff --git a/model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala b/model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala index ec64d2062..c513cb77e 100644 --- a/model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala +++ b/model/src/test/scala/za/co/absa/atum/testing/implicits/StringImplicits.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.model.testing.implicits +package za.co.absa.atum.testing.implicits object StringImplicits { implicit class StringLinearization(val str: String) extends AnyVal { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5fe116bd5..3538c0e62 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -238,7 +238,6 @@ object Dependencies { } def readerDependencies(scalaVersion: Version): Seq[ModuleID] = { - val zioOrg = "dev.zio" val sbtOrg = "com.github.sbt" val sttpClient3Org = "com.softwaremill.sttp.client3" val typeLevelOrg = "org.typelevel" @@ -251,19 +250,11 @@ object Dependencies { val catsEffect = typeLevelOrg %% "cats-effect" % Versions.catsEffect % Optional val sttpCats = sttpClient3Org %% "cats" % Versions.sttpClient % Optional - // ZIO backend - val sttpZio = sttpClient3Org %% "zio" % Versions.sttpClient % Optional - - // testing - val zioTest = zioOrg %% "zio-test" % Versions.zio % Test - Seq( sttpCore, sttpCirce, sttpCats, - catsEffect, - sttpZio, - zioTest + catsEffect ) ++ testDependencies ++ jsonSerdeDependencies diff --git a/reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala b/reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala deleted file mode 100644 index 41651397a..000000000 --- a/reader/src/main/scala-2.13/za/co/absa/atum/reader/implicits/zio.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.implicits - -import sttp.client3.impl.zio.RIOMonadAsyncError - -object zio { - implicit val ZIOMonad: RIOMonadAsyncError[Any] = new RIOMonadAsyncError[Any] -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index bddd17ebf..f952dc562 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import sttp.client3.SttpBackend import sttp.monad.MonadError import za.co.absa.atum.model.types.basic.AtumPartitions -import za.co.absa.atum.reader.basic.ReaderWithPartitioningId +import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.server.ServerConfig /** @@ -32,7 +32,8 @@ import za.co.absa.atum.reader.server.ServerConfig * @tparam F - the effect type (e.g. Future, IO, Task, etc.) */ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) - (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) extends ReaderWithPartitioningId[F] { + (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) + extends Reader[F] with PartitioningIdProvider[F]{ override def partitioning: AtumPartitions = mainFlowPartitioning diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index 2c3782ffc..7cd5db701 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import sttp.client3.SttpBackend import sttp.monad.MonadError import za.co.absa.atum.model.types.basic.AtumPartitions -import za.co.absa.atum.reader.basic.ReaderWithPartitioningId +import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.server.ServerConfig /** @@ -33,7 +33,7 @@ import za.co.absa.atum.reader.server.ServerConfig */ case class PartitioningReader[F[_]](partitioning: AtumPartitions) (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) - extends ReaderWithPartitioningId[F] { + extends Reader[F] with PartitioningIdProvider[F]{ def foo(): String = { // just to have some testable content "bar" diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala similarity index 83% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala rename to reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala index e733da82f..b32388a12 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningId.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.reader.basic -import sttp.client3.SttpBackend import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.dto.PartitioningWithIdDTO @@ -25,13 +24,11 @@ import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.model.types.basic.AtumPartitionsOps import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.reader.basic.RequestResult.RequestResult -import za.co.absa.atum.reader.server.ServerConfig -abstract class ReaderWithPartitioningId[F[_]: MonadError](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any]) - extends Reader[F] { +trait PartitioningIdProvider[F[_]] {self: Reader[F] => def partitioning: AtumPartitions - protected def partitioningId(): F[RequestResult[Long]] = { + def partitioningId()(implicit monad: MonadError[F]): F[RequestResult[Long]] = { val encodedPartitioning = partitioning.toPartitioningDTO.asBase64EncodedJsonString val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]]("/api/v2/partitionings", Map("partitioning" -> encodedPartitioning)) queryResult.map{result => diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index 7d0d7b61b..251d4b52d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -47,7 +47,3 @@ abstract class Reader[F[_]: MonadError](implicit val serverConfig: ServerConfig, response.map(_.toRequestResult) } } - -object Reader { - -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala index 0656bed77..23f6c0f80 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/future.scala @@ -21,5 +21,5 @@ import sttp.monad.{FutureMonad => SttpFutureMonad} import scala.concurrent.ExecutionContext.Implicits.global object future { - implicit val FutureMonad: SttpFutureMonad = new SttpFutureMonad + final implicit val futureMonadError: SttpFutureMonad = new SttpFutureMonad } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala index b43501da0..96a148e13 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/io.scala @@ -20,5 +20,5 @@ import cats.effect.IO import sttp.client3.impl.cats.CatsMonadAsyncError object io { - implicit val CatsIOMonad: CatsMonadAsyncError[IO] = new CatsMonadAsyncError[IO] + final implicit val catsIOMonadError: CatsMonadAsyncError[IO] = new CatsMonadAsyncError[IO] } diff --git a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala b/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala deleted file mode 100644 index 98de30598..000000000 --- a/reader/src/test/scala-2.13/za/co/absa/atum/reader/basic/Reader_ZIOUnitTests.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2024 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.basic - -import io.circe.Decoder -import sttp.capabilities.WebSockets -import sttp.client3.SttpBackend -import sttp.client3.impl.zio.RIOMonadAsyncError -import sttp.client3.testing.SttpBackendStub -import sttp.monad.MonadError -import za.co.absa.atum.model.dto.PartitionDTO -import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult -import za.co.absa.atum.reader.server.ServerConfig -import zio.test.{Spec, TestEnvironment, ZIOSpecDefault, assertTrue} -import zio.{Scope, Task} - -// This test is disabled as is breaks on JaCoCo execution -// Once the problem is figured out or how to cirmvent it, this can be re-enabled -// -//object Reader_ZIOUnitTests extends ZIOSpecDefault { -// private implicit val serverConfig: ServerConfig = ServerConfig("http://localhost:8080") -// -// private class ReaderForTest[F[_]](implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) -// extends Reader { -// override def getQuery[R: Decoder](endpointUri: String, params: Map[String, String]): F[RequestResult[R]] = super.getQuery(endpointUri, params) -// } -// -// override def spec: Spec[TestEnvironment with Scope, Any] = { -// suite("Reader_ZIO")( -// test("Using ZIO based backend") { -// import za.co.absa.atum.reader.implicits.zio.ZIOMonad -// -// val partitionDTO = PartitionDTO("someKey", "someValue") -// -// implicit val server: SttpBackendStub[Task, WebSockets] = SttpBackendStub[Task, WebSockets](new RIOMonadAsyncError[Any]) -// .whenAnyRequest.thenRespond(partitionDTO.asJsonString) -// -// val reader = new ReaderForTest -// val expected: RequestResult[PartitionDTO] = Right(partitionDTO) -// for { -// result <- reader.getQuery[PartitionDTO]("test/", Map.empty) -// } yield assertTrue(result == expected) -// } -// ) -// } -// -//} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index b276f3bce..cae0d852f 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -21,7 +21,7 @@ import sttp.client3.SttpBackend import sttp.client3.testing.SttpBackendStub import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.implicits.future.FutureMonad +import za.co.absa.atum.reader.implicits.future.futureMonadError import scala.concurrent.Future diff --git a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala index 1647fa9d3..183bfe851 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/PartitioningReaderUnitTests.scala @@ -21,7 +21,7 @@ import sttp.client3.SttpBackend import sttp.client3.testing.SttpBackendStub import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.implicits.future.FutureMonad +import za.co.absa.atum.reader.implicits.future.futureMonadError import scala.concurrent.Future diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala similarity index 91% rename from reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala rename to reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala index cba90f5df..04887d3a0 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/ReaderWithPartitioningIdUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala @@ -22,6 +22,7 @@ import sttp.client3._ import sttp.client3.monad.IdMonad import sttp.client3.testing.SttpBackendStub import sttp.model._ +import sttp.monad.MonadError import za.co.absa.atum.model.dto.PartitioningWithIdDTO import za.co.absa.atum.model.envelopes.NotFoundErrorResponse import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse @@ -30,7 +31,7 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.reader.basic.RequestResult._ import za.co.absa.atum.reader.server.ServerConfig -class ReaderWithPartitioningIdUnitTests extends AnyFunSuiteLike { +class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { private val serverUrl = "http://localhost:8080" private val atumPartitionsToReply = AtumPartitions("a", "b") private val atumPartitionsToFailedDecode = AtumPartitions("c", "d") @@ -53,10 +54,11 @@ class ReaderWithPartitioningIdUnitTests extends AnyFunSuiteLike { } - private case class ReaderWithPartitioningIdForTest[F[_]](partitioning: AtumPartitions) + private case class ReaderWithPartitioningIdForTest(partitioning: AtumPartitions) (implicit serverConfig: ServerConfig) - extends ReaderWithPartitioningId { - override def partitioningId(): Identity[RequestResult[Long]] = super.partitioningId() + extends Reader[Identity] with PartitioningIdProvider[Identity]{ + + override def partitioningId()(implicit monad: MonadError[Identity]): Identity[RequestResult[Long]] = super.partitioningId() } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala index 29051a1d0..1aaad0901 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala @@ -37,7 +37,7 @@ class Reader_CatsIOUnitTests extends AnyFunSuiteLike { test("Using Cats IO based backend") { import cats.effect.IO - import za.co.absa.atum.reader.implicits.io.CatsIOMonad + import za.co.absa.atum.reader.implicits.io.catsIOMonadError val partitionDTO = PartitionDTO("someKey", "someValue") implicit val server: SttpBackendStub[IO, Any] = SttpBackendStub[IO, Any](implicitly[MonadAsyncError[IO]]) @@ -47,11 +47,6 @@ class Reader_CatsIOUnitTests extends AnyFunSuiteLike { val query = reader.getQuery[PartitionDTO]("/test", Map.empty) val result = query.unsafeRunSync() assert(result == Right(partitionDTO)) - - -// .map { result => -// fail("This test is expected to fail") -// } } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala index 9ac933ebd..c19c6411d 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala @@ -38,7 +38,7 @@ class Reader_FutureUnitTests extends AnyFunSuiteLike { } test("Using Future based backend") { - import za.co.absa.atum.reader.implicits.future.FutureMonad + import za.co.absa.atum.reader.implicits.future.futureMonadError val partitionDTO = PartitionDTO("someKey", "someValue") implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture From c344249df39d97c26fa7c4a0a2081834d1800f52 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sat, 30 Nov 2024 23:43:11 +0100 Subject: [PATCH 32/52] * just better implementation --- .../scala/za/co/absa/atum/reader/FlowReader.scala | 2 -- .../atum/reader/basic/PartitioningIdProvider.scala | 4 +--- .../basic/PartitioningIdProviderUnitTests.scala | 13 ++++++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index f952dc562..4058fad15 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -35,6 +35,4 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) extends Reader[F] with PartitioningIdProvider[F]{ - override def partitioning: AtumPartitions = mainFlowPartitioning - } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala index b32388a12..8a802ff02 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala @@ -26,9 +26,7 @@ import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.reader.basic.RequestResult.RequestResult trait PartitioningIdProvider[F[_]] {self: Reader[F] => - def partitioning: AtumPartitions - - def partitioningId()(implicit monad: MonadError[F]): F[RequestResult[Long]] = { + def partitioningId(partitioning: AtumPartitions)(implicit monad: MonadError[F]): F[RequestResult[Long]] = { val encodedPartitioning = partitioning.toPartitioningDTO.asBase64EncodedJsonString val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]]("/api/v2/partitionings", Map("partitioning" -> encodedPartitioning)) queryResult.map{result => diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala index 04887d3a0..1d74fdb21 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala @@ -58,20 +58,22 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { (implicit serverConfig: ServerConfig) extends Reader[Identity] with PartitioningIdProvider[Identity]{ - override def partitioningId()(implicit monad: MonadError[Identity]): Identity[RequestResult[Long]] = super.partitioningId() + override def partitioningId(partitioning: AtumPartitions) + (implicit monad: MonadError[Identity]): Identity[RequestResult[Long]] = + super.partitioningId(partitioning) } test("Gets the partitioning id") { val reader = ReaderWithPartitioningIdForTest(atumPartitionsToReply) - val response = reader.partitioningId() + val response = reader.partitioningId(atumPartitionsToReply) val result: Long = response.getOrElse(throw new Exception("Failed to get partitioning id")) assert(result == 1) } test("Not found on the partitioning id") { val reader = ReaderWithPartitioningIdForTest(atumPartitionsToNotFound) - val result = reader.partitioningId() + val result = reader.partitioningId(atumPartitionsToNotFound) result match { case Right(_) => fail("Expected a failure, but OK response received") case Left(_: DeserializationException[CirceError]) => fail("Expected a not found response, but deserialization error received") @@ -82,9 +84,10 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { } } - test("Failure to decode response body") { + test("Failure to decode res " + + "]ponse body") { val reader = ReaderWithPartitioningIdForTest(atumPartitionsToFailedDecode) - val result = reader.partitioningId() + val result = reader.partitioningId(atumPartitionsToFailedDecode) assert(result.isLeft) result.swap.map(e => assert(e.isInstanceOf[DeserializationException[CirceError]])) } From c0b0988c6cd263c96b4a45cacbe09d2ab87d7eea Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 4 Dec 2024 01:46:17 +0100 Subject: [PATCH 33/52] #247: Implement basics of FlowReader * start of work --- .../za/co/absa/atum/reader/result/Page.scala | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala new file mode 100644 index 000000000..3d5f43bc3 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import za.co.absa.atum.reader.basic.Reader + +class Page[T, F[_]]( + parentReader: Reader[F[_]], + items: Vector[T], + hasNext: Boolean, + + ) { + + def apply(index: Int): T = items(index) + + def pageSize: Int = items.size + + def hasPrior: Boolean = { + ??? + } + + def prior: Page[T, F] = { + ??? + } + + def next: Page[T, F] = { + ??? + } +} From 55d60e1f95219fa4209d4c7233aefe71c80406b5 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 9 Dec 2024 15:05:19 +0100 Subject: [PATCH 34/52] * major progress --- .../za/co/absa/atum/model}/ApiPaths.scala | 2 +- .../absa/atum/model/dto/CheckpointV2DTO.scala | 5 +- .../dto/CheckpointWithPartitioningDTO.scala | 3 +- .../model/dto/traits/CheckpointCore.scala | 32 ++++++++++ .../co/absa/atum/model/types/Checkpoint.scala | 46 ++++++++++++++ .../absa/atum/model/types/Measurement.scala | 37 ++++++++++++ .../za/co/absa/atum/reader/FlowReader.scala | 49 +++++++++++++++ .../reader/basic/PartitioningIdProvider.scala | 5 +- .../za/co/absa/atum/reader/basic/Reader.scala | 1 + .../atum/reader/basic/RequestResult.scala | 20 +++++-- .../reader/exceptions/ReaderException.scala | 19 ++++++ .../reader/exceptions/RequestException.scala | 50 ++++++++++++++++ .../reader/implicits/EitherImplicits.scala | 30 ++++++++++ .../PaginatedResponseImplicits.scala | 37 ++++++++++++ .../za/co/absa/atum/reader/result/Page.scala | 60 ++++++++++++++----- .../PartitioningIdProviderUnitTests.scala | 8 +-- .../reader/basic/RequestResultUnitTests.scala | 32 ++++++---- .../api/controller/BaseController.scala | 2 +- .../controller/CheckpointControllerImpl.scala | 2 +- .../PartitioningControllerImpl.scala | 2 +- .../atum/server/api/http/BaseEndpoints.scala | 2 +- .../absa/atum/server/api/http/Endpoints.scala | 2 +- .../co/absa/atum/server/api/http/Routes.scala | 2 +- 23 files changed, 403 insertions(+), 45 deletions(-) rename {server/src/main/scala/za/co/absa/atum/server/api/http => model/src/main/scala/za/co/absa/atum/model}/ApiPaths.scala (96%) create mode 100644 model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala create mode 100644 model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala create mode 100644 model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/ApiPaths.scala b/model/src/main/scala/za/co/absa/atum/model/ApiPaths.scala similarity index 96% rename from server/src/main/scala/za/co/absa/atum/server/api/http/ApiPaths.scala rename to model/src/main/scala/za/co/absa/atum/model/ApiPaths.scala index 3e5903082..b7f99788f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/ApiPaths.scala +++ b/model/src/main/scala/za/co/absa/atum/model/ApiPaths.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.server.api.http +package za.co.absa.atum.model object ApiPaths { diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala index ad80b373e..0b6fb7037 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -18,11 +18,12 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import za.co.absa.atum.model.dto.traits.CheckpointCore import java.time.ZonedDateTime import java.util.UUID -case class CheckpointV2DTO( +case class CheckpointV2DTO ( id: UUID, name: String, author: String, @@ -30,7 +31,7 @@ case class CheckpointV2DTO( processStartTime: ZonedDateTime, processEndTime: Option[ZonedDateTime], measurements: Set[MeasurementDTO] -) +) extends CheckpointCore object CheckpointV2DTO { implicit val decodeCheckpointDTO: Decoder[CheckpointV2DTO] = deriveDecoder diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala index d72263201..e89954a74 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala @@ -18,6 +18,7 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} +import za.co.absa.atum.model.dto.traits.CheckpointCore import java.time.ZonedDateTime import java.util.UUID @@ -31,7 +32,7 @@ case class CheckpointWithPartitioningDTO( processEndTime: Option[ZonedDateTime], measurements: Set[MeasurementDTO], partitioning: PartitioningWithIdDTO -) +) extends CheckpointCore object CheckpointWithPartitioningDTO { diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala b/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala new file mode 100644 index 000000000..7f90fa6bb --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.dto.traits + +import za.co.absa.atum.model.dto.MeasurementDTO + +import java.time.ZonedDateTime +import java.util.UUID + +trait CheckpointCore { + def id: UUID + def name: String + def author: String + def measuredByAtumAgent: Boolean + def processStartTime: ZonedDateTime + def processEndTime: Option[ZonedDateTime] + def measurements: Set[MeasurementDTO] +} diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala new file mode 100644 index 000000000..d0e3a74dd --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.types + +import za.co.absa.atum.model.dto.traits.CheckpointCore + +import java.time.ZonedDateTime +import java.util.UUID + +case class Checkpoint ( + id: UUID, + name: String, + author: String, + measuredByAtumAgent: Boolean = false, + processStartTime: ZonedDateTime, + processEndTime: Option[ZonedDateTime], + measurements: Set[Measurement] + ) + +object Checkpoint { + def apply(from: CheckpointCore): Checkpoint = { + new Checkpoint( + id = from.id, + name = from.name, + author = from.author, + measuredByAtumAgent = from.measuredByAtumAgent, + processStartTime = from.processStartTime, + processEndTime = from.processEndTime, + measurements = from.measurements.map() + ) + } +} diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala new file mode 100644 index 000000000..b07d2293b --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.types + +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.MeasurementDTO + +case class Measurement[T] ( + measureName: String, + measuredColumns: Seq[String], + valueType: ResultValueType, + value: T + ) + +object Measurement { + def apply[T](from: MeasurementDTO): Measurement = { + new Measurement( + measureName = from.measure.measureName, + measuredColumns = from.measure.measuredColumns, + value = from.result.mainValue.value + ) + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 6a99bbe40..a7fe3547a 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -18,10 +18,21 @@ package za.co.absa.atum.reader import sttp.client3.SttpBackend import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} +import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.reader.basic.RequestResult.RequestResult import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} +import za.co.absa.atum.model.ApiPaths._ +import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements +import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements +import za.co.absa.atum.reader.result.Page +import za.co.absa.atum.reader.result.Page.PageRoller import za.co.absa.atum.reader.server.ServerConfig +import za.co.absa.atum.reader.basic.RequestResult.RequestPageResultOps + /** * This class is a reader that reads data tight to a flow. * @param mainFlowPartitioning - the partitioning of the main flow; renamed from ancestor's 'flowPartitioning' @@ -35,4 +46,42 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) extends Reader[F] with PartitioningIdProvider[F]{ + private def flowId(mainPartitioningId: Long): F[RequestResult[Long]] = { + val endpoint = s"/$Api/$V2/${V2Paths.Partitionings}/$mainPartitioningId/${V2Paths.MainFlow}" + val queryResult = getQuery[SingleSuccessResponse[FlowDTO]](endpoint) + queryResult.map{ result => + result.map(_.data.id) + } + } + + private def queryCheckpoints(flowId: Long, + checkpointName: Option[String], + pageSize: Int, + offset: Long): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { + val endpoint = s"/$Api/$V2/${V2Paths.Flows}/$flowId/${V2Paths.Checkpoints}" + val params = Map( + "limit" -> pageSize.toString, + "offset" -> offset.toString + ) ++ checkpointName.map(("checkpoint-name" -> _)) + getQuery(endpoint, params) + } + + private def doGetCheckpoints(checkpointName: Option[String], pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { + val pageRoller: PageRoller[CheckpointWithPartitioningDTO, F] = doGetCheckpoints(checkpointName, _, _) + + for { + mainPartitioningId <- partitioningId(mainFlowPartitioning) + flowId <- mainPartitioningId.project(flowId) + checkpoints <- flowId.project(queryCheckpoints(_, checkpointName, pageSize, offset)) + } yield checkpoints.map(_.toPage(pageRoller)) + + } + + def getCheckpoints(pageSize: Int = 10, offset: Long = 0) = { + doGetCheckpoints(None, pageSize, offset).map(_.pageMap(data =>)) + } + + def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0) = { + doGetCheckpoints(Some(name), pageSize, offset) + } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala index 8a802ff02..502202cb1 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala @@ -18,6 +18,7 @@ package za.co.absa.atum.reader.basic import sttp.monad.MonadError import sttp.monad.syntax._ +import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.dto.PartitioningWithIdDTO import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.AtumPartitions @@ -28,8 +29,8 @@ import za.co.absa.atum.reader.basic.RequestResult.RequestResult trait PartitioningIdProvider[F[_]] {self: Reader[F] => def partitioningId(partitioning: AtumPartitions)(implicit monad: MonadError[F]): F[RequestResult[Long]] = { val encodedPartitioning = partitioning.toPartitioningDTO.asBase64EncodedJsonString - val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]]("/api/v2/partitionings", Map("partitioning" -> encodedPartitioning)) - queryResult.map{result => + val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]](s"/$Api/$V2/${V2Paths.Partitionings}", Map("partitioning" -> encodedPartitioning)) + queryResult.map{ result => result.map(_.data.id) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala index 325f8c6fe..793e303c1 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala @@ -24,6 +24,7 @@ import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.reader.server.ServerConfig import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.exceptions.RequestException.CirceError /** * Reader is a base class for reading data from a remote server. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala index 76e8cbfa9..8fe9ec3b2 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala @@ -17,22 +17,32 @@ package za.co.absa.atum.reader.basic import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} +import sttp.monad.MonadError import za.co.absa.atum.model.envelopes.ErrorResponse +import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} +import za.co.absa.atum.reader.exceptions.{ReaderException, RequestException} +import za.co.absa.atum.reader.result.Page object RequestResult { - type CirceError = io.circe.Error - type RequestResult[R] = Either[ResponseException[ErrorResponse, CirceError], R] + type RequestResult[R] = Either[RequestException, R] + + def RequestOK[T](value: T): RequestResult[T] = Right(value) + def RequestFail[T](error: RequestException): RequestResult[T] = Left(error) implicit class ResponseOps[R](val response: Response[Either[ResponseException[String, CirceError], R]]) extends AnyVal { def toRequestResult: RequestResult[R] = { response.body.left.map { case he: HttpError[String] => ErrorResponse.basedOnStatusCode(he.statusCode.code, he.body) match { - case Right(er) => HttpError(er, he.statusCode) - case Left(ce) => DeserializationException(he.body, ce) + case Right(er) => HttpException(he.getMessage, he.statusCode, er, response.request.uri) + case Left(ce) => ParsingException.fromCirceError(ce, he.body) } - case de: DeserializationException[CirceError] => de + case de: DeserializationException[CirceError] => ParsingException.fromCirceError(de.error, de.body) } } } + + implicit class RequestPageResultOps[A, F[_]: MonadError](requestResult: RequestResult[Page[A, F]]) { + def pageMap[B](f: A => B): RequestResult[Page[B, F]] = requestResult.map(_.map(f)) + } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala new file mode 100644 index 000000000..764f09cfb --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.exceptions + +class ReaderException(message: String) extends Exception(message) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala new file mode 100644 index 000000000..da1e75ba7 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.exceptions + +import sttp.client3.HttpError +import sttp.model.{StatusCode, Uri} +import za.co.absa.atum.model.envelopes.ErrorResponse + +abstract class RequestException(message: String) extends ReaderException(message) + + +object RequestException { + type CirceError = io.circe.Error + + final case class HttpException( + message: String, + statusCode: StatusCode, + errorResponse: ErrorResponse, + request: Uri + ) extends RequestException(message) + + final case class ParsingException( + message: String, + body: String + ) extends RequestException(message) + object ParsingException { + def fromCirceError(error: CirceError, body: String): ParsingException = { + ParsingException(error.getMessage, body) + } + } + + + final case class NoDataException( + message: String + ) extends RequestException(message) +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala new file mode 100644 index 000000000..a1d344e54 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.implicits + +import sttp.monad.MonadError + +object EitherImplicits { + + implicit class EitherMonadEnhancements[A, B](val either: Either[A, B]) extends AnyVal { + def project[C, F[_]: MonadError](f: B => F[Either[A, C]]): F[Either[A, C]] = either match { + case Right(b) => f(b) + case Left(a) => implicitly[MonadError[F]].unit(Left(a)) + } + } + +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala new file mode 100644 index 000000000..c482769f4 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.implicits + +import sttp.monad.MonadError +import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse +import za.co.absa.atum.reader.result.Page +import za.co.absa.atum.reader.result.Page.PageRoller + +object PaginatedResponseImplicits { + implicit class PaginatedResponseMonadEnhancements[T](val paginatedResponse: PaginatedResponse[T]) extends AnyVal { + def toPage[F[_]: MonadError](pageRoller: PageRoller[T, F]): Page[T, F] = { + val data = paginatedResponse.data.toVector + Page( + items = data, + hasNext = paginatedResponse.pagination.hasMore, + limit = paginatedResponse.pagination.limit, + offset = paginatedResponse.pagination.offset, + pageRoller = pageRoller + ) + } + } +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala index 3d5f43bc3..ffb6397ef 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -16,28 +16,60 @@ package za.co.absa.atum.reader.result -import za.co.absa.atum.reader.basic.Reader +import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.reader.result.Page.PageRoller -class Page[T, F[_]]( - parentReader: Reader[F[_]], - items: Vector[T], - hasNext: Boolean, - - ) { +case class Page[T, F[_]: MonadError]( + items: Vector[T], + hasNext: Boolean, + limit: Int, + offset: Long, + private[reader] val pageRoller: PageRoller[T, F] + ) { def apply(index: Int): T = items(index) + def map[B](f: T => B): Page[B, F] = { + val newItems = items.map(f) + val newPageRoller: PageRoller[B, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.map(f))) + this.copy(items = newItems, pageRoller = newPageRoller) + } + +// def flatMap[B](f: T => IterableOnce[B]): Page[B, F] = { +// val newItems = items.flatMap(f) +// ??? + // TODO +// } + def pageSize: Int = items.size - def hasPrior: Boolean = { - ??? - } + def hasPrior: Boolean = offset > 0 - def prior: Page[T, F] = { - ??? + def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = { + if (hasPrior) { + val newOffset = (offset - limit).max(0) + pageRoller(newPageSize, newOffset) + } else { + MonadError[F].unit(RequestFail(NoDataException("No prior page"))) + } } - def next: Page[T, F] = { - ??? + def prior(): F[RequestResult[Page[T, F]]] = prior(limit) + + def next(newPageSize: Int): F[RequestResult[Page[T, F]]] = { + if (hasNext) { + pageRoller(newPageSize, offset + limit) + } else { + MonadError[F].unit(RequestFail(NoDataException("No next page"))) + } } + + def next: F[RequestResult[Page[T, F]]] = next(limit) +} + +object Page { + type PageRoller[T, F[_]] = (Int, Long) => F[RequestResult[Page[T, F]]] } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala index 20cac1a02..90b9c5f79 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala @@ -29,6 +29,7 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.exceptions.RequestException.{HttpException, ParsingException} import za.co.absa.atum.reader.server.ServerConfig class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { @@ -76,9 +77,8 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { val result = reader.partitioningId(atumPartitionsToNotFound) result match { case Right(_) => fail("Expected a failure, but OK response received") - case Left(_: DeserializationException[CirceError]) => fail("Expected a not found response, but deserialization error received") - case Left(x: HttpError[_]) => - assert(x.body.isInstanceOf[NotFoundErrorResponse]) + case Left(x: HttpException) => + assert(x.errorResponse.isInstanceOf[NotFoundErrorResponse]) assert(x.statusCode == StatusCode.NotFound) case _ => fail("Unexpected response") } @@ -88,6 +88,6 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { val reader = ReaderWithPartitioningIdForTest(atumPartitionsToFailedDecode) val result = reader.partitioningId(atumPartitionsToFailedDecode) assert(result.isLeft) - result.swap.map(e => assert(e.isInstanceOf[DeserializationException[CirceError]])) + result.swap.map(e => assert(e.isInstanceOf[ParsingException])) } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala index d181154df..4c2613d51 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala @@ -19,11 +19,12 @@ package za.co.absa.atum.reader.basic import io.circe.ParsingFailure import org.scalatest.funsuite.AnyFunSuiteLike import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} -import sttp.model.StatusCode +import sttp.model.{StatusCode, Uri} import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.envelopes.NotFoundErrorResponse import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} class RequestResultUnitTests extends AnyFunSuiteLike { test("Response.toRequestResult keeps the right value") { @@ -37,7 +38,8 @@ class RequestResultUnitTests extends AnyFunSuiteLike { assert(result == body) } - test("Response.toRequestResult keeps the left value if it's a CirceError") { + test("Response.toRequestResult keeps the left value if it's a CirceError with its message") { + val circeError: CirceError = ParsingFailure("Just a test error", new Exception) val deserializationException = DeserializationException("This is not a json", circeError) val body = Left(deserializationException) @@ -46,20 +48,30 @@ class RequestResultUnitTests extends AnyFunSuiteLike { StatusCode.Ok ) val result = source.toRequestResult - assert(result == body) + result match { + case Left(ParsingException(message, body)) => + assert(message == "Just a test error") + assert(body == "This is not a json") + case _ => fail("Unexpected result") + } } test("Response.toRequestResult decodes NotFound error") { - val error = NotFoundErrorResponse("This is a test") - val errorResponse = error.asJsonString - val httpError = HttpError(errorResponse, StatusCode.NotFound) + val sourceError = NotFoundErrorResponse("This is a test") + val sourceErrorResponse = sourceError.asJsonString + val httpError = HttpError(sourceErrorResponse, StatusCode.NotFound) val source: Response[Either[ResponseException[String, CirceError], PartitionDTO]] = Response( Left(httpError), StatusCode.Ok ) val result = source.toRequestResult - val expected: RequestResult[PartitionDTO] = Left(HttpError(error, httpError.statusCode)) - assert(result == expected) + result match { + case Left(HttpException(_, statusCode, errorResponse, request)) => + assert(statusCode == StatusCode.NotFound) + assert(errorResponse == sourceError) + assert(request == Uri("example.com")) + case _ => fail("Unexpected result") + } } test("Response.toRequestResult fails to decode InternalServerErrorResponse error") { @@ -74,8 +86,8 @@ class RequestResultUnitTests extends AnyFunSuiteLike { assert(result.isLeft) result.swap.foreach { e => // investigate the error - assert(e.isInstanceOf[DeserializationException[_]]) - val ce = e.asInstanceOf[DeserializationException[ParsingFailure]] + assert(e.isInstanceOf[ParsingException]) + val ce = e.asInstanceOf[ParsingException] assert(ce.body == responseBody) } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala index 066c28f3a..1d6da5b89 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/BaseController.scala @@ -16,10 +16,10 @@ package za.co.absa.atum.server.api.controller +import za.co.absa.atum.model.ApiPaths import za.co.absa.atum.model.envelopes.{ConflictErrorResponse, ErrorInDataErrorResponse, ErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse, Pagination} import za.co.absa.atum.server.api.exception.ServiceError import za.co.absa.atum.server.api.exception.ServiceError._ -import za.co.absa.atum.server.api.http.ApiPaths import za.co.absa.atum.server.model.PaginatedResult.{ResultHasMore, ResultNoMore} import za.co.absa.atum.model.envelopes.SuccessResponse._ import za.co.absa.atum.server.model._ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala index 0966805bf..fefab4cfd 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/CheckpointControllerImpl.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto.{CheckpointDTO, CheckpointV2DTO} import za.co.absa.atum.model.envelopes.ErrorResponse -import za.co.absa.atum.server.api.http.ApiPaths.V2Paths +import za.co.absa.atum.model.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.CheckpointService import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.server.model.PaginatedResult diff --git a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala index 20071545e..604206028 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/controller/PartitioningControllerImpl.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.server.api.controller import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.{ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse} import za.co.absa.atum.server.api.exception.ServiceError -import za.co.absa.atum.server.api.http.ApiPaths.V2Paths +import za.co.absa.atum.model.ApiPaths.V2Paths import za.co.absa.atum.server.api.service.PartitioningService import za.co.absa.atum.model.envelopes.SuccessResponse._ import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonDeserializationSyntax diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala index f6c928264..b68c0ec31 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/BaseEndpoints.scala @@ -24,7 +24,7 @@ import sttp.tapir.typelevel.MatchType import sttp.tapir.ztapir._ import sttp.tapir.{EndpointOutput, PublicEndpoint} import za.co.absa.atum.model.envelopes.{BadRequestResponse, ConflictErrorResponse, ErrorInDataErrorResponse, ErrorResponse, GeneralErrorResponse, InternalServerErrorResponse, NotFoundErrorResponse} -import za.co.absa.atum.server.api.http.ApiPaths._ +import za.co.absa.atum.model.ApiPaths._ import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala index 8a138d4b2..007f2ca4e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Endpoints.scala @@ -24,7 +24,7 @@ import za.co.absa.atum.model.dto._ import za.co.absa.atum.model.envelopes.SuccessResponse._ import sttp.tapir.{PublicEndpoint, Validator, endpoint} import za.co.absa.atum.model.envelopes.{ErrorResponse, StatusResponse} -import za.co.absa.atum.server.api.http.ApiPaths._ +import za.co.absa.atum.model.ApiPaths._ import java.util.UUID diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 00ade985b..04027f47d 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -132,7 +132,7 @@ trait Routes extends Endpoints with ServerOptions { getPartitioningEndpointV2, // getPartitioningMeasuresEndpointV2, // getFlowPartitioningsEndpointV2, - // getPartitioningMainFlowEndpointV2, + getPartitioningMainFlowEndpointV2, // getFlowCheckpointsEndpointV2, healthEndpoint ) From 5dfe5c5fb78bf651be5626bc9014c5216f2aea63 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Mon, 9 Dec 2024 17:30:35 +0100 Subject: [PATCH 35/52] * Further progress --- .../types/AtumPartitionsCheckpoint.scala | 24 +++++++ .../co/absa/atum/model/types/Checkpoint.scala | 2 +- .../absa/atum/model/types/Measurement.scala | 67 ++++++++++++++++--- .../za/co/absa/atum/reader/FlowReader.scala | 19 ++++-- 4 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala new file mode 100644 index 000000000..1dfa8f2a7 --- /dev/null +++ b/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala @@ -0,0 +1,24 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.model.types + +import za.co.absa.atum.model.types.basic.AtumPartitions + +case class AtumPartitionsCheckpoint( + partitioning: AtumPartitions, + checkpoint: Checkpoint + ) diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala index d0e3a74dd..bc7d8e43c 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala @@ -40,7 +40,7 @@ object Checkpoint { measuredByAtumAgent = from.measuredByAtumAgent, processStartTime = from.processStartTime, processEndTime = from.processEndTime, - measurements = from.measurements.map() + measurements = from.measurements.map(Measurement(_)) ) } } diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala index b07d2293b..eadd9ad38 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala @@ -19,19 +19,64 @@ package za.co.absa.atum.model.types import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasurementDTO -case class Measurement[T] ( - measureName: String, - measuredColumns: Seq[String], - valueType: ResultValueType, - value: T - ) +trait Measurement { + type T + def measureName: String + def measuredColumns: Seq[String] + def valueType: ResultValueType + def value: T + def stringValue: String +} object Measurement { + def apply[T](from: MeasurementDTO): Measurement = { - new Measurement( - measureName = from.measure.measureName, - measuredColumns = from.measure.measuredColumns, - value = from.result.mainValue.value - ) + from.result.mainValue.valueType match { + case ResultValueType.StringValue => StringMeasurement(from.measure.measureName, from.measure.measuredColumns, from.result.mainValue.value) + case ResultValueType.LongValue => LongMeasurement(from.measure.measureName, from.measure.measuredColumns, from.result.mainValue.value.toLong) + case ResultValueType.BigDecimalValue => BigDecimalMeasurement(from.measure.measureName, from.measure.measuredColumns, BigDecimal(from.result.mainValue.value)) + case ResultValueType.DoubleValue => DoubleMeasurement(from.measure.measureName, from.measure.measuredColumns, from.result.mainValue.value.toDouble) + } + } + + case class StringMeasurement( + measureName: String, + measuredColumns: Seq[String], + value: String + ) extends Measurement { + override type T = String + override def valueType: ResultValueType = ResultValueType.StringValue + override def stringValue: String = value + } + + case class LongMeasurement( + measureName: String, + measuredColumns: Seq[String], + value: Long + ) extends Measurement { + override type T = Long + override def valueType: ResultValueType = ResultValueType.LongValue + override def stringValue: String = value.toString } + + case class BigDecimalMeasurement( + measureName: String, + measuredColumns: Seq[String], + value: BigDecimal + ) extends Measurement { + override type T = BigDecimal + override def valueType: ResultValueType = ResultValueType.BigDecimalValue + override def stringValue: String = value.toString + } + + case class DoubleMeasurement( + measureName: String, + measuredColumns: Seq[String], + value: Double + ) extends Measurement { + override type T = Double + override def valueType: ResultValueType = ResultValueType.DoubleValue + override def stringValue: String = value.toString + } + } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index a7fe3547a..87a110927 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -21,16 +21,16 @@ import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} -import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.model.types.basic.{AtumPartitions, PartitioningDTOOps} import za.co.absa.atum.reader.basic.RequestResult.RequestResult import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} import za.co.absa.atum.model.ApiPaths._ +import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements import za.co.absa.atum.reader.result.Page import za.co.absa.atum.reader.result.Page.PageRoller import za.co.absa.atum.reader.server.ServerConfig - import za.co.absa.atum.reader.basic.RequestResult.RequestPageResultOps /** @@ -66,8 +66,8 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) getQuery(endpoint, params) } - private def doGetCheckpoints(checkpointName: Option[String], pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { - val pageRoller: PageRoller[CheckpointWithPartitioningDTO, F] = doGetCheckpoints(checkpointName, _, _) + private def geetCheckpointDTOs(checkpointName: Option[String], pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { + val pageRoller: PageRoller[CheckpointWithPartitioningDTO, F] = geetCheckpointDTOs(checkpointName, _, _) for { mainPartitioningId <- partitioningId(mainFlowPartitioning) @@ -77,11 +77,16 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) } - def getCheckpoints(pageSize: Int = 10, offset: Long = 0) = { - doGetCheckpoints(None, pageSize, offset).map(_.pageMap(data =>)) + def getCheckpoints(pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[AtumPartitionsCheckpoint, F]]] = { + def checkpointMapper(data: CheckpointWithPartitioningDTO): AtumPartitionsCheckpoint = { + val atumPartitions = data.partitioning.partitioning.toAtumPartitions + val checkpoint = Checkpoint(data) + AtumPartitionsCheckpoint(atumPartitions, checkpoint) + } + geetCheckpointDTOs(None, pageSize, offset).map(_.pageMap(checkpointMapper)) } def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0) = { - doGetCheckpoints(Some(name), pageSize, offset) + geetCheckpointDTOs(Some(name), pageSize, offset) } } From 67ffe0780cbf8f0fd099f46c4c7b5cdb06a623d2 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 11 Dec 2024 13:52:38 +0100 Subject: [PATCH 36/52] * Flow reader methods to read checkpoints * `Page` class to nicely handle paginated results * `GroupedPage` class to handle paginated results that can be grouped --- .../za/co/absa/atum/reader/FlowReader.scala | 12 +-- .../atum/reader/basic/RequestResult.scala | 3 +- .../atum/reader/result/AbstractPage.scala | 31 +++++++ .../absa/atum/reader/result/GroupedPage.scala | 93 +++++++++++++++++++ .../za/co/absa/atum/reader/result/Page.scala | 34 ++++--- 5 files changed, 155 insertions(+), 18 deletions(-) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 87a110927..36040dc5f 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -22,16 +22,16 @@ import sttp.monad.syntax._ import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.{AtumPartitions, PartitioningDTOOps} -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.basic.RequestResult.{RequestPageResultOps, RequestResult} import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements +import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits import za.co.absa.atum.reader.result.Page -import za.co.absa.atum.reader.result.Page.PageRoller import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.basic.RequestResult.RequestPageResultOps +import za.co.absa.atum.reader.result.Page.PageRoller /** * This class is a reader that reads data tight to a flow. @@ -62,7 +62,7 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) val params = Map( "limit" -> pageSize.toString, "offset" -> offset.toString - ) ++ checkpointName.map(("checkpoint-name" -> _)) + ) ++ checkpointName.map("checkpoint-name" -> _) getQuery(endpoint, params) } @@ -83,10 +83,10 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) val checkpoint = Checkpoint(data) AtumPartitionsCheckpoint(atumPartitions, checkpoint) } - geetCheckpointDTOs(None, pageSize, offset).map(_.pageMap(checkpointMapper)) + geetCheckpointDTOs(None, pageSize, offset).map(_.pageMap((checkpointMapper))) } - def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0) = { + def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { geetCheckpointDTOs(Some(name), pageSize, offset) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala b/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala index 8fe9ec3b2..f98799099 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala @@ -21,7 +21,7 @@ import sttp.monad.MonadError import za.co.absa.atum.model.envelopes.ErrorResponse import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} import za.co.absa.atum.reader.exceptions.{ReaderException, RequestException} -import za.co.absa.atum.reader.result.Page +import za.co.absa.atum.reader.result.{GroupedPage, Page} object RequestResult { type RequestResult[R] = Either[RequestException, R] @@ -45,4 +45,5 @@ object RequestResult { implicit class RequestPageResultOps[A, F[_]: MonadError](requestResult: RequestResult[Page[A, F]]) { def pageMap[B](f: A => B): RequestResult[Page[B, F]] = requestResult.map(_.map(f)) } + } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala new file mode 100644 index 000000000..925f90a2c --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import sttp.monad.MonadError +import za.co.absa.atum.reader.basic.RequestResult.RequestResult + +abstract class AbstractPage [T <: Iterable[_], F[_]: MonadError] { + def items: T + def hasNext: Boolean + def limit: Int + def offset: Long + + def pageSize: Int = items.size + def hasPrior: Boolean = offset > 0 +} + diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala new file mode 100644 index 000000000..9a2c161c3 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller +import za.co.absa.atum.reader.result.Page.PageRoller + +import scala.collection.immutable.ListMap + +case class GroupedPage[K, V, F[_]: MonadError]( + items: ListMap[K, Vector[V]], + hasNext: Boolean, + limit: Int, + offset: Long, + private[reader] val pageRoller: GroupPageRoller[K, V, F] + ) extends AbstractPage[Map[K, Vector[V]], F] { + + def apply(key: K): Vector[V] = items(key) + + def map[K1, V1](f: ((K, Vector[V])) => (K1, Vector[V1])): GroupedPage[K1, V1, F] = { + val newItems = items.map(f) + val newPageRoller: GroupPageRoller[K1, V1, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.map(f))) + this.copy(items = newItems, pageRoller = newPageRoller) + } + + def mapValues[B](f: V => B): GroupedPage[K, B, F] = { + def mapper(item: (K, Vector[V])): (K, Vector[B]) = (item._1, item._2.map(f)) + + val newItems = items.map(mapper) + val newPageRoller: GroupPageRoller[K, B, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.mapValues(f))) + this.copy(items = newItems, pageRoller = newPageRoller) + + } + + def flatten: Page[V, F] = { + val newItems = items.values.flatten.toVector + val newPageRoller: PageRoller[V, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.flatten)) + Page( + items = newItems, + hasNext = hasNext, + limit = limit, + offset = offset, + pageRoller = newPageRoller + ) + } + + def flatMap[K1, T](f: ((K, Vector[V])) => (K1, Vector[T])): Page[T, F] = { + map(f).flatten + } + + def prior(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { + if (hasPrior) { + val newOffset = (offset - limit).max(0) + pageRoller(newPageSize, newOffset) + } else { + MonadError[F].unit(RequestFail(NoDataException("No prior page"))) + } + } + + def prior(): F[RequestResult[GroupedPage[K, V, F]]] = prior(limit) + + def next(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { + if (hasNext) { + pageRoller(newPageSize, offset + limit) + } else { + MonadError[F].unit(RequestFail(NoDataException("No next page"))) + } + } + + def next: F[RequestResult[GroupedPage[K, V, F]]] = next(limit) +} + +object GroupedPage { + type GroupPageRoller[K, V, F[_]] = (Int, Long) => F[RequestResult[GroupedPage[K, V, F]]] +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala index ffb6397ef..a85c0cd4c 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -18,35 +18,47 @@ package za.co.absa.atum.reader.result import sttp.monad.MonadError import sttp.monad.syntax._ -import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestResult} +import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestPageResultOps, RequestResult} import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.reader.implicits.VectorImplicits.VectorEnhancements +import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller import za.co.absa.atum.reader.result.Page.PageRoller +import scala.collection.immutable.ListMap + case class Page[T, F[_]: MonadError]( items: Vector[T], hasNext: Boolean, limit: Int, offset: Long, private[reader] val pageRoller: PageRoller[T, F] - ) { + ) extends AbstractPage[Vector[T], F] { def apply(index: Int): T = items(index) def map[B](f: T => B): Page[B, F] = { val newItems = items.map(f) - val newPageRoller: PageRoller[B, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.map(f))) + val newPageRoller: PageRoller[B, F] = (limit, offset) => pageRoller(limit, offset).map(_.pageMap(f)) this.copy(items = newItems, pageRoller = newPageRoller) } -// def flatMap[B](f: T => IterableOnce[B]): Page[B, F] = { -// val newItems = items.flatMap(f) -// ??? - // TODO -// } - - def pageSize: Int = items.size + def groupBy[K](f: T => K): GroupedPage[K, T, F] = { + val newItems = items.foldLeft(ListMap.empty[K, Vector[T]]) { (acc, x) => + val k = f(x) + acc.updated(k, acc.getOrElse(k, Vector.empty) :+ x) + } + val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) => + pageRoller(limit, offset) + .map(_.map(_.groupBy(f))) - def hasPrior: Boolean = offset > 0 + GroupedPage( + newItems, + hasNext, + limit, + offset, + newPageRoller + ) + } def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = { if (hasPrior) { From 09e2ed8543ffabee7d6dbe597c28e34ab4f98149 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 11 Dec 2024 18:48:33 +0100 Subject: [PATCH 37/52] * small fixes * added + function to `Page` and `GroupedPage` classes --- .../absa/atum/reader/result/GroupedPage.scala | 10 ++++++++++ .../za/co/absa/atum/reader/result/Page.scala | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala index 9a2c161c3..5ea100520 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala @@ -30,10 +30,13 @@ case class GroupedPage[K, V, F[_]: MonadError]( hasNext: Boolean, limit: Int, offset: Long, + override val pageSize: Int, private[reader] val pageRoller: GroupPageRoller[K, V, F] ) extends AbstractPage[Map[K, Vector[V]], F] { def apply(key: K): Vector[V] = items(key) + def keys: Iterable[K] = items.keys + def groupCount: Int = items.size def map[K1, V1](f: ((K, Vector[V])) => (K1, Vector[V1])): GroupedPage[K1, V1, F] = { val newItems = items.map(f) @@ -86,6 +89,13 @@ case class GroupedPage[K, V, F[_]: MonadError]( } def next: F[RequestResult[GroupedPage[K, V, F]]] = next(limit) + + def +(other: GroupedPage[K, V, F]): GroupedPage[K, V, F] = { + val newItems = items ++ other.items + val newOffset = offset min other.offset + val newPageSize = pageSize + other.pageSize + this.copy(items = newItems, offset = newOffset, pageSize = newPageSize) + } } object GroupedPage { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala index a85c0cd4c..7ddc2c128 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -20,7 +20,6 @@ import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestPageResultOps, RequestResult} import za.co.absa.atum.reader.exceptions.RequestException.NoDataException -import za.co.absa.atum.reader.implicits.VectorImplicits.VectorEnhancements import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller import za.co.absa.atum.reader.result.Page.PageRoller @@ -43,9 +42,9 @@ case class Page[T, F[_]: MonadError]( } def groupBy[K](f: T => K): GroupedPage[K, T, F] = { - val newItems = items.foldLeft(ListMap.empty[K, Vector[T]]) { (acc, x) => - val k = f(x) - acc.updated(k, acc.getOrElse(k, Vector.empty) :+ x) + val (newItems, itemsCounts) = items.foldLeft(ListMap.empty[K, Vector[T]], 0) { case ((groupsAcc, count), item) => + val k = f(item) + (groupsAcc.updated(k, groupsAcc.getOrElse(k, Vector.empty) :+ item), count + 1) } val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) => pageRoller(limit, offset) @@ -56,13 +55,14 @@ case class Page[T, F[_]: MonadError]( hasNext, limit, offset, + itemsCounts, newPageRoller ) } def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = { if (hasPrior) { - val newOffset = (offset - limit).max(0) + val newOffset = (offset - newPageSize).max(0) pageRoller(newPageSize, newOffset) } else { MonadError[F].unit(RequestFail(NoDataException("No prior page"))) @@ -73,13 +73,19 @@ case class Page[T, F[_]: MonadError]( def next(newPageSize: Int): F[RequestResult[Page[T, F]]] = { if (hasNext) { - pageRoller(newPageSize, offset + limit) + pageRoller(newPageSize, offset + pageSize) } else { MonadError[F].unit(RequestFail(NoDataException("No next page"))) } } def next: F[RequestResult[Page[T, F]]] = next(limit) + + def +(other: Page[T, F]): Page[T, F] = { + val newItems = items ++ other.items + val newOffset = offset min other.offset + this.copy(items = newItems, offset = newOffset) + } } object Page { From e7ff7323f36e06b602c80c1d7bdfbf9a62ed7643 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 11 Dec 2024 19:01:17 +0100 Subject: [PATCH 38/52] * License year --- .../scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala | 2 +- .../za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala | 2 +- .../src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala | 2 +- .../main/scala/za/co/absa/atum/model/types/Measurement.scala | 2 +- .../za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala | 2 +- .../atum/model/utils/JsonDeserializationSyntaxUnitTests.scala | 2 +- .../atum/model/utils/JsonSerializationSyntaxUnitTests.scala | 2 +- .../za/co/absa/atum/reader/exceptions/ReaderException.scala | 2 +- .../za/co/absa/atum/reader/exceptions/RequestException.scala | 2 +- .../za/co/absa/atum/reader/implicits/EitherImplicits.scala | 2 +- .../absa/atum/reader/implicits/PaginatedResponseImplicits.scala | 2 +- .../main/scala/za/co/absa/atum/reader/result/AbstractPage.scala | 2 +- .../main/scala/za/co/absa/atum/reader/result/GroupedPage.scala | 2 +- reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala b/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala index 7f90fa6bb..12494b1b8 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala index 1dfa8f2a7..3990511ab 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala index bc7d8e43c..14af96e41 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala index eadd9ad38..c8245e2b5 100644 --- a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala +++ b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala index 11a529d02..cbd5e5fc1 100644 --- a/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala index 4ca1ac9df..f07b8727e 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala index 830f4e9b7..25a815891 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala index 764f09cfb..61ae91e06 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala index da1e75ba7..8ded23f63 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala index a1d344e54..9410eb22d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/EitherImplicits.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala index c482769f4..ae4635f1b 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala index 925f90a2c..0a7ed537f 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala index 5ea100520..159605a42 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala index 7ddc2c128..2b39e127d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From e63a2e42c00a8b98ca02e4592a22530da3c55bd7 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Sun, 2 Feb 2025 10:21:10 +0100 Subject: [PATCH 39/52] * Finished implementation * Finished UTs --- .../za/co/absa/atum/reader/FlowReader.scala | 30 +- .../absa/atum/reader/PartitioningReader.scala | 2 +- .../PartitioningIdProvider.scala | 4 +- .../atum/reader/{basic => core}/Reader.scala | 8 +- .../{basic => core}/RequestResult.scala | 6 +- .../PaginatedResponseImplicits.scala | 3 +- .../atum/reader/result/AbstractPage.scala | 9 +- .../absa/atum/reader/result/GroupedPage.scala | 43 +-- .../za/co/absa/atum/reader/result/Page.scala | 53 +-- .../atum/reader/FlowReaderUnitTests.scala | 318 +++++++++++++++++- .../PartitioningIdProviderUnitTests.scala | 5 +- .../reader/basic/Reader_CatsIOUnitTests.scala | 3 +- .../reader/basic/Reader_FutureUnitTests.scala | 3 +- .../reader/basic/RequestResultUnitTests.scala | 2 +- .../implicits/EitherImplicitsUnitsTests.scala | 41 +++ .../PaginatedResponseImplicitsTest.scala | 47 +++ .../reader/result/AbstractPageUnitTests.scala | 53 +++ .../reader/result/GroupedPageUnitTests.scala | 179 ++++++++++ .../atum/reader/result/PageUnitTests.scala | 89 +++++ .../co/absa/atum/testing/data/PageData.scala | 72 ++++ .../implicits/AbstractPageImplicits.scala | 51 +++ .../implicits/RequestResultImplicits.scala | 37 ++ 22 files changed, 970 insertions(+), 88 deletions(-) rename reader/src/main/scala/za/co/absa/atum/reader/{basic => core}/PartitioningIdProvider.scala (93%) rename reader/src/main/scala/za/co/absa/atum/reader/{basic => core}/Reader.scala (90%) rename reader/src/main/scala/za/co/absa/atum/reader/{basic => core}/RequestResult.scala (91%) create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 36040dc5f..4ac52802d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -22,13 +22,12 @@ import sttp.monad.syntax._ import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} import za.co.absa.atum.model.types.basic.{AtumPartitions, PartitioningDTOOps} -import za.co.absa.atum.reader.basic.RequestResult.{RequestPageResultOps, RequestResult} -import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} +import za.co.absa.atum.reader.core.RequestResult.{RequestPageResultOps, RequestResult} import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} +import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements -import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits import za.co.absa.atum.reader.result.Page import za.co.absa.atum.reader.server.ServerConfig import za.co.absa.atum.reader.result.Page.PageRoller @@ -46,7 +45,7 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) extends Reader[F] with PartitioningIdProvider[F]{ - private def flowId(mainPartitioningId: Long): F[RequestResult[Long]] = { + private def queryFlowId(mainPartitioningId: Long): F[RequestResult[Long]] = { val endpoint = s"/$Api/$V2/${V2Paths.Partitionings}/$mainPartitioningId/${V2Paths.MainFlow}" val queryResult = getQuery[SingleSuccessResponse[FlowDTO]](endpoint) queryResult.map{ result => @@ -66,27 +65,28 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) getQuery(endpoint, params) } - private def geetCheckpointDTOs(checkpointName: Option[String], pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { - val pageRoller: PageRoller[CheckpointWithPartitioningDTO, F] = geetCheckpointDTOs(checkpointName, _, _) + private def checkpointMapper(data: CheckpointWithPartitioningDTO): AtumPartitionsCheckpoint = { + val atumPartitions = data.partitioning.partitioning.toAtumPartitions + val checkpoint = Checkpoint(data) + AtumPartitionsCheckpoint(atumPartitions, checkpoint) + } + + def getCheckpointDTOs(checkpointName: Option[String], pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { + val pageRoller: PageRoller[CheckpointWithPartitioningDTO, F] = getCheckpointDTOs(checkpointName, _, _) for { mainPartitioningId <- partitioningId(mainFlowPartitioning) - flowId <- mainPartitioningId.project(flowId) + flowId <- mainPartitioningId.project(queryFlowId) checkpoints <- flowId.project(queryCheckpoints(_, checkpointName, pageSize, offset)) } yield checkpoints.map(_.toPage(pageRoller)) } def getCheckpoints(pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[AtumPartitionsCheckpoint, F]]] = { - def checkpointMapper(data: CheckpointWithPartitioningDTO): AtumPartitionsCheckpoint = { - val atumPartitions = data.partitioning.partitioning.toAtumPartitions - val checkpoint = Checkpoint(data) - AtumPartitionsCheckpoint(atumPartitions, checkpoint) - } - geetCheckpointDTOs(None, pageSize, offset).map(_.pageMap((checkpointMapper))) + getCheckpointDTOs(None, pageSize, offset).map(_.pageMap(checkpointMapper)) } - def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { - geetCheckpointDTOs(Some(name), pageSize, offset) + def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0): F[RequestResult[Page[AtumPartitionsCheckpoint, F]]] = { + getCheckpointDTOs(Some(name), pageSize, offset).map(_.pageMap(checkpointMapper)) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index f103605b6..334643f44 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import sttp.client3.SttpBackend import sttp.monad.MonadError import za.co.absa.atum.model.types.basic.AtumPartitions -import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} +import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.server.ServerConfig /** diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala similarity index 93% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala rename to reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala index 502202cb1..f6f6deb35 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import sttp.monad.MonadError import sttp.monad.syntax._ @@ -24,7 +24,7 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.model.types.basic.AtumPartitionsOps import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import RequestResult.RequestResult trait PartitioningIdProvider[F[_]] {self: Reader[F] => def partitioningId(partitioning: AtumPartitions)(implicit monad: MonadError[F]): F[RequestResult[Long]] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala similarity index 90% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala rename to reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala index 793e303c1..632598551 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala @@ -14,16 +14,16 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import io.circe.Decoder -import sttp.client3.{Identity, RequestT, ResponseException, SttpBackend, basicRequest} +import sttp.client3.{DeserializationException, Identity, RequestT, ResponseException, SttpBackend, basicRequest} import sttp.client3.circe.asJson import sttp.model.Uri import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.basic.RequestResult._ +import RequestResult._ import za.co.absa.atum.reader.exceptions.RequestException.CirceError /** @@ -39,12 +39,12 @@ abstract class Reader[F[_]: MonadError](implicit val serverConfig: ServerConfig, protected def getQuery[R: Decoder](endpointUri: String, params: Map[String, String] = Map.empty): F[RequestResult[R]] = { val endpointToQuery = serverConfig.host + endpointUri val uri = Uri.unsafeParse(endpointToQuery).addParams(params) + println(s"Uri: $uri") //TODO --- val request: RequestT[Identity, Either[ResponseException[String, CirceError], R], Any] = basicRequest .get(uri) .response(asJson[R]) val response = backend.send(request) - response.map(_.toRequestResult) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala similarity index 91% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala rename to reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala index f98799099..6861a3993 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala @@ -14,14 +14,14 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} import sttp.monad.MonadError import za.co.absa.atum.model.envelopes.ErrorResponse import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} -import za.co.absa.atum.reader.exceptions.{ReaderException, RequestException} -import za.co.absa.atum.reader.result.{GroupedPage, Page} +import za.co.absa.atum.reader.exceptions.RequestException +import za.co.absa.atum.reader.result.Page object RequestResult { type RequestResult[R] = Either[RequestException, R] diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala index ae4635f1b..298f09248 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala @@ -29,7 +29,8 @@ object PaginatedResponseImplicits { items = data, hasNext = paginatedResponse.pagination.hasMore, limit = paginatedResponse.pagination.limit, - offset = paginatedResponse.pagination.offset, + pageStart = paginatedResponse.pagination.offset, + pageEnd = paginatedResponse.pagination.offset + data.size - 1, pageRoller = pageRoller ) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala index 0a7ed537f..fc2694609 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala @@ -17,15 +17,16 @@ package za.co.absa.atum.reader.result import sttp.monad.MonadError -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.core.RequestResult.RequestResult abstract class AbstractPage [T <: Iterable[_], F[_]: MonadError] { def items: T def hasNext: Boolean def limit: Int - def offset: Long + def pageStart: Long + def pageEnd: Long - def pageSize: Int = items.size - def hasPrior: Boolean = offset > 0 + def pageSize: Int = (pageEnd - pageStart).toInt + 1 + def hasPrior: Boolean = pageStart > 0 } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala index 159605a42..79b9e324e 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.reader.result import sttp.monad.MonadError import sttp.monad.syntax._ -import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestResult} +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestResult} import za.co.absa.atum.reader.exceptions.RequestException.NoDataException import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller import za.co.absa.atum.reader.result.Page.PageRoller @@ -29,8 +29,8 @@ case class GroupedPage[K, V, F[_]: MonadError]( items: ListMap[K, Vector[V]], hasNext: Boolean, limit: Int, - offset: Long, - override val pageSize: Int, + pageStart: Long, + pageEnd: Long, private[reader] val pageRoller: GroupPageRoller[K, V, F] ) extends AbstractPage[Map[K, Vector[V]], F] { @@ -53,36 +53,20 @@ case class GroupedPage[K, V, F[_]: MonadError]( } - def flatten: Page[V, F] = { - val newItems = items.values.flatten.toVector - val newPageRoller: PageRoller[V, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.flatten)) - Page( - items = newItems, - hasNext = hasNext, - limit = limit, - offset = offset, - pageRoller = newPageRoller - ) - } - - def flatMap[K1, T](f: ((K, Vector[V])) => (K1, Vector[T])): Page[T, F] = { - map(f).flatten - } - def prior(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { if (hasPrior) { - val newOffset = (offset - limit).max(0) + val newOffset = (pageStart - limit).max(0) pageRoller(newPageSize, newOffset) } else { MonadError[F].unit(RequestFail(NoDataException("No prior page"))) } } - def prior(): F[RequestResult[GroupedPage[K, V, F]]] = prior(limit) + def prior: F[RequestResult[GroupedPage[K, V, F]]] = prior(limit) def next(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { if (hasNext) { - pageRoller(newPageSize, offset + limit) + pageRoller(newPageSize, pageStart + limit) } else { MonadError[F].unit(RequestFail(NoDataException("No next page"))) } @@ -91,10 +75,17 @@ case class GroupedPage[K, V, F[_]: MonadError]( def next: F[RequestResult[GroupedPage[K, V, F]]] = next(limit) def +(other: GroupedPage[K, V, F]): GroupedPage[K, V, F] = { - val newItems = items ++ other.items - val newOffset = offset min other.offset - val newPageSize = pageSize + other.pageSize - this.copy(items = newItems, offset = newOffset, pageSize = newPageSize) + val newItems = other.items.foldLeft(items) { case (acc, (k, v)) => + if (acc.contains(k)) { + acc.updated(k, acc(k) ++ v) + } else { + acc + (k -> v) + } + } + val newHasNext = hasNext && other.hasNext + val newPageStart = pageStart min other.pageStart + val newPageEnd = pageEnd max other.pageEnd + this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala index 2b39e127d..6d2705f59 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -18,7 +18,7 @@ package za.co.absa.atum.reader.result import sttp.monad.MonadError import sttp.monad.syntax._ -import za.co.absa.atum.reader.basic.RequestResult.{RequestFail, RequestPageResultOps, RequestResult} +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestPageResultOps, RequestResult} import za.co.absa.atum.reader.exceptions.RequestException.NoDataException import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller import za.co.absa.atum.reader.result.Page.PageRoller @@ -29,7 +29,8 @@ case class Page[T, F[_]: MonadError]( items: Vector[T], hasNext: Boolean, limit: Int, - offset: Long, + pageStart: Long, + pageEnd: Long, private[reader] val pageRoller: PageRoller[T, F] ) extends AbstractPage[Vector[T], F] { @@ -41,28 +42,9 @@ case class Page[T, F[_]: MonadError]( this.copy(items = newItems, pageRoller = newPageRoller) } - def groupBy[K](f: T => K): GroupedPage[K, T, F] = { - val (newItems, itemsCounts) = items.foldLeft(ListMap.empty[K, Vector[T]], 0) { case ((groupsAcc, count), item) => - val k = f(item) - (groupsAcc.updated(k, groupsAcc.getOrElse(k, Vector.empty) :+ item), count + 1) - } - val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) => - pageRoller(limit, offset) - .map(_.map(_.groupBy(f))) - - GroupedPage( - newItems, - hasNext, - limit, - offset, - itemsCounts, - newPageRoller - ) - } - def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = { if (hasPrior) { - val newOffset = (offset - newPageSize).max(0) + val newOffset = (pageStart - newPageSize).max(0) pageRoller(newPageSize, newOffset) } else { MonadError[F].unit(RequestFail(NoDataException("No prior page"))) @@ -73,7 +55,7 @@ case class Page[T, F[_]: MonadError]( def next(newPageSize: Int): F[RequestResult[Page[T, F]]] = { if (hasNext) { - pageRoller(newPageSize, offset + pageSize) + pageRoller(newPageSize, pageStart + pageSize) } else { MonadError[F].unit(RequestFail(NoDataException("No next page"))) } @@ -83,8 +65,29 @@ case class Page[T, F[_]: MonadError]( def +(other: Page[T, F]): Page[T, F] = { val newItems = items ++ other.items - val newOffset = offset min other.offset - this.copy(items = newItems, offset = newOffset) + val newPageStart = pageStart min other.pageStart + val newPageEnd = pageEnd max other.pageEnd + val newHasNext = hasNext && other.hasNext + this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd) + } + + def groupBy[K](f: T => K): GroupedPage[K, T, F] = { + val (newItems, itemsCounts) = items.foldLeft(ListMap.empty[K, Vector[T]], 0) { case ((groupsAcc, count), item) => + val k = f(item) + (groupsAcc.updated(k, groupsAcc.getOrElse(k, Vector.empty) :+ item), count + 1) + } + val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) => + pageRoller(limit, offset) + .map(_.map(_.groupBy(f))) + + GroupedPage( + newItems, + hasNext, + limit, + pageStart, + pageEnd, + newPageRoller + ) } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index 3a85f6124..81769adce 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -17,16 +17,30 @@ package za.co.absa.atum.reader import org.scalatest.funsuite.AnyFunSuiteLike -import sttp.client3.SttpBackend +import sttp.capabilities +import sttp.client3.monad.IdMonad +import sttp.client3.{Identity, Response, SttpBackend} import sttp.client3.testing.SttpBackendStub -import za.co.absa.atum.model.types.basic.AtumPartitions +import sttp.model.Uri.QuerySegment.KeyValue +import sttp.monad.MonadError +import za.co.absa.atum.model.ResultValueType +import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue +import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningWithIdDTO} +import za.co.absa.atum.model.types.Measurement.LongMeasurement +import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} +import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} +import za.co.absa.atum.reader.FlowReaderUnitTests._ import za.co.absa.atum.reader.server.ServerConfig import za.co.absa.atum.reader.implicits.future.futureMonadError +import za.co.absa.atum.testing.implicits.RequestResultImplicits.RequestResultPageEnhancements +import java.time.ZonedDateTime +import java.util.UUID import scala.concurrent.Future class FlowReaderUnitTests extends AnyFunSuiteLike { private implicit val serverConfig: ServerConfig = ServerConfig.fromConfig() + private implicit val monad: MonadError[Identity] = IdMonad test("mainFlowPartitioning is the same as partitioning") { val atumPartitions: AtumPartitions = AtumPartitions(List( @@ -38,4 +52,304 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { val result = new FlowReader(atumPartitions).mainFlowPartitioning assert(result == atumPartitions) } + + test("The flow checkpoints are properly queried and delivered as DTO") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + assert(r.uri.querySegments.contains(KeyValue("partitioning", partitioningEncoded))) + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("checkpoints")) => + assert(r.uri.querySegments.contains(KeyValue("offset", "0"))) + assert(r.uri.querySegments.contains(KeyValue("limit", "10"))) + Response.ok(checkpointsResponse) + } + + val atumPartitions: AtumPartitions = AtumPartitions(List( + "a" -> "b", + "c" -> "d" + )) + val expectedData = Vector( + CheckpointWithPartitioningDTO( + id = UUID.fromString("51ee4257-0842-4d28-8779-8ecb19ae7bf0"), + name = "Test checkpoints 1", + author = "Jason Bourne", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.parse("2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]"), + processEndTime = Some(ZonedDateTime.parse("2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]")), + measurements = Set( + MeasurementDTO( + measure = MeasureDTO( + measureName = "Fictional", + measuredColumns = Seq("x", "y", "z") + ), + result = MeasureResultDTO( + mainValue = TypedValue("1", ResultValueType.LongValue), + ) + ) + ), + partitioning = PartitioningWithIdDTO( + id = 7, + atumPartitions.toPartitioningDTO, + author = "James Bond" + ) + ), + CheckpointWithPartitioningDTO( + id = UUID.fromString("8b7f603e-3fc3-474f-aced-a7af054589a2"), + name = "Test checkpoints 2", + author = "John McClane", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.parse("2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]"), + processEndTime = None, + measurements = Set(), + partitioning = PartitioningWithIdDTO( + id = 7, + atumPartitions.toPartitioningDTO, + author = "James Bond" + ) + ) + ) + + val reader = new FlowReader(atumPartitions) + val result = reader.getCheckpointDTOs(None) + result.assertPage(expectedData, hasNext = false, 10, 0, 1) + } + + test("The flow checkpoints are properly queried and delivered as AtumPartitionsCheckpoint instances") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + assert(r.uri.querySegments.contains(KeyValue("partitioning", partitioningEncoded))) + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("checkpoints")) => + assert(r.uri.querySegments.contains(KeyValue("offset", "3"))) + assert(r.uri.querySegments.contains(KeyValue("limit", "11"))) + Response.ok(checkpointsResponse) + } + + val atumPartitions: AtumPartitions = AtumPartitions(List( + "a" -> "b", + "c" -> "d" + )) + val expectedData = Vector( + AtumPartitionsCheckpoint( + partitioning = atumPartitions, + checkpoint = Checkpoint( + id = UUID.fromString("51ee4257-0842-4d28-8779-8ecb19ae7bf0"), + name = "Test checkpoints 1", + author = "Jason Bourne", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.parse("2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]"), + processEndTime = Some(ZonedDateTime.parse("2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]")), + measurements = Set( + LongMeasurement( + + measureName = "Fictional", + measuredColumns = Seq("x", "y", "z"), + value = 1 + ) + ) + ) + ), + AtumPartitionsCheckpoint( + partitioning = atumPartitions, + checkpoint = Checkpoint( + id = UUID.fromString("8b7f603e-3fc3-474f-aced-a7af054589a2"), + name = "Test checkpoints 2", + author = "John McClane", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.parse("2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]"), + processEndTime = None, + measurements = Set() + ) + ) + ) + + val reader = new FlowReader(atumPartitions) + val result = reader.getCheckpoints(11, 3) + result.assertPage(expectedData, hasNext = false, 10, 0, 1) + } + + test("The flow checkpoints of certain name are properly queried and delivered as AtumPartitionsCheckpoint instances") { + implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous + .whenRequestMatchesPartial { + case r if r.uri.path.endsWith(List("partitionings")) => + assert(r.uri.querySegments.contains(KeyValue("partitioning", partitioningEncoded))) + Response.ok(partitioningResponse) + case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => + Response.ok(flowResponse) + case r if r.uri.path.endsWith(List("checkpoints")) => + assert(r.uri.querySegments.contains(KeyValue("offset", "3"))) + assert(r.uri.querySegments.contains(KeyValue("limit", "11"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Foo"))) + Response.ok(checkpointsResponse) + } + + val atumPartitions: AtumPartitions = AtumPartitions(List( + "a" -> "b", + "c" -> "d" + )) + val expectedData = Vector( + AtumPartitionsCheckpoint( + partitioning = atumPartitions, + checkpoint = Checkpoint( + id = UUID.fromString("51ee4257-0842-4d28-8779-8ecb19ae7bf0"), + name = "Test checkpoints 1", + author = "Jason Bourne", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.parse("2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]"), + processEndTime = Some(ZonedDateTime.parse("2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]")), + measurements = Set( + LongMeasurement( + + measureName = "Fictional", + measuredColumns = Seq("x", "y", "z"), + value = 1 + ) + ) + ) + ), + AtumPartitionsCheckpoint( + partitioning = atumPartitions, + checkpoint = Checkpoint( + id = UUID.fromString("8b7f603e-3fc3-474f-aced-a7af054589a2"), + name = "Test checkpoints 2", + author = "John McClane", + measuredByAtumAgent = true, + processStartTime = ZonedDateTime.parse("2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]"), + processEndTime = None, + measurements = Set() + ) + ) + ) + + val reader = new FlowReader(atumPartitions) + val result = reader.getCheckpointsOfName("Foo", 11, 3) + result.assertPage(expectedData, hasNext = false, 10, 0, 1) + } + +} + +object FlowReaderUnitTests { + + private val partitioningEncoded = "W3sia2V5IjoiYSIsInZhbHVlIjoiYiJ9LHsia2V5IjoiYyIsInZhbHVlIjoiZCJ9XQ==" + + private val partitioningResponse = + """ + |{ + | "data" : { + | "id" : 7, + | "partitioning" : [ + | { + | "key" : "a", + | "value" : "b" + | }, + | { + | "key" : "c", + | "value" : "d" + | } + | ], + | "author" : "James Bond" + | }, + | "requestId" : "a8463570-b61f-4c35-9362-4d550848767e" + |} + |""".stripMargin + + + private val flowResponse = + """ + |{ + | "data" : { + | "id" : 42, + | "name" : "Test flow", + | "description" : "This is a test flow", + | "fromPattern" : false + | }, + | "requestId" : "c1343c53-463e-4ac0-80f8-c597c2f1f895" + |} + |""".stripMargin + + private val checkpointsResponse = + """ + |{ + | "data" : [ + | { + | "id" : "51ee4257-0842-4d28-8779-8ecb19ae7bf0", + | "name" : "Test checkpoints 1", + | "author" : "Jason Bourne", + | "measuredByAtumAgent" : true, + | "processStartTime" : "2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]", + | "processEndTime" : "2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]", + | "measurements" : [ + | { + | "measure" : { + | "measureName" : "Fictional", + | "measuredColumns" : [ + | "x", + | "y", + | "z" + | ] + | }, + | "result" : { + | "mainValue" : { + | "value" : "1", + | "valueType" : "Long" + | }, + | "supportValues" : { + | + | } + | } + | } + | ], + | "partitioning" : { + | "id" : 7, + | "partitioning" : [ + | { + | "key" : "a", + | "value" : "b" + | }, + | { + | "key" : "c", + | "value" : "d" + | } + | ], + | "author" : "James Bond" + | } + | }, + | { + | "id" : "8b7f603e-3fc3-474f-aced-a7af054589a2", + | "name" : "Test checkpoints 2", + | "author" : "John McClane", + | "measuredByAtumAgent" : true, + | "processStartTime" : "2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]", + | "measurements" : [ + | ], + | "partitioning" : { + | "id" : 7, + | "partitioning" : [ + | { + | "key" : "a", + | "value" : "b" + | }, + | { + | "key" : "c", + | "value" : "d" + | } + | ], + | "author" : "James Bond" + | } + | } + | ], + | "pagination" : { + | "limit" : 10, + | "offset" : 0, + | "hasMore" : false + | }, + | "requestId" : "29ce91a7-b668-41d2-a160-26402551fb0b" + |} + |""".stripMargin } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala index 90b9c5f79..c60fa408c 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala @@ -28,7 +28,8 @@ import za.co.absa.atum.model.envelopes.NotFoundErrorResponse import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} +import za.co.absa.atum.reader.core.RequestResult._ import za.co.absa.atum.reader.exceptions.RequestException.{HttpException, ParsingException} import za.co.absa.atum.reader.server.ServerConfig @@ -39,7 +40,7 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { private val atumPartitionsToNotFound = AtumPartitions(List.empty) private implicit val serverConfig: ServerConfig = ServerConfig(serverUrl) - private implicit val monad: IdMonad.type = IdMonad + private implicit val monad: MonadError[Identity] = IdMonad private implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatches(request => isUriOfAtumPartitions(request.uri, atumPartitionsToReply)) .thenRespond(SingleSuccessResponse(PartitioningWithIdDTO(1, atumPartitionsToReply.toPartitioningDTO, "Gimli")).asJsonString) diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala index 1aaad0901..6109da871 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala @@ -24,7 +24,8 @@ import sttp.client3.testing.SttpBackendStub import sttp.monad.{MonadAsyncError, MonadError} import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.core.Reader +import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.reader.server.ServerConfig class Reader_CatsIOUnitTests extends AnyFunSuiteLike { diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala index c19c6411d..bc17fdfac 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala @@ -23,7 +23,8 @@ import sttp.client3.testing.SttpBackendStub import sttp.monad.MonadError import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.core.Reader +import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.reader.server.ServerConfig import scala.concurrent.duration.Duration diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala index 4c2613d51..9e682869f 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala @@ -23,7 +23,7 @@ import sttp.model.{StatusCode, Uri} import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.envelopes.NotFoundErrorResponse import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.core.RequestResult._ import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} class RequestResultUnitTests extends AnyFunSuiteLike { diff --git a/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala new file mode 100644 index 000000000..65468ba90 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.implicits + +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.Identity +import sttp.client3.monad.IdMonad +import sttp.monad.MonadError +import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements + +class EitherImplicitsUnitsTests extends AnyFunSuiteLike { + private implicit val monad: MonadError[Identity] = IdMonad + + test("EitherMonadEnhancements should project Right") { + def fnc(b: Int): Identity[Either[String, String]] = Right(b.toString) + val either = Right(1) + val result = either.project(fnc) + assert(result == Right("1")) + } + + test("EitherMonadEnhancements should not project Left") { + def fnc(b: Int): Identity[Either[Exception, String]] = Right(b.toString) + val either = Left(new Exception("error")) + val result = either.project(fnc) + assert(result == either) + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala b/reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala new file mode 100644 index 000000000..96c6b8b3b --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2024 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.implicits + +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.Identity +import sttp.client3.monad.IdMonad +import sttp.monad.MonadError +import za.co.absa.atum.model.envelopes.Pagination +import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException +import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements +import za.co.absa.atum.reader.result.Page + +class PaginatedResponseImplicitsTest extends AnyFunSuiteLike { + private implicit val monad: MonadError[Identity] = IdMonad + + test("toPage should convert PaginatedResponse to Page") { + def roll(pageSize: Int, offset: Long): Identity[RequestResult[Page[Int, Identity]]] = { + IdMonad.unit(RequestFail(new RequestException("Not used"){})) + } + + val source = PaginatedResponse(Seq(1, 2, 3), + pagination = Pagination(offset = 1, limit = 3, hasMore = false) + ) + val result = source.toPage(roll) + assert(result.items == Vector(1, 2, 3)) + assert(!result.hasNext) + assert(result.limit == 3) + assert(result.pageStart == 1) + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala new file mode 100644 index 000000000..94623783f --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.Identity +import sttp.client3.monad.IdMonad +import sttp.monad.MonadError + + +class AbstractPageUnitTests extends AnyFunSuiteLike { + private implicit val monad: MonadError[Identity] = IdMonad + + test("Basic test") { + val page = new AbstractPage[Iterable[Int], Identity] { + override def items: Iterable[Int] = Seq(1, 2, 3) + override def hasNext: Boolean = true + override def limit: Int = 3 + override def pageStart: Long = 0 + override def pageEnd: Long = 2 + } + + assert(page.items.size == 3) + assert(page.hasNext) + assert(page.limit == 3) + assert(page.pageStart == 0) + assert(page.pageSize == 3) + assert(!page.hasPrior) + + val anotherPage = new AbstractPage[Iterable[Int], Identity] { + override def items: Iterable[Int] = Seq(1, 2, 3) + override def hasNext: Boolean = true + override def limit: Int = 3 + override def pageStart: Long = 1 + override def pageEnd: Long = 2 + } + assert(anotherPage.hasPrior) + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala new file mode 100644 index 000000000..5c79f9e79 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala @@ -0,0 +1,179 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.reader.core.RequestResult.RequestFail +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.testing.data.PageData +import za.co.absa.atum.testing.implicits.AbstractPageImplicits.GroupedPageEnhancements + +import scala.collection.immutable.ListMap + + +class GroupedPageUnitTests extends AnyFunSuiteLike { + private val initialPage = { + val Right(ungrouped) = PageData.roller(6, 0) + ungrouped.groupBy(_.group) + } + + test("Getting items, group keys, and group count") { + assert(initialPage(3) == Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f"))) + assert(initialPage.groupCount == 3) + assert(initialPage.keys.toList == List(1, 2, 3)) + } + + test("Mapping of the page") { + val mappedPage1 = initialPage.map(x => (x._1.toString, x._2.map(_.value.toUpperCase))) + + mappedPage1.assertGroupedPage( + ListMap( + "1" -> Vector("A", "B"), + "2" -> Vector("C", "D"), + "3" -> Vector("E", "F") + ), + hasNext = true, + limit = 6, + pageStart = 0, + pageEnd = 5 + ) + + val Right(mappedPage2) = mappedPage1.next + mappedPage2.assertGroupedPage( + ListMap( + "3" -> Vector("AA"), + "4" -> Vector("BB", "CC"), + "5" -> Vector("DD") + ), + hasNext = false, + limit = 6, + pageStart = 6, + pageEnd = 9 + ) + } + + test("Mapping of the page values") { + val mappedPage1 = initialPage.mapValues(_.value.toUpperCase) + + mappedPage1.assertGroupedPage( + ListMap( + 1 -> Vector("A", "B"), + 2 -> Vector("C", "D"), + 3 -> Vector("E", "F") + ), + hasNext = true, + limit = 6, + pageStart = 0, + pageEnd = 5 + ) + + val Right(mappedPage2) = mappedPage1.next + mappedPage2.assertGroupedPage( + ListMap( + 3 -> Vector("AA"), + 4 -> Vector("BB", "CC"), + 5 -> Vector("DD") + ), + hasNext = false, + limit = 6, + pageStart = 6, + pageEnd = 9 + ) + } + + test("Rolling of the pages") { + val Right(nextPage1) = initialPage.next + nextPage1.assertGroupedPage( + ListMap( + 3 -> Vector(PageData.TestItem(3, "aa")), + 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), + 5 -> Vector(PageData.TestItem(5, "dd")) + ), + hasNext = false, + limit = 6, + pageStart = 6, + pageEnd = 9 + ) + assert(nextPage1.hasPrior) + + val noPage1 = nextPage1.next + assert(noPage1 == RequestFail(NoDataException("No next page"))) + + val Right(nextPage2) = initialPage.next(17) + nextPage2.assertGroupedPage( + ListMap( + 3 -> Vector(PageData.TestItem(3, "aa")), + 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), + 5 -> Vector(PageData.TestItem(5, "dd")) + ), + hasNext = false, + limit = 17, + pageStart = 6, + pageEnd = 9 + ) + + val Right(priorPage1) = nextPage2.prior + priorPage1.assertGroupedPage( + ListMap( + 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), + 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), + 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f")) + ), + hasNext = true, + limit = 17, + pageStart = 0, + pageEnd = 5 + ) + + val Right(priorPage2) = nextPage2.prior(7) + priorPage2.assertGroupedPage( + ListMap( + 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), + 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), + 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f")) + ), + hasNext = true, + limit = 7, + pageStart = 0, + pageEnd = 5 + ) + + assert(!priorPage2.hasPrior) + + val noPage2 = priorPage2.prior + assert(noPage2 == RequestFail(NoDataException("No prior page"))) + } + + test("Concatenation of pages") { + val Right(nextPage1) = initialPage.next + val concatenatedPage = initialPage + nextPage1 + concatenatedPage.assertGroupedPage( + ListMap( + 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), + 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), + 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f"), PageData.TestItem(3, "aa")), + 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), + 5 -> Vector(PageData.TestItem(5, "dd")) + ), + hasNext = false, + limit = 6, + pageStart = 0, + pageEnd = 9 + ) + } + +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala new file mode 100644 index 000000000..3731bf8be --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestOK, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.testing.data.PageData +import za.co.absa.atum.testing.data.PageData.TestItem +import za.co.absa.atum.testing.implicits.AbstractPageImplicits.PageEnhancements + +class PageUnitTests extends AnyFunSuiteLike { + + test("Get the items of a page") { + val Right(page) = PageData.roller(6, 0) + + assert(page(0) == TestItem(1, "a")) + assert(page(5) == TestItem(3, "f")) + } + + test("Mapping the page items") { + val Right(origPage) = PageData.roller(6, 0) + + val mapped = origPage.map(_.value.toUpperCase) + + assert(mapped.hasNext == origPage.hasNext) + assert(mapped.limit == origPage.limit) + assert(mapped.pageStart == origPage.pageStart) + assert(mapped.pageEnd == origPage.pageEnd) + assert(mapped.items == Vector("A", "B", "C", "D", "E", "F")) + + val Right(nextPage) = mapped.next + assert(nextPage.items == Vector("AA", "BB", "DD", "CC")) + } + + test("Rolling of the pages") { + val Right(page1) = PageData.roller(6, 0) + val Right(page2) = page1.next + + page2.assertPage(PageData.items2, hasNext = false, limit = 6, pageStart = 6, pageEnd = 9) + val noPage1 = page2.next + assert(noPage1 == RequestFail(NoDataException("No next page"))) + + val Right(page3) = page1.next(100) + page3.assertPage(PageData.items2, hasNext = false, limit = 100, pageStart = 6, pageEnd = 9) + + val Right(page4) = page2.prior(42) + page4.assertPage(PageData.items1, hasNext = true, limit = 42, pageStart = 0, pageEnd = 5) + assert(!page4.hasPrior) + + val noPage2 = page4.prior() + assert(noPage2 == RequestFail(NoDataException("No prior page"))) + } + + test("Concatenation of pages") { + val Right(page1) = PageData.roller(6, 0) + val Right(page2) = page1.next + + val page3 = page1 + page2 + page3.assertPage(PageData.items1 ++ PageData.items2, hasNext = false, limit = 6, pageStart = 0, pageEnd = 9) + } + + test("Grouping of the page items") { + val Right(page1) = PageData.roller(6, 0) + val grouped = page1.groupBy(_.group) + + assert(grouped.groupCount == 3) + assert(grouped.keys.toList == List(1, 2, 3)) + + assert(grouped(1) == Vector(TestItem(1, "a"), TestItem(1, "b"))) + assert(grouped(2) == Vector(TestItem(2, "c"), TestItem(2, "d"))) + assert(grouped(3) == Vector(TestItem(3, "e"), TestItem(3, "f"))) + } +} + diff --git a/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala b/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala new file mode 100644 index 000000000..180753da4 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.testing.data + +import sttp.client3.Identity +import sttp.client3.monad.IdMonad +import sttp.monad.MonadError +import za.co.absa.atum.reader.core.RequestResult.{RequestOK, RequestResult} +import za.co.absa.atum.reader.result.Page + +object PageData { + private implicit val monad: MonadError[Identity] = IdMonad + + case class TestItem(group: Int, value: String) + + val items1: Vector[TestItem] = Vector( + TestItem(1, "a"), + TestItem(1, "b"), + TestItem(2, "c"), + TestItem(2, "d"), + TestItem(3, "e"), + TestItem(3, "f") + ) + + val items2: Vector[TestItem] = Vector( + TestItem(3, "aa"), + TestItem(4, "bb"), + TestItem(5, "dd"), + TestItem(4, "cc") + ) + + def roller(limit: Int, offset: Long): RequestResult[Page[TestItem, Identity]] = { + offset match { + case 0 => + RequestOK( + Page[TestItem, Identity]( + items = items1, + hasNext = true, + limit = limit, + pageStart = 0, + pageEnd = 5, + roller + ) + ) + case 6 => RequestOK( + Page[TestItem, Identity]( + items = items2, + hasNext = false, + limit = limit, + pageStart = 6, + pageEnd = 9, + roller + ) + ) + + } + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala b/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala new file mode 100644 index 000000000..2d8b0b4fe --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.testing.implicits + +import za.co.absa.atum.reader.result.{AbstractPage, GroupedPage, Page} + +import scala.collection.immutable.ListMap + +object AbstractPageImplicits { + implicit class PageEnhancements[T, F[_]](val page: Page[T, F]) extends AnyVal { + def assertPage(items: Vector[T], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long): Unit = { + assert(items == page.items, s"Expected items: $items, but got: ${page.items}") + assert(hasNext == page.hasNext, s"Expected hasNext: $hasNext, but got: ${page.hasNext}") + assert(limit == page.limit, s"Expected limit: $limit, but got: ${page.limit}") + assert(pageStart == page.pageStart, s"Expected pageStart: $pageStart, but got: ${page.pageStart}") + assert(pageEnd == page.pageEnd, s"Expected pageEnd: $pageEnd, but got: ${page.pageEnd}") + } + } + + implicit class GroupedPageEnhancements[K, V, F[_]](val page: GroupedPage[K, V, F]) extends AnyVal { + def assertGroupedPage(items: ListMap[K, Vector[V]], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long): Unit = { + assert(items == page.items, s"Expected items: $items, but got: ${page.items}") + assert(hasNext == page.hasNext, s"Expected hasNext: $hasNext, but got: ${page.hasNext}") + assert(limit == page.limit, s"Expected limit: $limit, but got: ${page.limit}") + assert(pageStart == page.pageStart, s"Expected pageStart: $pageStart, but got: ${page.pageStart}") + assert(pageEnd == page.pageEnd, s"Expected pageEnd: $pageEnd, but got: ${page.pageEnd}") + } + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala b/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala new file mode 100644 index 000000000..16ca2c37b --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.testing.implicits + +import za.co.absa.atum.reader.core.RequestResult.RequestResult +import za.co.absa.atum.reader.result.Page +import za.co.absa.atum.testing.implicits.AbstractPageImplicits.PageEnhancements + +object RequestResultImplicits { + implicit class RequestResultPageEnhancements[T, F[_]](val pageResult: RequestResult[Page[T, F]]) extends AnyVal { + def assertPage(items: Vector[T], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long): Unit = { + pageResult match { + case Right(page) => page.assertPage(items, hasNext, limit, pageStart, pageEnd) + case _ => throw new AssertionError("Expected a page result") + } + } + } + +} From 1488d1fdb549e7167ff6a737ecdbbb8428d76e25 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 4 Feb 2025 11:00:49 +0100 Subject: [PATCH 40/52] #247: Paging support in Reader module * classes abstracting work with paging and rolling through them * `RequestResult` class refactoring --- .github/workflows/test_filenames_check.yml | 1 + .../za/co/absa/atum/reader/FlowReader.scala | 2 +- .../absa/atum/reader/PartitioningReader.scala | 2 +- .../PartitioningIdProvider.scala | 4 +- .../atum/reader/{basic => core}/Reader.scala | 5 +- .../{basic => core}/RequestResult.scala | 23 ++++-- .../PartitioningIdProviderUnitTests.scala | 14 ++-- .../Reader_CatsIOUnitTests.scala | 4 +- .../Reader_FutureUnitTests.scala | 4 +- .../RequestResultUnitTests.scala | 36 ++++++---- .../co/absa/atum/testing/data/PageData.scala | 72 +++++++++++++++++++ .../implicits/AbstractPageImplicits.scala | 51 +++++++++++++ .../implicits/RequestResultImplicits.scala | 37 ++++++++++ 13 files changed, 220 insertions(+), 35 deletions(-) rename reader/src/main/scala/za/co/absa/atum/reader/{basic => core}/PartitioningIdProvider.scala (93%) rename reader/src/main/scala/za/co/absa/atum/reader/{basic => core}/Reader.scala (92%) rename reader/src/main/scala/za/co/absa/atum/reader/{basic => core}/RequestResult.scala (54%) rename reader/src/test/scala/za/co/absa/atum/reader/{basic => core}/PartitioningIdProviderUnitTests.scala (89%) rename reader/src/test/scala/za/co/absa/atum/reader/{basic => core}/Reader_CatsIOUnitTests.scala (95%) rename reader/src/test/scala/za/co/absa/atum/reader/{basic => core}/Reader_FutureUnitTests.scala (95%) rename reader/src/test/scala/za/co/absa/atum/reader/{basic => core}/RequestResultUnitTests.scala (71%) create mode 100644 reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index b0cf8b7bf..cf2d448f7 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -43,5 +43,6 @@ jobs: server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala, server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala, model/src/test/scala/za/co/absa/atum/testing/* + reader/src/test/scala/za/co/absa/atum/testing/* verbose-logging: 'false' fail-on-violation: 'true' diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 6a99bbe40..be3a3d70d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import sttp.client3.SttpBackend import sttp.monad.MonadError import za.co.absa.atum.model.types.basic.AtumPartitions -import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} +import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.server.ServerConfig /** diff --git a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala index f103605b6..334643f44 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/PartitioningReader.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader import sttp.client3.SttpBackend import sttp.monad.MonadError import za.co.absa.atum.model.types.basic.AtumPartitions -import za.co.absa.atum.reader.basic.{PartitioningIdProvider, Reader} +import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.server.ServerConfig /** diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala similarity index 93% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala rename to reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala index 8a802ff02..20c6330c3 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/PartitioningIdProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import sttp.monad.MonadError import sttp.monad.syntax._ @@ -23,7 +23,7 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.model.types.basic.AtumPartitionsOps import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import RequestResult.RequestResult trait PartitioningIdProvider[F[_]] {self: Reader[F] => def partitioningId(partitioning: AtumPartitions)(implicit monad: MonadError[F]): F[RequestResult[Long]] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala similarity index 92% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala rename to reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala index 325f8c6fe..3d1e6e4d8 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/Reader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/Reader.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import io.circe.Decoder import sttp.client3.{Identity, RequestT, ResponseException, SttpBackend, basicRequest} @@ -22,8 +22,9 @@ import sttp.client3.circe.asJson import sttp.model.Uri import sttp.monad.MonadError import sttp.monad.syntax._ +import za.co.absa.atum.reader.core.RequestResult._ import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.exceptions.RequestException.CirceError /** * Reader is a base class for reading data from a remote server. diff --git a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala similarity index 54% rename from reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala rename to reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala index 76e8cbfa9..6861a3993 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/basic/RequestResult.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala @@ -14,25 +14,36 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} +import sttp.monad.MonadError import za.co.absa.atum.model.envelopes.ErrorResponse +import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} +import za.co.absa.atum.reader.exceptions.RequestException +import za.co.absa.atum.reader.result.Page object RequestResult { - type CirceError = io.circe.Error - type RequestResult[R] = Either[ResponseException[ErrorResponse, CirceError], R] + type RequestResult[R] = Either[RequestException, R] + + def RequestOK[T](value: T): RequestResult[T] = Right(value) + def RequestFail[T](error: RequestException): RequestResult[T] = Left(error) implicit class ResponseOps[R](val response: Response[Either[ResponseException[String, CirceError], R]]) extends AnyVal { def toRequestResult: RequestResult[R] = { response.body.left.map { case he: HttpError[String] => ErrorResponse.basedOnStatusCode(he.statusCode.code, he.body) match { - case Right(er) => HttpError(er, he.statusCode) - case Left(ce) => DeserializationException(he.body, ce) + case Right(er) => HttpException(he.getMessage, he.statusCode, er, response.request.uri) + case Left(ce) => ParsingException.fromCirceError(ce, he.body) } - case de: DeserializationException[CirceError] => de + case de: DeserializationException[CirceError] => ParsingException.fromCirceError(de.error, de.body) } } } + + implicit class RequestPageResultOps[A, F[_]: MonadError](requestResult: RequestResult[Page[A, F]]) { + def pageMap[B](f: A => B): RequestResult[Page[B, F]] = requestResult.map(_.map(f)) + } + } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/core/PartitioningIdProviderUnitTests.scala similarity index 89% rename from reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala rename to reader/src/test/scala/za/co/absa/atum/reader/core/PartitioningIdProviderUnitTests.scala index 20cac1a02..22672eabf 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/PartitioningIdProviderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/core/PartitioningIdProviderUnitTests.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import org.scalatest.funsuite.AnyFunSuiteLike import sttp.capabilities @@ -28,7 +28,8 @@ import za.co.absa.atum.model.envelopes.NotFoundErrorResponse import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.core.RequestResult._ +import za.co.absa.atum.reader.exceptions.RequestException.{HttpException, ParsingException} import za.co.absa.atum.reader.server.ServerConfig class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { @@ -38,7 +39,7 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { private val atumPartitionsToNotFound = AtumPartitions(List.empty) private implicit val serverConfig: ServerConfig = ServerConfig(serverUrl) - private implicit val monad: IdMonad.type = IdMonad + private implicit val monad: MonadError[Identity] = IdMonad private implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatches(request => isUriOfAtumPartitions(request.uri, atumPartitionsToReply)) .thenRespond(SingleSuccessResponse(PartitioningWithIdDTO(1, atumPartitionsToReply.toPartitioningDTO, "Gimli")).asJsonString) @@ -76,9 +77,8 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { val result = reader.partitioningId(atumPartitionsToNotFound) result match { case Right(_) => fail("Expected a failure, but OK response received") - case Left(_: DeserializationException[CirceError]) => fail("Expected a not found response, but deserialization error received") - case Left(x: HttpError[_]) => - assert(x.body.isInstanceOf[NotFoundErrorResponse]) + case Left(x: HttpException) => + assert(x.errorResponse.isInstanceOf[NotFoundErrorResponse]) assert(x.statusCode == StatusCode.NotFound) case _ => fail("Unexpected response") } @@ -88,6 +88,6 @@ class PartitioningIdProviderUnitTests extends AnyFunSuiteLike { val reader = ReaderWithPartitioningIdForTest(atumPartitionsToFailedDecode) val result = reader.partitioningId(atumPartitionsToFailedDecode) assert(result.isLeft) - result.swap.map(e => assert(e.isInstanceOf[DeserializationException[CirceError]])) + result.swap.map(e => assert(e.isInstanceOf[ParsingException])) } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/core/Reader_CatsIOUnitTests.scala similarity index 95% rename from reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala rename to reader/src/test/scala/za/co/absa/atum/reader/core/Reader_CatsIOUnitTests.scala index 1aaad0901..b02cbcd21 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_CatsIOUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/core/Reader_CatsIOUnitTests.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import cats.effect.unsafe.implicits.global import io.circe.Decoder @@ -24,7 +24,7 @@ import sttp.client3.testing.SttpBackendStub import sttp.monad.{MonadAsyncError, MonadError} import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.reader.server.ServerConfig class Reader_CatsIOUnitTests extends AnyFunSuiteLike { diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/core/Reader_FutureUnitTests.scala similarity index 95% rename from reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala rename to reader/src/test/scala/za/co/absa/atum/reader/core/Reader_FutureUnitTests.scala index c19c6411d..cf200137a 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/Reader_FutureUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/core/Reader_FutureUnitTests.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import io.circe.Decoder import org.scalatest.funsuite.AnyFunSuiteLike @@ -23,7 +23,7 @@ import sttp.client3.testing.SttpBackendStub import sttp.monad.MonadError import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult.RequestResult +import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.reader.server.ServerConfig import scala.concurrent.duration.Duration diff --git a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/core/RequestResultUnitTests.scala similarity index 71% rename from reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala rename to reader/src/test/scala/za/co/absa/atum/reader/core/RequestResultUnitTests.scala index d181154df..34f8c7933 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/basic/RequestResultUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/core/RequestResultUnitTests.scala @@ -14,16 +14,17 @@ * limitations under the License. */ -package za.co.absa.atum.reader.basic +package za.co.absa.atum.reader.core import io.circe.ParsingFailure import org.scalatest.funsuite.AnyFunSuiteLike import sttp.client3.{DeserializationException, HttpError, Response, ResponseException} -import sttp.model.StatusCode +import sttp.model.{StatusCode, Uri} import za.co.absa.atum.model.dto.PartitionDTO import za.co.absa.atum.model.envelopes.NotFoundErrorResponse import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import za.co.absa.atum.reader.basic.RequestResult._ +import za.co.absa.atum.reader.core.RequestResult._ +import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} class RequestResultUnitTests extends AnyFunSuiteLike { test("Response.toRequestResult keeps the right value") { @@ -37,7 +38,8 @@ class RequestResultUnitTests extends AnyFunSuiteLike { assert(result == body) } - test("Response.toRequestResult keeps the left value if it's a CirceError") { + test("Response.toRequestResult keeps the left value if it's a CirceError with its message") { + val circeError: CirceError = ParsingFailure("Just a test error", new Exception) val deserializationException = DeserializationException("This is not a json", circeError) val body = Left(deserializationException) @@ -46,20 +48,30 @@ class RequestResultUnitTests extends AnyFunSuiteLike { StatusCode.Ok ) val result = source.toRequestResult - assert(result == body) + result match { + case Left(ParsingException(message, body)) => + assert(message == "Just a test error") + assert(body == "This is not a json") + case _ => fail("Unexpected result") + } } test("Response.toRequestResult decodes NotFound error") { - val error = NotFoundErrorResponse("This is a test") - val errorResponse = error.asJsonString - val httpError = HttpError(errorResponse, StatusCode.NotFound) + val sourceError = NotFoundErrorResponse("This is a test") + val sourceErrorResponse = sourceError.asJsonString + val httpError = HttpError(sourceErrorResponse, StatusCode.NotFound) val source: Response[Either[ResponseException[String, CirceError], PartitionDTO]] = Response( Left(httpError), StatusCode.Ok ) val result = source.toRequestResult - val expected: RequestResult[PartitionDTO] = Left(HttpError(error, httpError.statusCode)) - assert(result == expected) + result match { + case Left(HttpException(_, statusCode, errorResponse, request)) => + assert(statusCode == StatusCode.NotFound) + assert(errorResponse == sourceError) + assert(request == Uri("example.com")) + case _ => fail("Unexpected result") + } } test("Response.toRequestResult fails to decode InternalServerErrorResponse error") { @@ -74,8 +86,8 @@ class RequestResultUnitTests extends AnyFunSuiteLike { assert(result.isLeft) result.swap.foreach { e => // investigate the error - assert(e.isInstanceOf[DeserializationException[_]]) - val ce = e.asInstanceOf[DeserializationException[ParsingFailure]] + assert(e.isInstanceOf[ParsingException]) + val ce = e.asInstanceOf[ParsingException] assert(ce.body == responseBody) } } diff --git a/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala b/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala new file mode 100644 index 000000000..180753da4 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.testing.data + +import sttp.client3.Identity +import sttp.client3.monad.IdMonad +import sttp.monad.MonadError +import za.co.absa.atum.reader.core.RequestResult.{RequestOK, RequestResult} +import za.co.absa.atum.reader.result.Page + +object PageData { + private implicit val monad: MonadError[Identity] = IdMonad + + case class TestItem(group: Int, value: String) + + val items1: Vector[TestItem] = Vector( + TestItem(1, "a"), + TestItem(1, "b"), + TestItem(2, "c"), + TestItem(2, "d"), + TestItem(3, "e"), + TestItem(3, "f") + ) + + val items2: Vector[TestItem] = Vector( + TestItem(3, "aa"), + TestItem(4, "bb"), + TestItem(5, "dd"), + TestItem(4, "cc") + ) + + def roller(limit: Int, offset: Long): RequestResult[Page[TestItem, Identity]] = { + offset match { + case 0 => + RequestOK( + Page[TestItem, Identity]( + items = items1, + hasNext = true, + limit = limit, + pageStart = 0, + pageEnd = 5, + roller + ) + ) + case 6 => RequestOK( + Page[TestItem, Identity]( + items = items2, + hasNext = false, + limit = limit, + pageStart = 6, + pageEnd = 9, + roller + ) + ) + + } + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala b/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala new file mode 100644 index 000000000..2d8b0b4fe --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.testing.implicits + +import za.co.absa.atum.reader.result.{AbstractPage, GroupedPage, Page} + +import scala.collection.immutable.ListMap + +object AbstractPageImplicits { + implicit class PageEnhancements[T, F[_]](val page: Page[T, F]) extends AnyVal { + def assertPage(items: Vector[T], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long): Unit = { + assert(items == page.items, s"Expected items: $items, but got: ${page.items}") + assert(hasNext == page.hasNext, s"Expected hasNext: $hasNext, but got: ${page.hasNext}") + assert(limit == page.limit, s"Expected limit: $limit, but got: ${page.limit}") + assert(pageStart == page.pageStart, s"Expected pageStart: $pageStart, but got: ${page.pageStart}") + assert(pageEnd == page.pageEnd, s"Expected pageEnd: $pageEnd, but got: ${page.pageEnd}") + } + } + + implicit class GroupedPageEnhancements[K, V, F[_]](val page: GroupedPage[K, V, F]) extends AnyVal { + def assertGroupedPage(items: ListMap[K, Vector[V]], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long): Unit = { + assert(items == page.items, s"Expected items: $items, but got: ${page.items}") + assert(hasNext == page.hasNext, s"Expected hasNext: $hasNext, but got: ${page.hasNext}") + assert(limit == page.limit, s"Expected limit: $limit, but got: ${page.limit}") + assert(pageStart == page.pageStart, s"Expected pageStart: $pageStart, but got: ${page.pageStart}") + assert(pageEnd == page.pageEnd, s"Expected pageEnd: $pageEnd, but got: ${page.pageEnd}") + } + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala b/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala new file mode 100644 index 000000000..16ca2c37b --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.testing.implicits + +import za.co.absa.atum.reader.core.RequestResult.RequestResult +import za.co.absa.atum.reader.result.Page +import za.co.absa.atum.testing.implicits.AbstractPageImplicits.PageEnhancements + +object RequestResultImplicits { + implicit class RequestResultPageEnhancements[T, F[_]](val pageResult: RequestResult[Page[T, F]]) extends AnyVal { + def assertPage(items: Vector[T], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long): Unit = { + pageResult match { + case Right(page) => page.assertPage(items, hasNext, limit, pageStart, pageEnd) + case _ => throw new AssertionError("Expected a page result") + } + } + } + +} From 96ffa33d857b80ae03fc80324e8fa43584df5785 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 4 Feb 2025 11:05:47 +0100 Subject: [PATCH 41/52] * some omitted files --- .../reader/exceptions/ReaderException.scala | 19 ++ .../reader/exceptions/RequestException.scala | 49 +++++ .../atum/reader/result/AbstractPage.scala | 31 +++ .../absa/atum/reader/result/GroupedPage.scala | 93 +++++++++ .../za/co/absa/atum/reader/result/Page.scala | 96 ++++++++++ .../reader/result/AbstractPageUnitTests.scala | 53 ++++++ .../reader/result/GroupedPageUnitTests.scala | 179 ++++++++++++++++++ .../atum/reader/result/PageUnitTests.scala | 89 +++++++++ 8 files changed, 609 insertions(+) create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala create mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala new file mode 100644 index 000000000..61ae91e06 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/ReaderException.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.exceptions + +class ReaderException(message: String) extends Exception(message) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala new file mode 100644 index 000000000..0cde4d94f --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.exceptions + +import sttp.model.{StatusCode, Uri} +import za.co.absa.atum.model.envelopes.ErrorResponse + +abstract class RequestException(message: String) extends ReaderException(message) + + +object RequestException { + type CirceError = io.circe.Error + + final case class HttpException( + message: String, + statusCode: StatusCode, + errorResponse: ErrorResponse, + request: Uri + ) extends RequestException(message) + + final case class ParsingException( + message: String, + body: String + ) extends RequestException(message) + object ParsingException { + def fromCirceError(error: CirceError, body: String): ParsingException = { + ParsingException(error.getMessage, body) + } + } + + + final case class NoDataException( + message: String + ) extends RequestException(message) +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala new file mode 100644 index 000000000..042339572 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import sttp.monad.MonadError + +abstract class AbstractPage [T <: Iterable[_], F[_]: MonadError] { + def items: T + def hasNext: Boolean + def limit: Int + def pageStart: Long + def pageEnd: Long + + def pageSize: Int = (pageEnd - pageStart).toInt + 1 + def hasPrior: Boolean = pageStart > 0 +} + diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala new file mode 100644 index 000000000..7a929bb9f --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala @@ -0,0 +1,93 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller + +import scala.collection.immutable.ListMap + +case class GroupedPage[K, V, F[_]: MonadError]( + items: ListMap[K, Vector[V]], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long, + private[reader] val pageRoller: GroupPageRoller[K, V, F] + ) extends AbstractPage[Map[K, Vector[V]], F] { + + def apply(key: K): Vector[V] = items(key) + def keys: Iterable[K] = items.keys + def groupCount: Int = items.size + + def map[K1, V1](f: ((K, Vector[V])) => (K1, Vector[V1])): GroupedPage[K1, V1, F] = { + val newItems = items.map(f) + val newPageRoller: GroupPageRoller[K1, V1, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.map(f))) + this.copy(items = newItems, pageRoller = newPageRoller) + } + + def mapValues[B](f: V => B): GroupedPage[K, B, F] = { + def mapper(item: (K, Vector[V])): (K, Vector[B]) = (item._1, item._2.map(f)) + + val newItems = items.map(mapper) + val newPageRoller: GroupPageRoller[K, B, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.mapValues(f))) + this.copy(items = newItems, pageRoller = newPageRoller) + + } + + def prior(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { + if (hasPrior) { + val newOffset = (pageStart - limit).max(0) + pageRoller(newPageSize, newOffset) + } else { + MonadError[F].unit(RequestFail(NoDataException("No prior page"))) + } + } + + def prior: F[RequestResult[GroupedPage[K, V, F]]] = prior(limit) + + def next(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { + if (hasNext) { + pageRoller(newPageSize, pageStart + limit) + } else { + MonadError[F].unit(RequestFail(NoDataException("No next page"))) + } + } + + def next: F[RequestResult[GroupedPage[K, V, F]]] = next(limit) + + def +(other: GroupedPage[K, V, F]): GroupedPage[K, V, F] = { + val newItems = other.items.foldLeft(items) { case (acc, (k, v)) => + if (acc.contains(k)) { + acc.updated(k, acc(k) ++ v) + } else { + acc + (k -> v) + } + } + val newHasNext = hasNext && other.hasNext + val newPageStart = pageStart min other.pageStart + val newPageEnd = pageEnd max other.pageEnd + this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd) + } +} + +object GroupedPage { + type GroupPageRoller[K, V, F[_]] = (Int, Long) => F[RequestResult[GroupedPage[K, V, F]]] +} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala new file mode 100644 index 000000000..6d2705f59 --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import sttp.monad.MonadError +import sttp.monad.syntax._ +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestPageResultOps, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller +import za.co.absa.atum.reader.result.Page.PageRoller + +import scala.collection.immutable.ListMap + +case class Page[T, F[_]: MonadError]( + items: Vector[T], + hasNext: Boolean, + limit: Int, + pageStart: Long, + pageEnd: Long, + private[reader] val pageRoller: PageRoller[T, F] + ) extends AbstractPage[Vector[T], F] { + + def apply(index: Int): T = items(index) + + def map[B](f: T => B): Page[B, F] = { + val newItems = items.map(f) + val newPageRoller: PageRoller[B, F] = (limit, offset) => pageRoller(limit, offset).map(_.pageMap(f)) + this.copy(items = newItems, pageRoller = newPageRoller) + } + + def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = { + if (hasPrior) { + val newOffset = (pageStart - newPageSize).max(0) + pageRoller(newPageSize, newOffset) + } else { + MonadError[F].unit(RequestFail(NoDataException("No prior page"))) + } + } + + def prior(): F[RequestResult[Page[T, F]]] = prior(limit) + + def next(newPageSize: Int): F[RequestResult[Page[T, F]]] = { + if (hasNext) { + pageRoller(newPageSize, pageStart + pageSize) + } else { + MonadError[F].unit(RequestFail(NoDataException("No next page"))) + } + } + + def next: F[RequestResult[Page[T, F]]] = next(limit) + + def +(other: Page[T, F]): Page[T, F] = { + val newItems = items ++ other.items + val newPageStart = pageStart min other.pageStart + val newPageEnd = pageEnd max other.pageEnd + val newHasNext = hasNext && other.hasNext + this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd) + } + + def groupBy[K](f: T => K): GroupedPage[K, T, F] = { + val (newItems, itemsCounts) = items.foldLeft(ListMap.empty[K, Vector[T]], 0) { case ((groupsAcc, count), item) => + val k = f(item) + (groupsAcc.updated(k, groupsAcc.getOrElse(k, Vector.empty) :+ item), count + 1) + } + val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) => + pageRoller(limit, offset) + .map(_.map(_.groupBy(f))) + + GroupedPage( + newItems, + hasNext, + limit, + pageStart, + pageEnd, + newPageRoller + ) + } +} + +object Page { + type PageRoller[T, F[_]] = (Int, Long) => F[RequestResult[Page[T, F]]] +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala new file mode 100644 index 000000000..94623783f --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import org.scalatest.funsuite.AnyFunSuiteLike +import sttp.client3.Identity +import sttp.client3.monad.IdMonad +import sttp.monad.MonadError + + +class AbstractPageUnitTests extends AnyFunSuiteLike { + private implicit val monad: MonadError[Identity] = IdMonad + + test("Basic test") { + val page = new AbstractPage[Iterable[Int], Identity] { + override def items: Iterable[Int] = Seq(1, 2, 3) + override def hasNext: Boolean = true + override def limit: Int = 3 + override def pageStart: Long = 0 + override def pageEnd: Long = 2 + } + + assert(page.items.size == 3) + assert(page.hasNext) + assert(page.limit == 3) + assert(page.pageStart == 0) + assert(page.pageSize == 3) + assert(!page.hasPrior) + + val anotherPage = new AbstractPage[Iterable[Int], Identity] { + override def items: Iterable[Int] = Seq(1, 2, 3) + override def hasNext: Boolean = true + override def limit: Int = 3 + override def pageStart: Long = 1 + override def pageEnd: Long = 2 + } + assert(anotherPage.hasPrior) + } +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala new file mode 100644 index 000000000..5c79f9e79 --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala @@ -0,0 +1,179 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.reader.core.RequestResult.RequestFail +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.testing.data.PageData +import za.co.absa.atum.testing.implicits.AbstractPageImplicits.GroupedPageEnhancements + +import scala.collection.immutable.ListMap + + +class GroupedPageUnitTests extends AnyFunSuiteLike { + private val initialPage = { + val Right(ungrouped) = PageData.roller(6, 0) + ungrouped.groupBy(_.group) + } + + test("Getting items, group keys, and group count") { + assert(initialPage(3) == Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f"))) + assert(initialPage.groupCount == 3) + assert(initialPage.keys.toList == List(1, 2, 3)) + } + + test("Mapping of the page") { + val mappedPage1 = initialPage.map(x => (x._1.toString, x._2.map(_.value.toUpperCase))) + + mappedPage1.assertGroupedPage( + ListMap( + "1" -> Vector("A", "B"), + "2" -> Vector("C", "D"), + "3" -> Vector("E", "F") + ), + hasNext = true, + limit = 6, + pageStart = 0, + pageEnd = 5 + ) + + val Right(mappedPage2) = mappedPage1.next + mappedPage2.assertGroupedPage( + ListMap( + "3" -> Vector("AA"), + "4" -> Vector("BB", "CC"), + "5" -> Vector("DD") + ), + hasNext = false, + limit = 6, + pageStart = 6, + pageEnd = 9 + ) + } + + test("Mapping of the page values") { + val mappedPage1 = initialPage.mapValues(_.value.toUpperCase) + + mappedPage1.assertGroupedPage( + ListMap( + 1 -> Vector("A", "B"), + 2 -> Vector("C", "D"), + 3 -> Vector("E", "F") + ), + hasNext = true, + limit = 6, + pageStart = 0, + pageEnd = 5 + ) + + val Right(mappedPage2) = mappedPage1.next + mappedPage2.assertGroupedPage( + ListMap( + 3 -> Vector("AA"), + 4 -> Vector("BB", "CC"), + 5 -> Vector("DD") + ), + hasNext = false, + limit = 6, + pageStart = 6, + pageEnd = 9 + ) + } + + test("Rolling of the pages") { + val Right(nextPage1) = initialPage.next + nextPage1.assertGroupedPage( + ListMap( + 3 -> Vector(PageData.TestItem(3, "aa")), + 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), + 5 -> Vector(PageData.TestItem(5, "dd")) + ), + hasNext = false, + limit = 6, + pageStart = 6, + pageEnd = 9 + ) + assert(nextPage1.hasPrior) + + val noPage1 = nextPage1.next + assert(noPage1 == RequestFail(NoDataException("No next page"))) + + val Right(nextPage2) = initialPage.next(17) + nextPage2.assertGroupedPage( + ListMap( + 3 -> Vector(PageData.TestItem(3, "aa")), + 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), + 5 -> Vector(PageData.TestItem(5, "dd")) + ), + hasNext = false, + limit = 17, + pageStart = 6, + pageEnd = 9 + ) + + val Right(priorPage1) = nextPage2.prior + priorPage1.assertGroupedPage( + ListMap( + 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), + 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), + 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f")) + ), + hasNext = true, + limit = 17, + pageStart = 0, + pageEnd = 5 + ) + + val Right(priorPage2) = nextPage2.prior(7) + priorPage2.assertGroupedPage( + ListMap( + 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), + 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), + 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f")) + ), + hasNext = true, + limit = 7, + pageStart = 0, + pageEnd = 5 + ) + + assert(!priorPage2.hasPrior) + + val noPage2 = priorPage2.prior + assert(noPage2 == RequestFail(NoDataException("No prior page"))) + } + + test("Concatenation of pages") { + val Right(nextPage1) = initialPage.next + val concatenatedPage = initialPage + nextPage1 + concatenatedPage.assertGroupedPage( + ListMap( + 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), + 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), + 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f"), PageData.TestItem(3, "aa")), + 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), + 5 -> Vector(PageData.TestItem(5, "dd")) + ), + hasNext = false, + limit = 6, + pageStart = 0, + pageEnd = 9 + ) + } + +} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala new file mode 100644 index 000000000..3731bf8be --- /dev/null +++ b/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.result + +import org.scalatest.funsuite.AnyFunSuiteLike +import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestOK, RequestResult} +import za.co.absa.atum.reader.exceptions.RequestException.NoDataException +import za.co.absa.atum.testing.data.PageData +import za.co.absa.atum.testing.data.PageData.TestItem +import za.co.absa.atum.testing.implicits.AbstractPageImplicits.PageEnhancements + +class PageUnitTests extends AnyFunSuiteLike { + + test("Get the items of a page") { + val Right(page) = PageData.roller(6, 0) + + assert(page(0) == TestItem(1, "a")) + assert(page(5) == TestItem(3, "f")) + } + + test("Mapping the page items") { + val Right(origPage) = PageData.roller(6, 0) + + val mapped = origPage.map(_.value.toUpperCase) + + assert(mapped.hasNext == origPage.hasNext) + assert(mapped.limit == origPage.limit) + assert(mapped.pageStart == origPage.pageStart) + assert(mapped.pageEnd == origPage.pageEnd) + assert(mapped.items == Vector("A", "B", "C", "D", "E", "F")) + + val Right(nextPage) = mapped.next + assert(nextPage.items == Vector("AA", "BB", "DD", "CC")) + } + + test("Rolling of the pages") { + val Right(page1) = PageData.roller(6, 0) + val Right(page2) = page1.next + + page2.assertPage(PageData.items2, hasNext = false, limit = 6, pageStart = 6, pageEnd = 9) + val noPage1 = page2.next + assert(noPage1 == RequestFail(NoDataException("No next page"))) + + val Right(page3) = page1.next(100) + page3.assertPage(PageData.items2, hasNext = false, limit = 100, pageStart = 6, pageEnd = 9) + + val Right(page4) = page2.prior(42) + page4.assertPage(PageData.items1, hasNext = true, limit = 42, pageStart = 0, pageEnd = 5) + assert(!page4.hasPrior) + + val noPage2 = page4.prior() + assert(noPage2 == RequestFail(NoDataException("No prior page"))) + } + + test("Concatenation of pages") { + val Right(page1) = PageData.roller(6, 0) + val Right(page2) = page1.next + + val page3 = page1 + page2 + page3.assertPage(PageData.items1 ++ PageData.items2, hasNext = false, limit = 6, pageStart = 0, pageEnd = 9) + } + + test("Grouping of the page items") { + val Right(page1) = PageData.roller(6, 0) + val grouped = page1.groupBy(_.group) + + assert(grouped.groupCount == 3) + assert(grouped.keys.toList == List(1, 2, 3)) + + assert(grouped(1) == Vector(TestItem(1, "a"), TestItem(1, "b"))) + assert(grouped(2) == Vector(TestItem(2, "c"), TestItem(2, "d"))) + assert(grouped(3) == Vector(TestItem(3, "e"), TestItem(3, "f"))) + } +} + From 04b56bdbff597fa6b10fa397f2b41f2799b1343c Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 4 Feb 2025 11:57:20 +0100 Subject: [PATCH 42/52] * filenames check exclusions fix --- .github/workflows/test_filenames_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index cf2d448f7..30b04ceb9 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -42,7 +42,7 @@ jobs: server/src/test/scala/za/co/absa/atum/server/api/TestData.scala, server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala, server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala, - model/src/test/scala/za/co/absa/atum/testing/* - reader/src/test/scala/za/co/absa/atum/testing/* + model/src/test/scala/za/co/absa/atum/testing/** + reader/src/test/scala/za/co/absa/atum/testing/** verbose-logging: 'false' fail-on-violation: 'true' From 855a333113d3e7f1d11f49aa530aa10dc35335a9 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 4 Feb 2025 12:08:08 +0100 Subject: [PATCH 43/52] * further fix --- .github/workflows/test_filenames_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index 30b04ceb9..61d3a1d1b 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -42,7 +42,7 @@ jobs: server/src/test/scala/za/co/absa/atum/server/api/TestData.scala, server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala, server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala, - model/src/test/scala/za/co/absa/atum/testing/** - reader/src/test/scala/za/co/absa/atum/testing/** + model/src/test/scala/za/co/absa/atum/testing/**/*.* + reader/src/test/scala/za/co/absa/atum/testing/**/* verbose-logging: 'false' fail-on-violation: 'true' From be9071153a325779bc6039d0057c5fb86d021fe5 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 4 Feb 2025 12:21:53 +0100 Subject: [PATCH 44/52] * fix trial 3 --- .github/workflows/test_filenames_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_filenames_check.yml b/.github/workflows/test_filenames_check.yml index 61d3a1d1b..b5c805c55 100644 --- a/.github/workflows/test_filenames_check.yml +++ b/.github/workflows/test_filenames_check.yml @@ -42,7 +42,7 @@ jobs: server/src/test/scala/za/co/absa/atum/server/api/TestData.scala, server/src/test/scala/za/co/absa/atum/server/api/TestTransactorProvider.scala, server/src/test/scala/za/co/absa/atum/server/ConfigProviderTest.scala, - model/src/test/scala/za/co/absa/atum/testing/**/*.* - reader/src/test/scala/za/co/absa/atum/testing/**/* + model/src/test/scala/za/co/absa/atum/testing/*, + reader/src/test/scala/za/co/absa/atum/testing/* verbose-logging: 'false' fail-on-violation: 'true' From afd64a6dc191237f134094ccd7a1745eccdae266 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 4 Feb 2025 17:26:06 +0100 Subject: [PATCH 45/52] * Licenses fix --- .../za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala | 2 +- .../atum/model/utils/JsonDeserializationSyntaxUnitTests.scala | 2 +- .../atum/model/utils/JsonSerializationSyntaxUnitTests.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala index 11a529d02..cbd5e5fc1 100644 --- a/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/types/AtumPartitionsUnitTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala index 4ca1ac9df..f07b8727e 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonDeserializationSyntaxUnitTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala index 830f4e9b7..25a815891 100644 --- a/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala +++ b/model/src/test/scala/za/co/absa/atum/model/utils/JsonSerializationSyntaxUnitTests.scala @@ -1,5 +1,5 @@ /* - * Copyright 2024 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From f803ea376fc51d5c4d800a70f7e504b3d71a5c35 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Wed, 5 Feb 2025 13:53:37 +0100 Subject: [PATCH 46/52] * uncommented endpoint for Swagger documentation creation --- .../src/main/scala/za/co/absa/atum/server/api/http/Routes.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala index 04027f47d..978633997 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Routes.scala @@ -133,7 +133,7 @@ trait Routes extends Endpoints with ServerOptions { // getPartitioningMeasuresEndpointV2, // getFlowPartitioningsEndpointV2, getPartitioningMainFlowEndpointV2, - // getFlowCheckpointsEndpointV2, + getFlowCheckpointsEndpointV2, healthEndpoint ) ZHttp4sServerInterpreter[HttpEnv.Env](http4sServerOptions(None)) From 1a56f3d57a83ae624c7084486075479c020b858b Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Feb 2025 12:29:11 +0100 Subject: [PATCH 47/52] * complete refactoring --- .../za/co/absa/atum/reader/FlowReader.scala | 34 ++-- .../absa/atum/reader/core/RequestResult.scala | 5 - .../reader/exceptions/RequestException.scala | 2 +- .../PaginatedResponseImplicits.scala | 38 ---- .../atum/reader/result/AbstractPage.scala | 31 --- .../absa/atum/reader/result/GroupedPage.scala | 93 --------- .../za/co/absa/atum/reader/result/Page.scala | 96 ---------- .../atum/reader/FlowReaderUnitTests.scala | 177 +++++++---------- .../PaginatedResponseImplicitsTest.scala | 47 ----- .../reader/result/AbstractPageUnitTests.scala | 53 ------ .../reader/result/GroupedPageUnitTests.scala | 179 ------------------ .../atum/reader/result/PageUnitTests.scala | 89 --------- .../co/absa/atum/testing/data/PageData.scala | 72 ------- .../implicits/AbstractPageImplicits.scala | 51 ----- .../implicits/RequestResultImplicits.scala | 37 ---- 15 files changed, 84 insertions(+), 920 deletions(-) delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala delete mode 100644 reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala delete mode 100644 reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 4ac52802d..84155f6d8 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -21,16 +21,12 @@ import sttp.monad.MonadError import sttp.monad.syntax._ import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, FlowDTO} import za.co.absa.atum.model.envelopes.SuccessResponse.{PaginatedResponse, SingleSuccessResponse} -import za.co.absa.atum.model.types.basic.{AtumPartitions, PartitioningDTOOps} -import za.co.absa.atum.reader.core.RequestResult.{RequestPageResultOps, RequestResult} +import za.co.absa.atum.model.types.basic.AtumPartitions +import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.model.ApiPaths._ -import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} -import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements -import za.co.absa.atum.reader.result.Page import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.result.Page.PageRoller /** * This class is a reader that reads data tight to a flow. @@ -65,28 +61,20 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) getQuery(endpoint, params) } - private def checkpointMapper(data: CheckpointWithPartitioningDTO): AtumPartitionsCheckpoint = { - val atumPartitions = data.partitioning.partitioning.toAtumPartitions - val checkpoint = Checkpoint(data) - AtumPartitionsCheckpoint(atumPartitions, checkpoint) - } - - def getCheckpointDTOs(checkpointName: Option[String], pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[CheckpointWithPartitioningDTO, F]]] = { - val pageRoller: PageRoller[CheckpointWithPartitioningDTO, F] = getCheckpointDTOs(checkpointName, _, _) - + def getCheckpointsPage(pageSize: Int = 10, offset: Long = 0): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { for { mainPartitioningId <- partitioningId(mainFlowPartitioning) flowId <- mainPartitioningId.project(queryFlowId) - checkpoints <- flowId.project(queryCheckpoints(_, checkpointName, pageSize, offset)) - } yield checkpoints.map(_.toPage(pageRoller)) - + checkpoints <- flowId.project(queryCheckpoints(_, None, pageSize, offset)) + } yield checkpoints } - def getCheckpoints(pageSize: Int = 10, offset: Long = 0): F[RequestResult[Page[AtumPartitionsCheckpoint, F]]] = { - getCheckpointDTOs(None, pageSize, offset).map(_.pageMap(checkpointMapper)) + def getCheckpointsOfNamePage(checkpointName: String, pageSize: Int = 10, offset: Long = 0): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { + for { + mainPartitioningId <- partitioningId(mainFlowPartitioning) + flowId <- mainPartitioningId.project(queryFlowId) + checkpoints <- flowId.project(queryCheckpoints(_, Some(checkpointName), pageSize, offset)) + } yield checkpoints } - def getCheckpointsOfName(name: String, pageSize: Int = 10, offset: Int = 0): F[RequestResult[Page[AtumPartitionsCheckpoint, F]]] = { - getCheckpointDTOs(Some(name), pageSize, offset).map(_.pageMap(checkpointMapper)) - } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala index 6861a3993..c54995597 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/RequestResult.scala @@ -21,7 +21,6 @@ import sttp.monad.MonadError import za.co.absa.atum.model.envelopes.ErrorResponse import za.co.absa.atum.reader.exceptions.RequestException.{CirceError, HttpException, ParsingException} import za.co.absa.atum.reader.exceptions.RequestException -import za.co.absa.atum.reader.result.Page object RequestResult { type RequestResult[R] = Either[RequestException, R] @@ -42,8 +41,4 @@ object RequestResult { } } - implicit class RequestPageResultOps[A, F[_]: MonadError](requestResult: RequestResult[Page[A, F]]) { - def pageMap[B](f: A => B): RequestResult[Page[B, F]] = requestResult.map(_.map(f)) - } - } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala index 0cde4d94f..5cc449165 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -19,7 +19,7 @@ package za.co.absa.atum.reader.exceptions import sttp.model.{StatusCode, Uri} import za.co.absa.atum.model.envelopes.ErrorResponse -abstract class RequestException(message: String) extends ReaderException(message) +sealed abstract class RequestException(message: String) extends ReaderException(message) object RequestException { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala b/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala deleted file mode 100644 index 298f09248..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicits.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.implicits - -import sttp.monad.MonadError -import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse -import za.co.absa.atum.reader.result.Page -import za.co.absa.atum.reader.result.Page.PageRoller - -object PaginatedResponseImplicits { - implicit class PaginatedResponseMonadEnhancements[T](val paginatedResponse: PaginatedResponse[T]) extends AnyVal { - def toPage[F[_]: MonadError](pageRoller: PageRoller[T, F]): Page[T, F] = { - val data = paginatedResponse.data.toVector - Page( - items = data, - hasNext = paginatedResponse.pagination.hasMore, - limit = paginatedResponse.pagination.limit, - pageStart = paginatedResponse.pagination.offset, - pageEnd = paginatedResponse.pagination.offset + data.size - 1, - pageRoller = pageRoller - ) - } - } -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala deleted file mode 100644 index 042339572..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/AbstractPage.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.result - -import sttp.monad.MonadError - -abstract class AbstractPage [T <: Iterable[_], F[_]: MonadError] { - def items: T - def hasNext: Boolean - def limit: Int - def pageStart: Long - def pageEnd: Long - - def pageSize: Int = (pageEnd - pageStart).toInt + 1 - def hasPrior: Boolean = pageStart > 0 -} - diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala deleted file mode 100644 index 7a929bb9f..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/GroupedPage.scala +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.result - -import sttp.monad.MonadError -import sttp.monad.syntax._ -import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestResult} -import za.co.absa.atum.reader.exceptions.RequestException.NoDataException -import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller - -import scala.collection.immutable.ListMap - -case class GroupedPage[K, V, F[_]: MonadError]( - items: ListMap[K, Vector[V]], - hasNext: Boolean, - limit: Int, - pageStart: Long, - pageEnd: Long, - private[reader] val pageRoller: GroupPageRoller[K, V, F] - ) extends AbstractPage[Map[K, Vector[V]], F] { - - def apply(key: K): Vector[V] = items(key) - def keys: Iterable[K] = items.keys - def groupCount: Int = items.size - - def map[K1, V1](f: ((K, Vector[V])) => (K1, Vector[V1])): GroupedPage[K1, V1, F] = { - val newItems = items.map(f) - val newPageRoller: GroupPageRoller[K1, V1, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.map(f))) - this.copy(items = newItems, pageRoller = newPageRoller) - } - - def mapValues[B](f: V => B): GroupedPage[K, B, F] = { - def mapper(item: (K, Vector[V])): (K, Vector[B]) = (item._1, item._2.map(f)) - - val newItems = items.map(mapper) - val newPageRoller: GroupPageRoller[K, B, F] = (limit, offset) => pageRoller(limit, offset).map(_.map(_.mapValues(f))) - this.copy(items = newItems, pageRoller = newPageRoller) - - } - - def prior(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { - if (hasPrior) { - val newOffset = (pageStart - limit).max(0) - pageRoller(newPageSize, newOffset) - } else { - MonadError[F].unit(RequestFail(NoDataException("No prior page"))) - } - } - - def prior: F[RequestResult[GroupedPage[K, V, F]]] = prior(limit) - - def next(newPageSize: Int): F[RequestResult[GroupedPage[K, V, F]]] = { - if (hasNext) { - pageRoller(newPageSize, pageStart + limit) - } else { - MonadError[F].unit(RequestFail(NoDataException("No next page"))) - } - } - - def next: F[RequestResult[GroupedPage[K, V, F]]] = next(limit) - - def +(other: GroupedPage[K, V, F]): GroupedPage[K, V, F] = { - val newItems = other.items.foldLeft(items) { case (acc, (k, v)) => - if (acc.contains(k)) { - acc.updated(k, acc(k) ++ v) - } else { - acc + (k -> v) - } - } - val newHasNext = hasNext && other.hasNext - val newPageStart = pageStart min other.pageStart - val newPageEnd = pageEnd max other.pageEnd - this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd) - } -} - -object GroupedPage { - type GroupPageRoller[K, V, F[_]] = (Int, Long) => F[RequestResult[GroupedPage[K, V, F]]] -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala b/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala deleted file mode 100644 index 6d2705f59..000000000 --- a/reader/src/main/scala/za/co/absa/atum/reader/result/Page.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.result - -import sttp.monad.MonadError -import sttp.monad.syntax._ -import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestPageResultOps, RequestResult} -import za.co.absa.atum.reader.exceptions.RequestException.NoDataException -import za.co.absa.atum.reader.result.GroupedPage.GroupPageRoller -import za.co.absa.atum.reader.result.Page.PageRoller - -import scala.collection.immutable.ListMap - -case class Page[T, F[_]: MonadError]( - items: Vector[T], - hasNext: Boolean, - limit: Int, - pageStart: Long, - pageEnd: Long, - private[reader] val pageRoller: PageRoller[T, F] - ) extends AbstractPage[Vector[T], F] { - - def apply(index: Int): T = items(index) - - def map[B](f: T => B): Page[B, F] = { - val newItems = items.map(f) - val newPageRoller: PageRoller[B, F] = (limit, offset) => pageRoller(limit, offset).map(_.pageMap(f)) - this.copy(items = newItems, pageRoller = newPageRoller) - } - - def prior(newPageSize: Int): F[RequestResult[Page[T, F]]] = { - if (hasPrior) { - val newOffset = (pageStart - newPageSize).max(0) - pageRoller(newPageSize, newOffset) - } else { - MonadError[F].unit(RequestFail(NoDataException("No prior page"))) - } - } - - def prior(): F[RequestResult[Page[T, F]]] = prior(limit) - - def next(newPageSize: Int): F[RequestResult[Page[T, F]]] = { - if (hasNext) { - pageRoller(newPageSize, pageStart + pageSize) - } else { - MonadError[F].unit(RequestFail(NoDataException("No next page"))) - } - } - - def next: F[RequestResult[Page[T, F]]] = next(limit) - - def +(other: Page[T, F]): Page[T, F] = { - val newItems = items ++ other.items - val newPageStart = pageStart min other.pageStart - val newPageEnd = pageEnd max other.pageEnd - val newHasNext = hasNext && other.hasNext - this.copy(items = newItems, hasNext = newHasNext, pageStart = newPageStart, pageEnd = newPageEnd) - } - - def groupBy[K](f: T => K): GroupedPage[K, T, F] = { - val (newItems, itemsCounts) = items.foldLeft(ListMap.empty[K, Vector[T]], 0) { case ((groupsAcc, count), item) => - val k = f(item) - (groupsAcc.updated(k, groupsAcc.getOrElse(k, Vector.empty) :+ item), count + 1) - } - val newPageRoller: GroupPageRoller[K, T, F] = (limit, offset) => - pageRoller(limit, offset) - .map(_.map(_.groupBy(f))) - - GroupedPage( - newItems, - hasNext, - limit, - pageStart, - pageEnd, - newPageRoller - ) - } -} - -object Page { - type PageRoller[T, F[_]] = (Int, Long) => F[RequestResult[Page[T, F]]] -} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index 81769adce..e05be7820 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -26,13 +26,14 @@ import sttp.monad.MonadError import za.co.absa.atum.model.ResultValueType import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningWithIdDTO} +import za.co.absa.atum.model.envelopes.Pagination +import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.types.Measurement.LongMeasurement import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.reader.FlowReaderUnitTests._ import za.co.absa.atum.reader.server.ServerConfig import za.co.absa.atum.reader.implicits.future.futureMonadError -import za.co.absa.atum.testing.implicits.RequestResultImplicits.RequestResultPageEnhancements import java.time.ZonedDateTime import java.util.UUID @@ -71,74 +72,9 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { "a" -> "b", "c" -> "d" )) - val expectedData = Vector( - CheckpointWithPartitioningDTO( - id = UUID.fromString("51ee4257-0842-4d28-8779-8ecb19ae7bf0"), - name = "Test checkpoints 1", - author = "Jason Bourne", - measuredByAtumAgent = true, - processStartTime = ZonedDateTime.parse("2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]"), - processEndTime = Some(ZonedDateTime.parse("2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]")), - measurements = Set( - MeasurementDTO( - measure = MeasureDTO( - measureName = "Fictional", - measuredColumns = Seq("x", "y", "z") - ), - result = MeasureResultDTO( - mainValue = TypedValue("1", ResultValueType.LongValue), - ) - ) - ), - partitioning = PartitioningWithIdDTO( - id = 7, - atumPartitions.toPartitioningDTO, - author = "James Bond" - ) - ), - CheckpointWithPartitioningDTO( - id = UUID.fromString("8b7f603e-3fc3-474f-aced-a7af054589a2"), - name = "Test checkpoints 2", - author = "John McClane", - measuredByAtumAgent = true, - processStartTime = ZonedDateTime.parse("2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]"), - processEndTime = None, - measurements = Set(), - partitioning = PartitioningWithIdDTO( - id = 7, - atumPartitions.toPartitioningDTO, - author = "James Bond" - ) - ) - ) - - val reader = new FlowReader(atumPartitions) - val result = reader.getCheckpointDTOs(None) - result.assertPage(expectedData, hasNext = false, 10, 0, 1) - } - - test("The flow checkpoints are properly queried and delivered as AtumPartitionsCheckpoint instances") { - implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous - .whenRequestMatchesPartial { - case r if r.uri.path.endsWith(List("partitionings")) => - assert(r.uri.querySegments.contains(KeyValue("partitioning", partitioningEncoded))) - Response.ok(partitioningResponse) - case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => - Response.ok(flowResponse) - case r if r.uri.path.endsWith(List("checkpoints")) => - assert(r.uri.querySegments.contains(KeyValue("offset", "3"))) - assert(r.uri.querySegments.contains(KeyValue("limit", "11"))) - Response.ok(checkpointsResponse) - } - - val atumPartitions: AtumPartitions = AtumPartitions(List( - "a" -> "b", - "c" -> "d" - )) - val expectedData = Vector( - AtumPartitionsCheckpoint( - partitioning = atumPartitions, - checkpoint = Checkpoint( + val expectedData: PaginatedResponse[CheckpointWithPartitioningDTO] = PaginatedResponse( + data = Seq( + CheckpointWithPartitioningDTO( id = UUID.fromString("51ee4257-0842-4d28-8779-8ecb19ae7bf0"), name = "Test checkpoints 1", author = "Jason Bourne", @@ -146,35 +82,51 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { processStartTime = ZonedDateTime.parse("2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]"), processEndTime = Some(ZonedDateTime.parse("2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]")), measurements = Set( - LongMeasurement( - - measureName = "Fictional", - measuredColumns = Seq("x", "y", "z"), - value = 1 + MeasurementDTO( + measure = MeasureDTO( + measureName = "Fictional", + measuredColumns = Seq("x", "y", "z") + ), + result = MeasureResultDTO( + mainValue = TypedValue("1", ResultValueType.LongValue), + ) ) + ), + partitioning = PartitioningWithIdDTO( + id = 7, + atumPartitions.toPartitioningDTO, + author = "James Bond" ) - ) - ), - AtumPartitionsCheckpoint( - partitioning = atumPartitions, - checkpoint = Checkpoint( + ), + CheckpointWithPartitioningDTO( id = UUID.fromString("8b7f603e-3fc3-474f-aced-a7af054589a2"), name = "Test checkpoints 2", author = "John McClane", measuredByAtumAgent = true, processStartTime = ZonedDateTime.parse("2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]"), processEndTime = None, - measurements = Set() - ) + measurements = Set(), + partitioning = PartitioningWithIdDTO( + id = 7, + atumPartitions.toPartitioningDTO, + author = "James Bond" + ) ) + ), + pagination = Pagination( + limit = 10, + offset = 0, + hasMore = false + ), + requestId = UUID.fromString("29ce91a7-b668-41d2-a160-26402551fb0b") ) val reader = new FlowReader(atumPartitions) - val result = reader.getCheckpoints(11, 3) - result.assertPage(expectedData, hasNext = false, 10, 0, 1) + val result = reader.getCheckpointsPage() + assert(result == Right(expectedData)) } - test("The flow checkpoints of certain name are properly queried and delivered as AtumPartitionsCheckpoint instances") { + test("The flow checkpoints are properly queried with name and delivered as DTO") { implicit val server: SttpBackendStub[Identity, capabilities.WebSockets] = SttpBackendStub.synchronous .whenRequestMatchesPartial { case r if r.uri.path.endsWith(List("partitionings")) => @@ -183,9 +135,9 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { case r if r.uri.path.endsWith(List("partitionings", "7", "main-flow")) => Response.ok(flowResponse) case r if r.uri.path.endsWith(List("checkpoints")) => - assert(r.uri.querySegments.contains(KeyValue("offset", "3"))) - assert(r.uri.querySegments.contains(KeyValue("limit", "11"))) - assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Foo"))) + assert(r.uri.querySegments.contains(KeyValue("offset", "0"))) + assert(r.uri.querySegments.contains(KeyValue("limit", "10"))) + assert(r.uri.querySegments.contains(KeyValue("checkpoint-name", "Test checkpoints 1"))) Response.ok(checkpointsResponse) } @@ -193,10 +145,9 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { "a" -> "b", "c" -> "d" )) - val expectedData = Vector( - AtumPartitionsCheckpoint( - partitioning = atumPartitions, - checkpoint = Checkpoint( + val expectedData: PaginatedResponse[CheckpointWithPartitioningDTO] = PaginatedResponse( + data = Seq( + CheckpointWithPartitioningDTO( id = UUID.fromString("51ee4257-0842-4d28-8779-8ecb19ae7bf0"), name = "Test checkpoints 1", author = "Jason Bourne", @@ -204,32 +155,48 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { processStartTime = ZonedDateTime.parse("2024-12-30T16:01:36.5042011+01:00[Europe/Budapest]"), processEndTime = Some(ZonedDateTime.parse("2024-12-30T16:01:36.5052109+01:00[Europe/Budapest]")), measurements = Set( - LongMeasurement( - - measureName = "Fictional", - measuredColumns = Seq("x", "y", "z"), - value = 1 + MeasurementDTO( + measure = MeasureDTO( + measureName = "Fictional", + measuredColumns = Seq("x", "y", "z") + ), + result = MeasureResultDTO( + mainValue = TypedValue("1", ResultValueType.LongValue), + ) ) + ), + partitioning = PartitioningWithIdDTO( + id = 7, + atumPartitions.toPartitioningDTO, + author = "James Bond" ) - ) - ), - AtumPartitionsCheckpoint( - partitioning = atumPartitions, - checkpoint = Checkpoint( + ), + CheckpointWithPartitioningDTO( id = UUID.fromString("8b7f603e-3fc3-474f-aced-a7af054589a2"), name = "Test checkpoints 2", author = "John McClane", measuredByAtumAgent = true, processStartTime = ZonedDateTime.parse("2024-12-30T16:02:36.5042011+01:00[Europe/Budapest]"), processEndTime = None, - measurements = Set() + measurements = Set(), + partitioning = PartitioningWithIdDTO( + id = 7, + atumPartitions.toPartitioningDTO, + author = "James Bond" + ) ) - ) + ), + pagination = Pagination( + limit = 10, + offset = 0, + hasMore = false + ), + requestId = UUID.fromString("29ce91a7-b668-41d2-a160-26402551fb0b") ) val reader = new FlowReader(atumPartitions) - val result = reader.getCheckpointsOfName("Foo", 11, 3) - result.assertPage(expectedData, hasNext = false, 10, 0, 1) + val result = reader.getCheckpointsOfNamePage("Test checkpoints 1") + assert(result == Right(expectedData)) } } diff --git a/reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala b/reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala deleted file mode 100644 index 96c6b8b3b..000000000 --- a/reader/src/test/scala/za/co/absa/atum/reader/implicits/PaginatedResponseImplicitsTest.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2024 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.implicits - -import org.scalatest.funsuite.AnyFunSuiteLike -import sttp.client3.Identity -import sttp.client3.monad.IdMonad -import sttp.monad.MonadError -import za.co.absa.atum.model.envelopes.Pagination -import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse -import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestResult} -import za.co.absa.atum.reader.exceptions.RequestException -import za.co.absa.atum.reader.implicits.PaginatedResponseImplicits.PaginatedResponseMonadEnhancements -import za.co.absa.atum.reader.result.Page - -class PaginatedResponseImplicitsTest extends AnyFunSuiteLike { - private implicit val monad: MonadError[Identity] = IdMonad - - test("toPage should convert PaginatedResponse to Page") { - def roll(pageSize: Int, offset: Long): Identity[RequestResult[Page[Int, Identity]]] = { - IdMonad.unit(RequestFail(new RequestException("Not used"){})) - } - - val source = PaginatedResponse(Seq(1, 2, 3), - pagination = Pagination(offset = 1, limit = 3, hasMore = false) - ) - val result = source.toPage(roll) - assert(result.items == Vector(1, 2, 3)) - assert(!result.hasNext) - assert(result.limit == 3) - assert(result.pageStart == 1) - } -} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala deleted file mode 100644 index 94623783f..000000000 --- a/reader/src/test/scala/za/co/absa/atum/reader/result/AbstractPageUnitTests.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.result - -import org.scalatest.funsuite.AnyFunSuiteLike -import sttp.client3.Identity -import sttp.client3.monad.IdMonad -import sttp.monad.MonadError - - -class AbstractPageUnitTests extends AnyFunSuiteLike { - private implicit val monad: MonadError[Identity] = IdMonad - - test("Basic test") { - val page = new AbstractPage[Iterable[Int], Identity] { - override def items: Iterable[Int] = Seq(1, 2, 3) - override def hasNext: Boolean = true - override def limit: Int = 3 - override def pageStart: Long = 0 - override def pageEnd: Long = 2 - } - - assert(page.items.size == 3) - assert(page.hasNext) - assert(page.limit == 3) - assert(page.pageStart == 0) - assert(page.pageSize == 3) - assert(!page.hasPrior) - - val anotherPage = new AbstractPage[Iterable[Int], Identity] { - override def items: Iterable[Int] = Seq(1, 2, 3) - override def hasNext: Boolean = true - override def limit: Int = 3 - override def pageStart: Long = 1 - override def pageEnd: Long = 2 - } - assert(anotherPage.hasPrior) - } -} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala deleted file mode 100644 index 5c79f9e79..000000000 --- a/reader/src/test/scala/za/co/absa/atum/reader/result/GroupedPageUnitTests.scala +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2025 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.result - -import org.scalatest.funsuite.AnyFunSuiteLike -import za.co.absa.atum.reader.core.RequestResult.RequestFail -import za.co.absa.atum.reader.exceptions.RequestException.NoDataException -import za.co.absa.atum.testing.data.PageData -import za.co.absa.atum.testing.implicits.AbstractPageImplicits.GroupedPageEnhancements - -import scala.collection.immutable.ListMap - - -class GroupedPageUnitTests extends AnyFunSuiteLike { - private val initialPage = { - val Right(ungrouped) = PageData.roller(6, 0) - ungrouped.groupBy(_.group) - } - - test("Getting items, group keys, and group count") { - assert(initialPage(3) == Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f"))) - assert(initialPage.groupCount == 3) - assert(initialPage.keys.toList == List(1, 2, 3)) - } - - test("Mapping of the page") { - val mappedPage1 = initialPage.map(x => (x._1.toString, x._2.map(_.value.toUpperCase))) - - mappedPage1.assertGroupedPage( - ListMap( - "1" -> Vector("A", "B"), - "2" -> Vector("C", "D"), - "3" -> Vector("E", "F") - ), - hasNext = true, - limit = 6, - pageStart = 0, - pageEnd = 5 - ) - - val Right(mappedPage2) = mappedPage1.next - mappedPage2.assertGroupedPage( - ListMap( - "3" -> Vector("AA"), - "4" -> Vector("BB", "CC"), - "5" -> Vector("DD") - ), - hasNext = false, - limit = 6, - pageStart = 6, - pageEnd = 9 - ) - } - - test("Mapping of the page values") { - val mappedPage1 = initialPage.mapValues(_.value.toUpperCase) - - mappedPage1.assertGroupedPage( - ListMap( - 1 -> Vector("A", "B"), - 2 -> Vector("C", "D"), - 3 -> Vector("E", "F") - ), - hasNext = true, - limit = 6, - pageStart = 0, - pageEnd = 5 - ) - - val Right(mappedPage2) = mappedPage1.next - mappedPage2.assertGroupedPage( - ListMap( - 3 -> Vector("AA"), - 4 -> Vector("BB", "CC"), - 5 -> Vector("DD") - ), - hasNext = false, - limit = 6, - pageStart = 6, - pageEnd = 9 - ) - } - - test("Rolling of the pages") { - val Right(nextPage1) = initialPage.next - nextPage1.assertGroupedPage( - ListMap( - 3 -> Vector(PageData.TestItem(3, "aa")), - 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), - 5 -> Vector(PageData.TestItem(5, "dd")) - ), - hasNext = false, - limit = 6, - pageStart = 6, - pageEnd = 9 - ) - assert(nextPage1.hasPrior) - - val noPage1 = nextPage1.next - assert(noPage1 == RequestFail(NoDataException("No next page"))) - - val Right(nextPage2) = initialPage.next(17) - nextPage2.assertGroupedPage( - ListMap( - 3 -> Vector(PageData.TestItem(3, "aa")), - 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), - 5 -> Vector(PageData.TestItem(5, "dd")) - ), - hasNext = false, - limit = 17, - pageStart = 6, - pageEnd = 9 - ) - - val Right(priorPage1) = nextPage2.prior - priorPage1.assertGroupedPage( - ListMap( - 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), - 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), - 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f")) - ), - hasNext = true, - limit = 17, - pageStart = 0, - pageEnd = 5 - ) - - val Right(priorPage2) = nextPage2.prior(7) - priorPage2.assertGroupedPage( - ListMap( - 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), - 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), - 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f")) - ), - hasNext = true, - limit = 7, - pageStart = 0, - pageEnd = 5 - ) - - assert(!priorPage2.hasPrior) - - val noPage2 = priorPage2.prior - assert(noPage2 == RequestFail(NoDataException("No prior page"))) - } - - test("Concatenation of pages") { - val Right(nextPage1) = initialPage.next - val concatenatedPage = initialPage + nextPage1 - concatenatedPage.assertGroupedPage( - ListMap( - 1 -> Vector(PageData.TestItem(1, "a"), PageData.TestItem(1, "b")), - 2 -> Vector(PageData.TestItem(2, "c"), PageData.TestItem(2, "d")), - 3 -> Vector(PageData.TestItem(3, "e"), PageData.TestItem(3, "f"), PageData.TestItem(3, "aa")), - 4 -> Vector(PageData.TestItem(4, "bb"), PageData.TestItem(4, "cc")), - 5 -> Vector(PageData.TestItem(5, "dd")) - ), - hasNext = false, - limit = 6, - pageStart = 0, - pageEnd = 9 - ) - } - -} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala deleted file mode 100644 index 3731bf8be..000000000 --- a/reader/src/test/scala/za/co/absa/atum/reader/result/PageUnitTests.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2025 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.reader.result - -import org.scalatest.funsuite.AnyFunSuiteLike -import za.co.absa.atum.reader.core.RequestResult.{RequestFail, RequestOK, RequestResult} -import za.co.absa.atum.reader.exceptions.RequestException.NoDataException -import za.co.absa.atum.testing.data.PageData -import za.co.absa.atum.testing.data.PageData.TestItem -import za.co.absa.atum.testing.implicits.AbstractPageImplicits.PageEnhancements - -class PageUnitTests extends AnyFunSuiteLike { - - test("Get the items of a page") { - val Right(page) = PageData.roller(6, 0) - - assert(page(0) == TestItem(1, "a")) - assert(page(5) == TestItem(3, "f")) - } - - test("Mapping the page items") { - val Right(origPage) = PageData.roller(6, 0) - - val mapped = origPage.map(_.value.toUpperCase) - - assert(mapped.hasNext == origPage.hasNext) - assert(mapped.limit == origPage.limit) - assert(mapped.pageStart == origPage.pageStart) - assert(mapped.pageEnd == origPage.pageEnd) - assert(mapped.items == Vector("A", "B", "C", "D", "E", "F")) - - val Right(nextPage) = mapped.next - assert(nextPage.items == Vector("AA", "BB", "DD", "CC")) - } - - test("Rolling of the pages") { - val Right(page1) = PageData.roller(6, 0) - val Right(page2) = page1.next - - page2.assertPage(PageData.items2, hasNext = false, limit = 6, pageStart = 6, pageEnd = 9) - val noPage1 = page2.next - assert(noPage1 == RequestFail(NoDataException("No next page"))) - - val Right(page3) = page1.next(100) - page3.assertPage(PageData.items2, hasNext = false, limit = 100, pageStart = 6, pageEnd = 9) - - val Right(page4) = page2.prior(42) - page4.assertPage(PageData.items1, hasNext = true, limit = 42, pageStart = 0, pageEnd = 5) - assert(!page4.hasPrior) - - val noPage2 = page4.prior() - assert(noPage2 == RequestFail(NoDataException("No prior page"))) - } - - test("Concatenation of pages") { - val Right(page1) = PageData.roller(6, 0) - val Right(page2) = page1.next - - val page3 = page1 + page2 - page3.assertPage(PageData.items1 ++ PageData.items2, hasNext = false, limit = 6, pageStart = 0, pageEnd = 9) - } - - test("Grouping of the page items") { - val Right(page1) = PageData.roller(6, 0) - val grouped = page1.groupBy(_.group) - - assert(grouped.groupCount == 3) - assert(grouped.keys.toList == List(1, 2, 3)) - - assert(grouped(1) == Vector(TestItem(1, "a"), TestItem(1, "b"))) - assert(grouped(2) == Vector(TestItem(2, "c"), TestItem(2, "d"))) - assert(grouped(3) == Vector(TestItem(3, "e"), TestItem(3, "f"))) - } -} - diff --git a/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala b/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala deleted file mode 100644 index 180753da4..000000000 --- a/reader/src/test/scala/za/co/absa/atum/testing/data/PageData.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2025 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.testing.data - -import sttp.client3.Identity -import sttp.client3.monad.IdMonad -import sttp.monad.MonadError -import za.co.absa.atum.reader.core.RequestResult.{RequestOK, RequestResult} -import za.co.absa.atum.reader.result.Page - -object PageData { - private implicit val monad: MonadError[Identity] = IdMonad - - case class TestItem(group: Int, value: String) - - val items1: Vector[TestItem] = Vector( - TestItem(1, "a"), - TestItem(1, "b"), - TestItem(2, "c"), - TestItem(2, "d"), - TestItem(3, "e"), - TestItem(3, "f") - ) - - val items2: Vector[TestItem] = Vector( - TestItem(3, "aa"), - TestItem(4, "bb"), - TestItem(5, "dd"), - TestItem(4, "cc") - ) - - def roller(limit: Int, offset: Long): RequestResult[Page[TestItem, Identity]] = { - offset match { - case 0 => - RequestOK( - Page[TestItem, Identity]( - items = items1, - hasNext = true, - limit = limit, - pageStart = 0, - pageEnd = 5, - roller - ) - ) - case 6 => RequestOK( - Page[TestItem, Identity]( - items = items2, - hasNext = false, - limit = limit, - pageStart = 6, - pageEnd = 9, - roller - ) - ) - - } - } -} diff --git a/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala b/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala deleted file mode 100644 index 2d8b0b4fe..000000000 --- a/reader/src/test/scala/za/co/absa/atum/testing/implicits/AbstractPageImplicits.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2025 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.testing.implicits - -import za.co.absa.atum.reader.result.{AbstractPage, GroupedPage, Page} - -import scala.collection.immutable.ListMap - -object AbstractPageImplicits { - implicit class PageEnhancements[T, F[_]](val page: Page[T, F]) extends AnyVal { - def assertPage(items: Vector[T], - hasNext: Boolean, - limit: Int, - pageStart: Long, - pageEnd: Long): Unit = { - assert(items == page.items, s"Expected items: $items, but got: ${page.items}") - assert(hasNext == page.hasNext, s"Expected hasNext: $hasNext, but got: ${page.hasNext}") - assert(limit == page.limit, s"Expected limit: $limit, but got: ${page.limit}") - assert(pageStart == page.pageStart, s"Expected pageStart: $pageStart, but got: ${page.pageStart}") - assert(pageEnd == page.pageEnd, s"Expected pageEnd: $pageEnd, but got: ${page.pageEnd}") - } - } - - implicit class GroupedPageEnhancements[K, V, F[_]](val page: GroupedPage[K, V, F]) extends AnyVal { - def assertGroupedPage(items: ListMap[K, Vector[V]], - hasNext: Boolean, - limit: Int, - pageStart: Long, - pageEnd: Long): Unit = { - assert(items == page.items, s"Expected items: $items, but got: ${page.items}") - assert(hasNext == page.hasNext, s"Expected hasNext: $hasNext, but got: ${page.hasNext}") - assert(limit == page.limit, s"Expected limit: $limit, but got: ${page.limit}") - assert(pageStart == page.pageStart, s"Expected pageStart: $pageStart, but got: ${page.pageStart}") - assert(pageEnd == page.pageEnd, s"Expected pageEnd: $pageEnd, but got: ${page.pageEnd}") - } - } -} diff --git a/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala b/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala deleted file mode 100644 index 16ca2c37b..000000000 --- a/reader/src/test/scala/za/co/absa/atum/testing/implicits/RequestResultImplicits.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2025 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.testing.implicits - -import za.co.absa.atum.reader.core.RequestResult.RequestResult -import za.co.absa.atum.reader.result.Page -import za.co.absa.atum.testing.implicits.AbstractPageImplicits.PageEnhancements - -object RequestResultImplicits { - implicit class RequestResultPageEnhancements[T, F[_]](val pageResult: RequestResult[Page[T, F]]) extends AnyVal { - def assertPage(items: Vector[T], - hasNext: Boolean, - limit: Int, - pageStart: Long, - pageEnd: Long): Unit = { - pageResult match { - case Right(page) => page.assertPage(items, hasNext, limit, pageStart, pageEnd) - case _ => throw new AssertionError("Expected a page result") - } - } - } - -} From bbed1f054e8b8920c5a4366ddc783201bc819dfd Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Feb 2025 12:35:56 +0100 Subject: [PATCH 48/52] * added adr --- ...f-FlowReader-and-PartitioningReader.drawio | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 adrs/01_Basics-of-FlowReader-and-PartitioningReader.drawio diff --git a/adrs/01_Basics-of-FlowReader-and-PartitioningReader.drawio b/adrs/01_Basics-of-FlowReader-and-PartitioningReader.drawio new file mode 100644 index 000000000..6dbcf05db --- /dev/null +++ b/adrs/01_Basics-of-FlowReader-and-PartitioningReader.drawio @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 15dd12a308a14fa3e3d9601cd58ffbe86df5bad7 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Feb 2025 12:50:40 +0100 Subject: [PATCH 49/52] * removing unnecessary changes --- .../absa/atum/model/dto/CheckpointV2DTO.scala | 5 +- .../dto/CheckpointWithPartitioningDTO.scala | 3 +- .../model/dto/traits/CheckpointCore.scala | 32 ------------- .../types/AtumPartitionsCheckpoint.scala | 24 ---------- .../co/absa/atum/model/types/Checkpoint.scala | 46 ------------------- .../atum/reader/FlowReaderUnitTests.scala | 2 - 6 files changed, 3 insertions(+), 109 deletions(-) delete mode 100644 model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala delete mode 100644 model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala delete mode 100644 model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala index 0b6fb7037..ad80b373e 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointV2DTO.scala @@ -18,12 +18,11 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} -import za.co.absa.atum.model.dto.traits.CheckpointCore import java.time.ZonedDateTime import java.util.UUID -case class CheckpointV2DTO ( +case class CheckpointV2DTO( id: UUID, name: String, author: String, @@ -31,7 +30,7 @@ case class CheckpointV2DTO ( processStartTime: ZonedDateTime, processEndTime: Option[ZonedDateTime], measurements: Set[MeasurementDTO] -) extends CheckpointCore +) object CheckpointV2DTO { implicit val decodeCheckpointDTO: Decoder[CheckpointV2DTO] = deriveDecoder diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala index e89954a74..d72263201 100644 --- a/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala +++ b/model/src/main/scala/za/co/absa/atum/model/dto/CheckpointWithPartitioningDTO.scala @@ -18,7 +18,6 @@ package za.co.absa.atum.model.dto import io.circe.{Decoder, Encoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} -import za.co.absa.atum.model.dto.traits.CheckpointCore import java.time.ZonedDateTime import java.util.UUID @@ -32,7 +31,7 @@ case class CheckpointWithPartitioningDTO( processEndTime: Option[ZonedDateTime], measurements: Set[MeasurementDTO], partitioning: PartitioningWithIdDTO -) extends CheckpointCore +) object CheckpointWithPartitioningDTO { diff --git a/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala b/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala deleted file mode 100644 index 12494b1b8..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/dto/traits/CheckpointCore.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.dto.traits - -import za.co.absa.atum.model.dto.MeasurementDTO - -import java.time.ZonedDateTime -import java.util.UUID - -trait CheckpointCore { - def id: UUID - def name: String - def author: String - def measuredByAtumAgent: Boolean - def processStartTime: ZonedDateTime - def processEndTime: Option[ZonedDateTime] - def measurements: Set[MeasurementDTO] -} diff --git a/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala deleted file mode 100644 index 3990511ab..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/types/AtumPartitionsCheckpoint.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.types - -import za.co.absa.atum.model.types.basic.AtumPartitions - -case class AtumPartitionsCheckpoint( - partitioning: AtumPartitions, - checkpoint: Checkpoint - ) diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala b/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala deleted file mode 100644 index 14af96e41..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/types/Checkpoint.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.types - -import za.co.absa.atum.model.dto.traits.CheckpointCore - -import java.time.ZonedDateTime -import java.util.UUID - -case class Checkpoint ( - id: UUID, - name: String, - author: String, - measuredByAtumAgent: Boolean = false, - processStartTime: ZonedDateTime, - processEndTime: Option[ZonedDateTime], - measurements: Set[Measurement] - ) - -object Checkpoint { - def apply(from: CheckpointCore): Checkpoint = { - new Checkpoint( - id = from.id, - name = from.name, - author = from.author, - measuredByAtumAgent = from.measuredByAtumAgent, - processStartTime = from.processStartTime, - processEndTime = from.processEndTime, - measurements = from.measurements.map(Measurement(_)) - ) - } -} diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index e05be7820..560c6b070 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -28,8 +28,6 @@ import za.co.absa.atum.model.dto.MeasureResultDTO.TypedValue import za.co.absa.atum.model.dto.{CheckpointWithPartitioningDTO, MeasureDTO, MeasureResultDTO, MeasurementDTO, PartitioningWithIdDTO} import za.co.absa.atum.model.envelopes.Pagination import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse -import za.co.absa.atum.model.types.Measurement.LongMeasurement -import za.co.absa.atum.model.types.{AtumPartitionsCheckpoint, Checkpoint} import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.reader.FlowReaderUnitTests._ import za.co.absa.atum.reader.server.ServerConfig From c630fa06c484504c81fa32286252371286fc2998 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Feb 2025 15:57:14 +0100 Subject: [PATCH 50/52] * removed unnecessary changes II * addressed PR comments --- .../absa/atum/model/types/Measurement.scala | 82 ------------------- .../za/co/absa/atum/reader/FlowReader.scala | 17 ++-- .../reader/exceptions/RequestException.scala | 4 - .../reader/requests/QueryParamNames.scala | 23 ++++++ 4 files changed, 32 insertions(+), 94 deletions(-) delete mode 100644 model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala create mode 100644 reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala diff --git a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala b/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala deleted file mode 100644 index c8245e2b5..000000000 --- a/model/src/main/scala/za/co/absa/atum/model/types/Measurement.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.model.types - -import za.co.absa.atum.model.ResultValueType -import za.co.absa.atum.model.dto.MeasurementDTO - -trait Measurement { - type T - def measureName: String - def measuredColumns: Seq[String] - def valueType: ResultValueType - def value: T - def stringValue: String -} - -object Measurement { - - def apply[T](from: MeasurementDTO): Measurement = { - from.result.mainValue.valueType match { - case ResultValueType.StringValue => StringMeasurement(from.measure.measureName, from.measure.measuredColumns, from.result.mainValue.value) - case ResultValueType.LongValue => LongMeasurement(from.measure.measureName, from.measure.measuredColumns, from.result.mainValue.value.toLong) - case ResultValueType.BigDecimalValue => BigDecimalMeasurement(from.measure.measureName, from.measure.measuredColumns, BigDecimal(from.result.mainValue.value)) - case ResultValueType.DoubleValue => DoubleMeasurement(from.measure.measureName, from.measure.measuredColumns, from.result.mainValue.value.toDouble) - } - } - - case class StringMeasurement( - measureName: String, - measuredColumns: Seq[String], - value: String - ) extends Measurement { - override type T = String - override def valueType: ResultValueType = ResultValueType.StringValue - override def stringValue: String = value - } - - case class LongMeasurement( - measureName: String, - measuredColumns: Seq[String], - value: Long - ) extends Measurement { - override type T = Long - override def valueType: ResultValueType = ResultValueType.LongValue - override def stringValue: String = value.toString - } - - case class BigDecimalMeasurement( - measureName: String, - measuredColumns: Seq[String], - value: BigDecimal - ) extends Measurement { - override type T = BigDecimal - override def valueType: ResultValueType = ResultValueType.BigDecimalValue - override def stringValue: String = value.toString - } - - case class DoubleMeasurement( - measureName: String, - measuredColumns: Seq[String], - value: Double - ) extends Measurement { - override type T = Double - override def valueType: ResultValueType = ResultValueType.DoubleValue - override def stringValue: String = value.toString - } - -} diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 84155f6d8..3119bfb40 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -26,6 +26,7 @@ import za.co.absa.atum.reader.core.RequestResult.RequestResult import za.co.absa.atum.model.ApiPaths._ import za.co.absa.atum.reader.core.{PartitioningIdProvider, Reader} import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements +import za.co.absa.atum.reader.requests.QueryParamNames import za.co.absa.atum.reader.server.ServerConfig /** @@ -51,22 +52,22 @@ class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) private def queryCheckpoints(flowId: Long, checkpointName: Option[String], - pageSize: Int, + limit: Int, offset: Long): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { val endpoint = s"/$Api/$V2/${V2Paths.Flows}/$flowId/${V2Paths.Checkpoints}" val params = Map( - "limit" -> pageSize.toString, - "offset" -> offset.toString - ) ++ checkpointName.map("checkpoint-name" -> _) + QueryParamNames.limit -> limit.toString, + QueryParamNames.offset -> offset.toString + ) ++ checkpointName.map(QueryParamNames.checkpointName -> _) getQuery(endpoint, params) } def getCheckpointsPage(pageSize: Int = 10, offset: Long = 0): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { for { - mainPartitioningId <- partitioningId(mainFlowPartitioning) - flowId <- mainPartitioningId.project(queryFlowId) - checkpoints <- flowId.project(queryCheckpoints(_, None, pageSize, offset)) - } yield checkpoints + mainPartitioningIdOrErrror <- partitioningId(mainFlowPartitioning) + flowIdOrError <- mainPartitioningIdOrErrror.project(queryFlowId) + checkpointsOrError <- flowIdOrError.project(queryCheckpoints(_, None, pageSize, offset)) + } yield checkpointsOrError } def getCheckpointsOfNamePage(checkpointName: String, pageSize: Int = 10, offset: Long = 0): F[RequestResult[PaginatedResponse[CheckpointWithPartitioningDTO]]] = { diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala index 5cc449165..4f713eb55 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -42,8 +42,4 @@ object RequestException { } } - - final case class NoDataException( - message: String - ) extends RequestException(message) } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala new file mode 100644 index 000000000..5376b1e0e --- /dev/null +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala @@ -0,0 +1,23 @@ +/* + * Copyright 2025 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.reader.requests + +object QueryParamNames { + val limit = "limit" + val offset = "offset" + val checkpointName = "checkpoint-name" +} From f4ce4d318ed1129e65d4dbbd63242c1c41146c77 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Feb 2025 16:33:19 +0100 Subject: [PATCH 51/52] * Typo fixes --- .../scala/za/co/absa/atum/reader/requests/QueryParamNames.scala | 2 +- ...ImplicitsUnitsTests.scala => EitherImplicitsUnitTests.scala} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename reader/src/test/scala/za/co/absa/atum/reader/implicits/{EitherImplicitsUnitsTests.scala => EitherImplicitsUnitTests.scala} (96%) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala index 5376b1e0e..42bc2d8b5 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/requests/QueryParamNames.scala @@ -1,5 +1,5 @@ /* - * Copyright 2025 ABSA Group Limited + * Copyright 2021 ABSA Group Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitTests.scala similarity index 96% rename from reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala rename to reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitTests.scala index 65468ba90..f27d99217 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitsTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/implicits/EitherImplicitsUnitTests.scala @@ -22,7 +22,7 @@ import sttp.client3.monad.IdMonad import sttp.monad.MonadError import za.co.absa.atum.reader.implicits.EitherImplicits.EitherMonadEnhancements -class EitherImplicitsUnitsTests extends AnyFunSuiteLike { +class EitherImplicitsUnitTests extends AnyFunSuiteLike { private implicit val monad: MonadError[Identity] = IdMonad test("EitherMonadEnhancements should project Right") { From 56cfa092a404b5a4b910584bc5decb2c66afd751 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Feb 2025 18:55:51 +0100 Subject: [PATCH 52/52] * addressed more PR comments --- .../za/co/absa/atum/reader/FlowReader.scala | 21 ++++++++++--------- .../reader/core/PartitioningIdProvider.scala | 12 +++++++---- .../reader/exceptions/RequestException.scala | 3 ++- .../atum/reader/FlowReaderUnitTests.scala | 3 +-- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala index 3119bfb40..031953a0d 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/FlowReader.scala @@ -31,21 +31,22 @@ import za.co.absa.atum.reader.server.ServerConfig /** * This class is a reader that reads data tight to a flow. - * @param mainFlowPartitioning - the partitioning of the main flow; renamed from ancestor's 'flowPartitioning' - * @param serverConfig - the Atum server configuration - * @param backend - sttp backend, that will be executing the requests - * @param ev - using evidence based approach to ensure that the type F is a MonadError instead of using context - * bounds, as it make the imports easier to follow - * @tparam F - the effect type (e.g. Future, IO, Task, etc.) + * + * @param mainFlowPartitioning - the partitioning of the main flow; renamed from ancestor's 'flowPartitioning' + * @param serverConfig - the Atum server configuration + * @param backend - sttp backend, that will be executing the requests + * @param ev - using evidence based approach to ensure that the type F is a MonadError instead of using context + * bounds, as it make the imports easier to follow + * @tparam F - the effect type (e.g. Future, IO, Task, etc.) */ -class FlowReader[F[_]](val mainFlowPartitioning: AtumPartitions) - (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any], ev: MonadError[F]) - extends Reader[F] with PartitioningIdProvider[F]{ +class FlowReader[F[_]: MonadError](val mainFlowPartitioning: AtumPartitions) + (implicit serverConfig: ServerConfig, backend: SttpBackend[F, Any]) + extends Reader[F] with PartitioningIdProvider[F] { private def queryFlowId(mainPartitioningId: Long): F[RequestResult[Long]] = { val endpoint = s"/$Api/$V2/${V2Paths.Partitionings}/$mainPartitioningId/${V2Paths.MainFlow}" val queryResult = getQuery[SingleSuccessResponse[FlowDTO]](endpoint) - queryResult.map{ result => + queryResult.map { result => result.map(_.data.id) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala b/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala index f6f6deb35..3398c7401 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/core/PartitioningIdProvider.scala @@ -24,13 +24,17 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.SingleSuccessResponse import za.co.absa.atum.model.types.basic.AtumPartitions import za.co.absa.atum.model.types.basic.AtumPartitionsOps import za.co.absa.atum.model.utils.JsonSyntaxExtensions.JsonSerializationSyntax -import RequestResult.RequestResult +import za.co.absa.atum.reader.core.RequestResult.RequestResult -trait PartitioningIdProvider[F[_]] {self: Reader[F] => +trait PartitioningIdProvider[F[_]] { + self: Reader[F] => def partitioningId(partitioning: AtumPartitions)(implicit monad: MonadError[F]): F[RequestResult[Long]] = { val encodedPartitioning = partitioning.toPartitioningDTO.asBase64EncodedJsonString - val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]](s"/$Api/$V2/${V2Paths.Partitionings}", Map("partitioning" -> encodedPartitioning)) - queryResult.map{ result => + val queryResult = getQuery[SingleSuccessResponse[PartitioningWithIdDTO]]( + s"/$Api/$V2/${V2Paths.Partitionings}", + Map("partitioning" -> encodedPartitioning) + ) + queryResult.map { result => result.map(_.data.id) } } diff --git a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala index 4f713eb55..6aba5f3d1 100644 --- a/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala +++ b/reader/src/main/scala/za/co/absa/atum/reader/exceptions/RequestException.scala @@ -35,7 +35,8 @@ object RequestException { final case class ParsingException( message: String, body: String - ) extends RequestException(message) + ) extends RequestException(message) + object ParsingException { def fromCirceError(error: CirceError, body: String): ParsingException = { ParsingException(error.getMessage, body) diff --git a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala index 560c6b070..8ae886996 100644 --- a/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala +++ b/reader/src/test/scala/za/co/absa/atum/reader/FlowReaderUnitTests.scala @@ -31,7 +31,6 @@ import za.co.absa.atum.model.envelopes.SuccessResponse.PaginatedResponse import za.co.absa.atum.model.types.basic.{AtumPartitions, AtumPartitionsOps} import za.co.absa.atum.reader.FlowReaderUnitTests._ import za.co.absa.atum.reader.server.ServerConfig -import za.co.absa.atum.reader.implicits.future.futureMonadError import java.time.ZonedDateTime import java.util.UUID @@ -46,7 +45,7 @@ class FlowReaderUnitTests extends AnyFunSuiteLike { "a" -> "b", "c" -> "d" )) - implicit val server: SttpBackend[Future, Any] = SttpBackendStub.asynchronousFuture + implicit val server: SttpBackend[Identity, Any] = SttpBackendStub.synchronous val result = new FlowReader(atumPartitions).mainFlowPartitioning assert(result == atumPartitions)