diff --git a/.github/workflows/deploy-documentation-old.yml b/.github/workflows/deploy-documentation-old.yml deleted file mode 100644 index 29d002286..000000000 --- a/.github/workflows/deploy-documentation-old.yml +++ /dev/null @@ -1,56 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Documentation to Pages - -on: - # Runs on pushes targeting the default branch - # push: - # branches: ["develop"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - # Build job - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup Pages - uses: actions/configure-pages@v3 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 - with: - source: ./docs - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - - # Deployment job - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v2 diff --git a/.github/workflows/deploy-documentation.yml b/.github/workflows/deploy-documentation.yml index ee3f261e3..a719d83ec 100644 --- a/.github/workflows/deploy-documentation.yml +++ b/.github/workflows/deploy-documentation.yml @@ -34,7 +34,7 @@ jobs: export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout) echo "version=$VERSION" >> $>> $GITHUB_OUTPUT - name: Setup Ruby - uses: ruby/setup-ruby@0a29871fe2b0200a17a4497bae54fe5df0d973aa + uses: ruby/setup-ruby@v1 with: ruby-version: '3.0' # Not needed with a .ruby-version file bundler-cache: true # runs 'bundle install' and caches installed gems automatically diff --git a/docs/_ldi-nifi/index.md b/docs/_ldi-nifi/index.md index 61bff68f6..0e806cb85 100644 --- a/docs/_ldi-nifi/index.md +++ b/docs/_ldi-nifi/index.md @@ -15,28 +15,61 @@ The processors can be imported into a NiFi docker instance via volume binding: 1. Create a `docker-compose.yml` file with the following content in a new directory ````yaml services: - nifi: - image: apache/nifi:2.0.0 - environment: - SINGLE_USER_CREDENTIALS_USERNAME: admin - SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB - ports: - - 8443:8443 - volumes: - - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw + nifi: + image: apache/nifi:2.0.0 + environment: + SINGLE_USER_CREDENTIALS_USERNAME: admin + SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB + ports: + - 8443:8443 + volumes: + - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw + - ./nifi-drivers:/opt/nifi/nifi-current/lib_temp:rw + entrypoint: [ "/bin/bash", + "-c", + "cp -r /opt/nifi/nifi-current/lib_temp/* /opt/nifi/nifi-current/lib/ && exec /opt/nifi/scripts/start.sh" ] ```` 2. Create a directory `nifi-ext` in your current directory. 3. Download either the `...-nar-bundle.jar` and unpack this or download the individual required processors (.nar extension) from the [nexus repository]. Next, place the required processors in the `nifi-ext` directory. -4. Finally, start your instance. +4. Additionally, for using any processor with a persistence layer, you will need to download the needed drivers. + Place these in the `nifi-drivers` directory. + Possible drivers are: + - [PostgreSQL driver](https://jdbc.postgresql.org/download.html) + - [SQLite driver](https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc) +5. Finally, start your instance. ````shell docker compose up ```` -5. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1 -6. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group. +6. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1 +7. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group. {: .note } All documentation and notes about configuration are available in the NiFi component itself. +## Setting up Database Connection Pools + +As of LDI 2.13.0, some processors require a database connection pool to be set up. +Although this is a pure NiFi configuration, a minimal explanation will be mentioned here. \ +For more details, please refer to +the [NiFi documentation on DBCP](https://nifi.apache.org/components/org.apache.nifi.dbcp.DBCPConnectionPool/). +This can be done by following these steps: + +1. Go to the `Controller Services` tab in the NiFi UI. +2. Click on the `Create new service` button. +3. Select the `DBCPConnectionPool` service. +4. Configure the service with the required properties: + - `Database Connection URL`, + - `jdbc:sqlite:/path/to/database.db` for SQLite + - `jdbc:postgresql://localhost:5432/database` for PostgreSQL, + - `Database Driver Class Name` + - `org.sqlite.JDBC` for SQLite + - `org.postgresql.Driver` for PostgreSQL, + - `Database User`, + - `Password`, +5. Save the service. +6. Enable the service. +7. Use the service in the processors that require it. + [Apache NiFi]: https://nifi.apache.org/ [nexus repository]: https://s01.oss.sonatype.org/#nexus-search;quick~be.vlaanderen.informatievlaanderen.ldes.ldi.nifi \ No newline at end of file diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index 4d96af833..82dc4dd05 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/change-detection-filter/pom.xml b/ldi-core/change-detection-filter/pom.xml index 68b792542..b0d1e99d7 100644 --- a/ldi-core/change-detection-filter/pom.xml +++ b/ldi-core/change-detection-filter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT change-detection-filter diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java index aaa0c62a4..0d79990df 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java @@ -27,11 +27,9 @@ public class ChangeDetectionFilter implements LdiOneToOneTransformer { private static final Lang NORMALIZING_LANG = Lang.NQUADS; private static final MediaType NORMALIZING_MEDIA_TYPE = MediaType.N_QUADS; private final HashedStateMemberRepository hashedStateMemberRepository; - private final boolean keepState; - public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository, boolean keepState) { + public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository) { this.hashedStateMemberRepository = hashedStateMemberRepository; - this.keepState = keepState; } /** @@ -54,16 +52,10 @@ public Model transform(Model model) { return model; } - /** - * Clean up the resources, should be called when the component is not used anymore - */ - public void destroyState() { - if (!keepState) { - hashedStateMemberRepository.destroyState(); - } + public void close() { + hashedStateMemberRepository.close(); } - private Resource getSingleNamedNodeFromStateObject(Model model) { final List namedNodes = model.listSubjects().filterDrop(RDFNode::isAnon).toList(); if (namedNodes.size() != 1) { diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java index 69928c3ac..258644ea0 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java @@ -18,5 +18,5 @@ public interface HashedStateMemberRepository { /** * Clean up the resources, should be called when the repository is not used anymore */ - void destroyState(); + void close(); } diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java index 645a5bca8..1b15b3936 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java @@ -3,7 +3,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; -import java.util.*; +import java.util.HashMap; +import java.util.Map; public class InMemoryHashedStateMemberRepository implements HashedStateMemberRepository { private final Map members; @@ -24,7 +25,7 @@ public void saveHashedStateMember(HashedStateMember hashedStateMember) { } @Override - public void destroyState() { + public void close() { members.clear(); } } diff --git a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java index 84c8cf733..af93889be 100644 --- a/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java +++ b/ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql; -import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember; import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMemberEntity; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; @@ -9,14 +8,10 @@ import javax.persistence.EntityManager; public class SqlHashedStateMemberRepository implements HashedStateMemberRepository { - private final EntityManagerFactory entityManagerFactory; private final EntityManager entityManager; - private final String instanceName; - public SqlHashedStateMemberRepository(EntityManagerFactory entityManagerFactory, String instanceName) { - this.entityManagerFactory = entityManagerFactory; - this.entityManager = entityManagerFactory.getEntityManager(); - this.instanceName = instanceName; + public SqlHashedStateMemberRepository(EntityManager entityManager) { + this.entityManager = entityManager; } @Override @@ -38,7 +33,7 @@ public void saveHashedStateMember(HashedStateMember hashedStateMember) { } @Override - public void destroyState() { - entityManagerFactory.destroyState(instanceName); + public void close() { + entityManager.close(); } } diff --git a/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterSteps.java b/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterSteps.java index 77054a064..c7cb78735 100644 --- a/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterSteps.java +++ b/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterSteps.java @@ -1,11 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.inmemory.InMemoryHashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql.SqlHashedStateMemberRepository; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -34,14 +32,14 @@ public void teardown() { @Given("A ChangeDetectionFilter with state persistence strategy MEMORY") public void aChangeDetectionFilterWithStatePersistenceStrategyMEMORY() { HashedStateMemberRepository hashedStateMemberRepository = new InMemoryHashedStateMemberRepository(); - changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false); + changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository); } @Given("A ChangeDetectionFilter with state persistence strategy SQLITE") public void aChangeDetectionFilterWithStatePersistenceStrategySQLITE() { - final var emf = SqliteEntityManagerFactory.getInstance(new SqliteProperties(DATABASE_INSTANCE_NAME, false)); - HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(emf, DATABASE_INSTANCE_NAME); - changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false); + var entityManager = HibernateUtil.createEntityManagerFromProperties(new SqliteProperties("target", DATABASE_INSTANCE_NAME, false).getProperties()); + HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(entityManager); + changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository); } @Given("A ChangeDetectionFilter with state persistence strategy POSTGRES") @@ -55,9 +53,9 @@ public void aChangeDetectionFilterWithStatePersistenceStrategyPOSTGRES() { PostgresProperties postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(), postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false); - final var emf = PostgresEntityManagerFactory.getInstance(DATABASE_INSTANCE_NAME, postgresProperties.getProperties()); - HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(emf, DATABASE_INSTANCE_NAME); - changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false); + var entityManager = HibernateUtil.createEntityManagerFromProperties(postgresProperties.getProperties()); + HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(entityManager); + changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository); } @Then("The filtered member is not empty") @@ -85,6 +83,6 @@ public void theFilteredMemberIsIsomorphicWith(String fileName) { @Then("The filter is destroyed") public void theFilterIsDestroyed() { - changeDetectionFilter.destroyState(); + changeDetectionFilter.close(); } } diff --git a/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterTest.java b/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterTest.java index ea52706c0..2000c0851 100644 --- a/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterTest.java +++ b/ldi-core/change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterTest.java @@ -11,7 +11,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -20,7 +19,7 @@ import static org.assertj.core.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class ChangeDetectionFilterTest { @@ -31,7 +30,7 @@ class ChangeDetectionFilterTest { @BeforeEach void setUp() { - changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false); + changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository); } @Test @@ -71,18 +70,6 @@ void given_EmptyRepository_when_FilterModel_then_ReturnExactModel() { assertThat(filteredModel).isSameAs(modelToFilter); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void given_KeepState_when_DestroyFilterState_then_VerifyRepositoryDestroy(boolean keepState) { - final int expectedDestroyRepositoryInvocations = keepState ? 0 : 1; - final var filter = new ChangeDetectionFilter(hashedStateMemberRepository, keepState); - - filter.destroyState(); - - verify(hashedStateMemberRepository, times(expectedDestroyRepositoryInvocations)).destroyState(); - } - - private static Stream supplyInvalidTriples() { final String twoStateObjects = """ . diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index 49a71dbf8..17ba18967 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index bbf33ed85..aab1cfb8d 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/http-sparql-out/pom.xml b/ldi-core/http-sparql-out/pom.xml index 932037d8f..56447e00d 100644 --- a/ldi-core/http-sparql-out/pom.xml +++ b/ldi-core/http-sparql-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT http-sparql-out diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index 553dba50d..3fe42171d 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml b/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml index 888b520f4..50069e2f9 100644 --- a/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml +++ b/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.client ldes-client - 2.12.0 + 2.13.0-SNAPSHOT event-stream-properties-fetcher diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index 10fb9a6b9..2463506bf 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 809262cc7..17ad41298 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index e737f3f06..386e57b25 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.12.0 + 2.13.0-SNAPSHOT diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index dfa9de6e8..d0dbd5601 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index 0de8f8e53..0c437a97e 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,11 +5,15 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 tree-node-supplier + + 2.2 + + be.vlaanderen.informatievlaanderen.ldes.ldi @@ -69,6 +73,16 @@ hibernate-core ${hibernate.version} + + javax.persistence + javax.persistence-api + ${javax.persistence-api.version} + + + org.hibernate + hibernate-entitymanager + ${hibernate.version} + com.zaxxer HikariCP diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java index c5f73deeb..c814308aa 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/TreeNodeProcessor.java @@ -32,11 +32,11 @@ public class TreeNodeProcessor { private final Consumer clientStatusConsumer; private MemberRecord memberRecord; - public TreeNodeProcessor(LdesMetaData ldesMetaData, StatePersistence statePersistence, + public TreeNodeProcessor(LdesMetaData ldesMetaData, LdesClientRepositories ldesClientRepositories, RequestExecutor requestExecutor, TimestampExtractor timestampExtractor, Consumer clientStatusConsumer) { - this.treeNodeRecordRepository = statePersistence.getTreeNodeRecordRepository(); - this.memberRepository = statePersistence.getMemberRepository(); + this.treeNodeRecordRepository = ldesClientRepositories.treeNodeRecordRepository(); + this.memberRepository = ldesClientRepositories.memberRepository(); this.requestExecutor = requestExecutor; this.clientStatusConsumer = clientStatusConsumer; this.treeNodeFetcher = new TreeNodeFetcher(requestExecutor, timestampExtractor); diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java index 436355f40..dc3ef5fcf 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberIdRepositoryFactory.java @@ -1,13 +1,12 @@ package ldes.client.treenodesupplier.domain.services; -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import ldes.client.treenodesupplier.repository.MemberIdRepository; import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberIdRepository; import ldes.client.treenodesupplier.repository.sql.SqlMemberIdRepository; +import javax.persistence.EntityManager; + public class MemberIdRepositoryFactory { private MemberIdRepositoryFactory() { @@ -15,18 +14,13 @@ private MemberIdRepositoryFactory() { /** * @param statePersistenceStrategy via what persistence strategy the repository should work - * @param properties a representation of the required config properties to set up the persistence unit - * @param instanceName will be used to be able to more easily keep track of the return repo + * @param entityManager a persistence unit to manage the entities * @return the memberIdRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) */ - public static MemberIdRepository getMemberIdRepository(StatePersistenceStrategy statePersistenceStrategy, - HibernateProperties properties, String instanceName) { + public static MemberIdRepository getMemberIdRepository(StatePersistenceStrategy statePersistenceStrategy, EntityManager entityManager) { return switch (statePersistenceStrategy) { - case SQLITE -> new SqlMemberIdRepository(instanceName, - SqliteEntityManagerFactory.getInstance(properties)); + case SQLITE, POSTGRES -> new SqlMemberIdRepository(entityManager); case MEMORY -> new InMemoryMemberIdRepository(); - case POSTGRES -> new SqlMemberIdRepository(instanceName, - PostgresEntityManagerFactory.getInstance(instanceName, properties.getProperties())); }; } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java index 9f5154b24..fc072c07c 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberRepositoryFactory.java @@ -1,13 +1,12 @@ package ldes.client.treenodesupplier.domain.services; -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import ldes.client.treenodesupplier.repository.MemberRepository; import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberRepository; import ldes.client.treenodesupplier.repository.sql.SqlMemberRepository; +import javax.persistence.EntityManager; + public class MemberRepositoryFactory { private MemberRepositoryFactory() { @@ -15,18 +14,13 @@ private MemberRepositoryFactory() { /** * @param statePersistenceStrategy via what persistence strategy the repository should work - * @param properties a representation of the required config properties to set up the persistence unit - * @param instanceName will be used to be able to more easily keep track of the return repo - * @return the memberNRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) + * @param entityManager a persistence unit to manage the entities + * @return the memberRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) */ - public static MemberRepository getMemberRepository(StatePersistenceStrategy statePersistenceStrategy, - HibernateProperties properties, String instanceName) { + public static MemberRepository getMemberRepository(StatePersistenceStrategy statePersistenceStrategy, EntityManager entityManager) { return switch (statePersistenceStrategy) { - case SQLITE -> new SqlMemberRepository(instanceName, - SqliteEntityManagerFactory.getInstance(properties)); + case SQLITE, POSTGRES -> new SqlMemberRepository(entityManager); case MEMORY -> new InMemoryMemberRepository(); - case POSTGRES -> new SqlMemberRepository(instanceName, - PostgresEntityManagerFactory.getInstance(instanceName, properties.getProperties())); }; } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java index 10fd541e3..7b3537676 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/MemberVersionRepositoryFactory.java @@ -1,13 +1,12 @@ package ldes.client.treenodesupplier.domain.services; -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import ldes.client.treenodesupplier.repository.MemberVersionRepository; import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberVersionRepository; import ldes.client.treenodesupplier.repository.sql.SqlMemberVersionRepository; +import javax.persistence.EntityManager; + public class MemberVersionRepositoryFactory { private MemberVersionRepositoryFactory() { @@ -15,18 +14,13 @@ private MemberVersionRepositoryFactory() { /** * @param statePersistenceStrategy via what persistence strategy the repository should work - * @param properties a representation of the required config properties to set up the persistence unit - * @param instanceName will be used to be able to more easily keep track of the return repo + * @param entityManager a persistence unit to manage the entities * @return the memberVersionRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) */ - public static MemberVersionRepository getMemberVersionRepositoryFactory(StatePersistenceStrategy statePersistenceStrategy, - HibernateProperties properties, String instanceName) { + public static MemberVersionRepository getMemberVersionRepositoryFactory(StatePersistenceStrategy statePersistenceStrategy, EntityManager entityManager) { return switch (statePersistenceStrategy) { - case SQLITE -> - new SqlMemberVersionRepository(SqliteEntityManagerFactory.getInstance(properties), instanceName); + case SQLITE, POSTGRES -> new SqlMemberVersionRepository(entityManager); case MEMORY -> new InMemoryMemberVersionRepository(); - case POSTGRES -> - new SqlMemberVersionRepository(PostgresEntityManagerFactory.getInstance(instanceName, properties.getProperties()), instanceName); }; } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java index 0aedc632a..e3ded3355 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/services/TreeNodeRecordRepositoryFactory.java @@ -1,31 +1,25 @@ package ldes.client.treenodesupplier.domain.services; -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; import ldes.client.treenodesupplier.repository.inmemory.InMemoryTreeNodeRecordRepository; import ldes.client.treenodesupplier.repository.sql.SqlTreeNodeRepository; +import javax.persistence.EntityManager; + public class TreeNodeRecordRepositoryFactory { private TreeNodeRecordRepositoryFactory() { } /** * @param statePersistenceStrategy via what persistence strategy the repository should work - * @param properties a representation of the required config properties to set up the persistence unit - * @param instanceName will be used to be able to more easily keep track of the return repo - * @return the treeNodeRecordRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) + * @param entityManager a persistence unit to manage the entities + * @return the treeNodeRepository for a specific instance (could be a NiFi flow or a LDIO pipeline) */ - public static TreeNodeRecordRepository getTreeNodeRecordRepository( - StatePersistenceStrategy statePersistenceStrategy, HibernateProperties properties, String instanceName) { + public static TreeNodeRecordRepository getTreeNodeRecordRepository(StatePersistenceStrategy statePersistenceStrategy, EntityManager entityManager) { return switch (statePersistenceStrategy) { - case SQLITE -> new SqlTreeNodeRepository(instanceName, - SqliteEntityManagerFactory.getInstance(properties)); + case SQLITE, POSTGRES -> new SqlTreeNodeRepository(entityManager); case MEMORY -> new InMemoryTreeNodeRecordRepository(); - case POSTGRES -> new SqlTreeNodeRepository(instanceName, - PostgresEntityManagerFactory.getInstance(instanceName, properties.getProperties())); }; } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesClientRepositories.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesClientRepositories.java new file mode 100644 index 000000000..87e378242 --- /dev/null +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/LdesClientRepositories.java @@ -0,0 +1,37 @@ +package ldes.client.treenodesupplier.domain.valueobject; + +import ldes.client.treenodesupplier.repository.MemberIdRepository; +import ldes.client.treenodesupplier.repository.MemberRepository; +import ldes.client.treenodesupplier.repository.MemberVersionRepository; +import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; +import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberIdRepository; +import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberRepository; +import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberVersionRepository; +import ldes.client.treenodesupplier.repository.inmemory.InMemoryTreeNodeRecordRepository; +import ldes.client.treenodesupplier.repository.sql.SqlMemberIdRepository; +import ldes.client.treenodesupplier.repository.sql.SqlMemberRepository; +import ldes.client.treenodesupplier.repository.sql.SqlMemberVersionRepository; +import ldes.client.treenodesupplier.repository.sql.SqlTreeNodeRepository; + +import javax.persistence.EntityManager; + +public record LdesClientRepositories(MemberRepository memberRepository, MemberIdRepository memberIdRepository, + TreeNodeRecordRepository treeNodeRecordRepository, + MemberVersionRepository memberVersionRepository) { + + public static LdesClientRepositories sqlBased(EntityManager entityManager) { + return new LdesClientRepositories( + new SqlMemberRepository(entityManager), + new SqlMemberIdRepository(entityManager), + new SqlTreeNodeRepository(entityManager), + new SqlMemberVersionRepository(entityManager)); + } + + public static LdesClientRepositories memoryBased() { + return new LdesClientRepositories( + new InMemoryMemberRepository(), + new InMemoryMemberIdRepository(), + new InMemoryTreeNodeRecordRepository(), + new InMemoryMemberVersionRepository()); + } +} diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java deleted file mode 100644 index 1f84fc065..000000000 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistence.java +++ /dev/null @@ -1,50 +0,0 @@ -package ldes.client.treenodesupplier.domain.valueobject; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; -import ldes.client.treenodesupplier.domain.services.*; -import ldes.client.treenodesupplier.repository.MemberIdRepository; -import ldes.client.treenodesupplier.repository.MemberRepository; -import ldes.client.treenodesupplier.repository.MemberVersionRepository; -import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; - -public class StatePersistence { - - private final MemberRepository memberRepository; - private final MemberIdRepository memberIdRepository; - private final TreeNodeRecordRepository treeNodeRecordRepository; - private final MemberVersionRepository memberVersionRepository; - - public StatePersistence(MemberRepository memberRepository, MemberIdRepository memberIdRepository, TreeNodeRecordRepository treeNodeRecordRepository, MemberVersionRepository memberVersionRepository) { - this.memberRepository = memberRepository; - this.memberIdRepository = memberIdRepository; - this.treeNodeRecordRepository = treeNodeRecordRepository; - this.memberVersionRepository = memberVersionRepository; - } - - public static StatePersistence from(StatePersistenceStrategy statePersistenceStrategy, - HibernateProperties properties, String instanceName) { - return new StatePersistence( - MemberRepositoryFactory.getMemberRepository(statePersistenceStrategy, properties, instanceName), - MemberIdRepositoryFactory.getMemberIdRepository(statePersistenceStrategy, properties, instanceName), - TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(statePersistenceStrategy, properties, instanceName), - MemberVersionRepositoryFactory.getMemberVersionRepositoryFactory(statePersistenceStrategy, properties, instanceName)); - } - - public MemberRepository getMemberRepository() { - return memberRepository; - } - - public MemberIdRepository getMemberIdRepository() { - return memberIdRepository; - } - - public TreeNodeRecordRepository getTreeNodeRecordRepository() { - return treeNodeRecordRepository; - } - - public MemberVersionRepository getMemberVersionRepository() { - return memberVersionRepository; - } -} diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberIdRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberIdRepository.java index 7fd149f2b..5b8c91f72 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberIdRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberIdRepository.java @@ -1,23 +1,24 @@ package ldes.client.treenodesupplier.repository.sql; -import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.StatelessQueryExecutor; import ldes.client.treenodesupplier.repository.MemberIdRepository; +import org.hibernate.Session; +import org.hibernate.StatelessSession; +import javax.persistence.EntityManager; import javax.transaction.Transactional; public class SqlMemberIdRepository implements MemberIdRepository { - private final EntityManagerFactory entityManagerFactory; - private final String instanceName; + private final EntityManager entityManager; - public SqlMemberIdRepository(String instanceName, EntityManagerFactory entityManagerFactory) { - this.entityManagerFactory = entityManagerFactory; - this.instanceName = instanceName; + public SqlMemberIdRepository(EntityManager entityManager) { + this.entityManager = entityManager; } @Transactional @Override public boolean addMemberIdIfNotExists(String memberId) { - return entityManagerFactory.executeStatelessQuery(session -> session + return executeStatelessQuery(session -> session .createNamedQuery("MemberId.insert") .setParameter("memberId", memberId) .executeUpdate()) > 0; @@ -25,6 +26,17 @@ public boolean addMemberIdIfNotExists(String memberId) { @Override public void destroyState() { - entityManagerFactory.destroyState(instanceName); + if (entityManager.isOpen()) { + entityManager.close(); + } + } + + private int executeStatelessQuery(StatelessQueryExecutor queryExecutor) { + final Session session = entityManager.unwrap(Session.class); + return session.doReturningWork(connection -> { + try (final StatelessSession statelessSession = session.getSessionFactory().openStatelessSession(connection)) { + return queryExecutor.execute(statelessSession); + } + }); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java index a1f9653ee..3b62235c0 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberRepository.java @@ -1,26 +1,27 @@ package ldes.client.treenodesupplier.repository.sql; -import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.StatelessQueryExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.MemberRecordEntity; import ldes.client.treenodesupplier.domain.entities.MemberRecord; import ldes.client.treenodesupplier.repository.MemberRepository; import ldes.client.treenodesupplier.repository.mapper.MemberRecordEntityMapper; +import org.hibernate.Session; +import org.hibernate.StatelessSession; +import javax.persistence.EntityManager; import java.util.Optional; import java.util.stream.Stream; public class SqlMemberRepository implements MemberRepository { - private final EntityManagerFactory entityManagerFactory; - private final String instanceName; + private final EntityManager entityManager; - public SqlMemberRepository(String instanceName, EntityManagerFactory entityManagerFactory) { - this.entityManagerFactory = entityManagerFactory; - this.instanceName = instanceName; + public SqlMemberRepository(EntityManager entityManager) { + this.entityManager = entityManager; } @Override public Optional getTreeMember() { - return entityManagerFactory.getEntityManager() + return entityManager .createNamedQuery("Member.getAllOrderedByCreation", MemberRecordEntity.class) .setMaxResults(1) .getResultStream() @@ -30,7 +31,7 @@ public Optional getTreeMember() { @Override public void deleteMember(MemberRecord member) { - entityManagerFactory.executeStatelessQuery(session -> session + executeStatelessQuery(session -> session .createNamedQuery("Member.deleteByMemberId") .setParameter("memberId", member.getMemberId()) .executeUpdate()); @@ -38,14 +39,23 @@ public void deleteMember(MemberRecord member) { @Override public void saveTreeMembers(Stream treeMemberStream) { - entityManagerFactory.getEntityManager().getTransaction().begin(); + entityManager.getTransaction().begin(); treeMemberStream.map(MemberRecordEntityMapper::fromMemberRecord) - .forEach(entityManagerFactory.getEntityManager()::merge); - entityManagerFactory.getEntityManager().getTransaction().commit(); + .forEach(entityManager::merge); + entityManager.getTransaction().commit(); } @Override public void destroyState() { - entityManagerFactory.destroyState(instanceName); + entityManager.clear(); + } + + private int executeStatelessQuery(StatelessQueryExecutor queryExecutor) { + final Session session = entityManager.unwrap(Session.class); + return session.doReturningWork(connection -> { + try (final StatelessSession statelessSession = session.getSessionFactory().openStatelessSession(connection)) { + return queryExecutor.execute(statelessSession); + } + }); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepository.java index 0e1b54011..569494bc2 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepository.java @@ -1,40 +1,40 @@ package ldes.client.treenodesupplier.repository.sql; -import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.MemberVersionRecordEntity; import ldes.client.treenodesupplier.domain.entities.MemberVersionRecord; import ldes.client.treenodesupplier.repository.MemberVersionRepository; import ldes.client.treenodesupplier.repository.mapper.MemberVersionRecordEntityMapper; +import javax.persistence.EntityManager; + public class SqlMemberVersionRepository implements MemberVersionRepository { - private final EntityManagerFactory entityManagerFactory; - private final String instanceName; - - public SqlMemberVersionRepository(EntityManagerFactory entityManagerFactory, String instanceName) { - this.entityManagerFactory = entityManagerFactory; - this.instanceName = instanceName; - } - - @Override - public void addMemberVersion(MemberVersionRecord memberVersion) { - entityManagerFactory.getEntityManager() - .merge(MemberVersionRecordEntityMapper.fromMemberVersionRecord(memberVersion)); - } - - @Override - public boolean isVersionAfterTimestamp(MemberVersionRecord memberVersion) { - return entityManagerFactory.getEntityManager() - .createNamedQuery("MemberVersion.findMemberVersionAfterTimestamp", MemberVersionRecordEntity.class) - .setParameter("versionOf", memberVersion.getVersionOf()) - .setParameter("timestamp", memberVersion.getTimestamp()) - .getResultStream() - .findFirst() - .isEmpty(); - } - - @Override - public void destroyState() { - entityManagerFactory.destroyState(instanceName); - } + private final EntityManager entityManager; + + public SqlMemberVersionRepository(EntityManager entityManager) { + this.entityManager = entityManager; + } + + @Override + public void addMemberVersion(MemberVersionRecord memberVersion) { + entityManager.merge(MemberVersionRecordEntityMapper.fromMemberVersionRecord(memberVersion)); + } + + @Override + public boolean isVersionAfterTimestamp(MemberVersionRecord memberVersion) { + return entityManager + .createNamedQuery("MemberVersion.findMemberVersionAfterTimestamp", MemberVersionRecordEntity.class) + .setParameter("versionOf", memberVersion.getVersionOf()) + .setParameter("timestamp", memberVersion.getTimestamp()) + .getResultStream() + .findFirst() + .isEmpty(); + } + + @Override + public void destroyState() { + if (entityManager.isOpen()) { + entityManager.close(); + } + } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlTreeNodeRepository.java b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlTreeNodeRepository.java index 7ecc64f2d..29fe3f135 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlTreeNodeRepository.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/main/java/ldes/client/treenodesupplier/repository/sql/SqlTreeNodeRepository.java @@ -1,26 +1,19 @@ package ldes.client.treenodesupplier.repository.sql; -import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.TreeNodeRecordEntity; import ldes.client.treenodesupplier.domain.entities.TreeNodeRecord; import ldes.client.treenodesupplier.domain.valueobject.TreeNodeStatus; import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; import ldes.client.treenodesupplier.repository.mapper.TreeNodeRecordEntityMapper; -import java.util.Optional; - import javax.persistence.EntityManager; +import java.util.Optional; public class SqlTreeNodeRepository implements TreeNodeRecordRepository { - - final EntityManagerFactory entityManagerFactory; private final EntityManager entityManager; - private final String instanceName; - public SqlTreeNodeRepository(String instanceName, EntityManagerFactory entityManagerFactory) { - this.entityManagerFactory = entityManagerFactory; - this.entityManager = entityManagerFactory.getEntityManager(); - this.instanceName = instanceName; + public SqlTreeNodeRepository(EntityManager entityManager) { + this.entityManager = entityManager; } @Override @@ -67,7 +60,9 @@ public boolean existsByIdAndStatus(String treeNodeId, TreeNodeStatus treeNodeSta @Override public void destroyState() { - entityManagerFactory.destroyState(instanceName); + if (entityManager.isOpen()) { + entityManager.close(); + } } @Override diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java index 311f71e4a..26ef16790 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/performance/TreeNodeProcessorFactory.java @@ -1,5 +1,6 @@ package ldes.client.performance; +import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateUtil; import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; @@ -8,16 +9,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import ldes.client.treenodesupplier.TreeNodeProcessor; -import ldes.client.treenodesupplier.domain.services.MemberIdRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.MemberRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.MemberVersionRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.TreeNodeRecordRepositoryFactory; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; import ldes.client.treenodesupplier.domain.valueobject.LdesMetaData; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; -import ldes.client.treenodesupplier.repository.MemberIdRepository; -import ldes.client.treenodesupplier.repository.MemberRepository; -import ldes.client.treenodesupplier.repository.MemberVersionRepository; -import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; import org.apache.jena.riot.Lang; import org.mockito.Mockito; import org.testcontainers.containers.PostgreSQLContainer; @@ -31,14 +24,14 @@ class TreeNodeProcessorFactory { TreeNodeProcessor createTreeNodeProcessor(StatePersistenceStrategy statePersistenceStrategy, List url, Lang sourceFormat) { final LdesMetaData ldesMetaData = new LdesMetaData(url, sourceFormat); - final StatePersistence statePersistence = switch (statePersistenceStrategy) { + final LdesClientRepositories ldesClientRepositories = switch (statePersistenceStrategy) { case MEMORY -> createInMemoryStatePersistence(); case SQLITE -> createSqliteStatePersistence(); case POSTGRES -> createPostgresPersistence(); }; final RequestExecutor requestExecutor = requestExecutorFactory.createNoAuthExecutor(); final TimestampExtractor timestampExtractor = new TimestampFromCurrentTimeExtractor(); - return new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor, Mockito.mock(Consumer.class)); + return new TreeNodeProcessor(ldesMetaData, ldesClientRepositories, requestExecutor, timestampExtractor, Mockito.mock(Consumer.class)); } private PostgreSQLContainer startPostgresContainer() { @@ -50,52 +43,33 @@ private PostgreSQLContainer startPostgresContainer() { return postgreSQLContainer; } - private StatePersistence createSqliteStatePersistence() { + private LdesClientRepositories createSqliteStatePersistence() { final SqliteProperties sqliteProperties = new SqliteProperties("instanceName", false); - MemberRepository memberRepository = MemberRepositoryFactory.getMemberRepository(StatePersistenceStrategy.SQLITE, - sqliteProperties, "instanceName"); - TreeNodeRecordRepository treeNodeRecordRepository = TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(StatePersistenceStrategy.SQLITE, sqliteProperties, "instanceName"); - MemberIdRepository memberIdRepository = MemberIdRepositoryFactory.getMemberIdRepository(StatePersistenceStrategy.SQLITE, - sqliteProperties, "instanceName"); - MemberVersionRepository memberVersionRepository = MemberVersionRepositoryFactory.getMemberVersionRepositoryFactory(StatePersistenceStrategy.SQLITE, - sqliteProperties, "instanceName"); + LdesClientRepositories clientRepositories = LdesClientRepositories.sqlBased( + HibernateUtil.createEntityManagerFromProperties(sqliteProperties.getProperties())); - return new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository); + return new LdesClientRepositories(clientRepositories.memberRepository(), clientRepositories.memberIdRepository(), + clientRepositories.treeNodeRecordRepository(), clientRepositories.memberVersionRepository()); } - private StatePersistence createPostgresPersistence() { + private LdesClientRepositories createPostgresPersistence() { final PostgreSQLContainer postgreSQLContainer = startPostgresContainer(); PostgresProperties postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(), postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false); - MemberRepository memberRepository = MemberRepositoryFactory.getMemberRepository( - StatePersistenceStrategy.POSTGRES, - postgresProperties, "instanceName"); - TreeNodeRecordRepository treeNodeRecordRepository = TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(StatePersistenceStrategy.POSTGRES, postgresProperties, - "instanceName"); - MemberIdRepository memberIdRepository = MemberIdRepositoryFactory.getMemberIdRepository( - StatePersistenceStrategy.POSTGRES, - postgresProperties, "instanceName"); - MemberVersionRepository memberVersionRepository = MemberVersionRepositoryFactory.getMemberVersionRepositoryFactory( - StatePersistenceStrategy.POSTGRES, - postgresProperties, "instanceName"); - return new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository); + LdesClientRepositories clientRepositories = LdesClientRepositories.sqlBased( + HibernateUtil.createEntityManagerFromProperties(postgresProperties.getProperties())); + + return new LdesClientRepositories(clientRepositories.memberRepository(), clientRepositories.memberIdRepository(), + clientRepositories.treeNodeRecordRepository(), clientRepositories.memberVersionRepository()); } - private StatePersistence createInMemoryStatePersistence() { - final SqliteProperties sqliteProperties = new SqliteProperties("instanceName", false); - MemberRepository memberRepository = MemberRepositoryFactory.getMemberRepository(StatePersistenceStrategy.MEMORY, - sqliteProperties, "instanceName"); - TreeNodeRecordRepository treeNodeRecordRepository = TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(StatePersistenceStrategy.MEMORY, sqliteProperties, "instanceName"); - MemberIdRepository memberIdRepository = MemberIdRepositoryFactory.getMemberIdRepository(StatePersistenceStrategy.MEMORY, - sqliteProperties, "instanceName"); - MemberVersionRepository memberVersionRepository = MemberVersionRepositoryFactory.getMemberVersionRepositoryFactory(StatePersistenceStrategy.MEMORY, - sqliteProperties, "instanceName"); - return new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository); + private LdesClientRepositories createInMemoryStatePersistence() { + LdesClientRepositories clientRepositories = LdesClientRepositories.memoryBased(); + + return new LdesClientRepositories(clientRepositories.memberRepository(), clientRepositories.memberIdRepository(), + clientRepositories.treeNodeRecordRepository(), clientRepositories.memberVersionRepository()); } } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java index 0c5c84ab8..c48638e7c 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/MemberSupplierSteps.java @@ -1,20 +1,16 @@ package ldes.client.treenodesupplier; +import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateUtil; import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromCurrentTimeExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import io.cucumber.java.Before; import io.cucumber.java.en.And; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; -import ldes.client.treenodesupplier.domain.services.MemberIdRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.MemberRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.MemberVersionRepositoryFactory; -import ldes.client.treenodesupplier.domain.services.TreeNodeRecordRepositoryFactory; import ldes.client.treenodesupplier.domain.valueobject.*; import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; import ldes.client.treenodesupplier.filters.LatestStateFilter; @@ -22,16 +18,13 @@ import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplierImpl; -import ldes.client.treenodesupplier.repository.MemberIdRepository; -import ldes.client.treenodesupplier.repository.MemberRepository; -import ldes.client.treenodesupplier.repository.MemberVersionRepository; -import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; import org.apache.jena.riot.Lang; import org.junit.After; import org.mockito.Mockito; import org.testcontainers.containers.PostgreSQLContainer; import java.util.List; +import java.util.UUID; import java.util.function.Consumer; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; @@ -41,10 +34,7 @@ public class MemberSupplierSteps { private final RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); private TreeNodeProcessor treeNodeProcessor; - private TreeNodeRecordRepository treeNodeRecordRepository; - private MemberRepository memberRepository; - private MemberIdRepository memberIdRepository; - private MemberVersionRepository memberVersionRepository; + private LdesClientRepositories ldesClientRepositories; private MemberSupplier memberSupplier; private LdesMetaData ldesMetaData; private SuppliedMember suppliedMember; @@ -77,12 +67,12 @@ public void iRequestOneMemberFromTheMemberSupplier() { @And("The TreeNode is not processed: {string}") public void theTreeNodeIsNotProcessed(String treeNodeId) { - assertFalse(treeNodeRecordRepository.existsById(treeNodeId)); + assertFalse(ldesClientRepositories.treeNodeRecordRepository().existsById(treeNodeId)); } @Then("Status {string} for TreeNodeRecord with identifier: {string}") public void statusForTreeNodeRecordWithIdentifier(String treeNodeStatus, String treeNodeId) { - assertTrue(treeNodeRecordRepository.existsByIdAndStatus(treeNodeId, TreeNodeStatus.valueOf(treeNodeStatus))); + assertTrue(ldesClientRepositories.treeNodeRecordRepository().existsByIdAndStatus(treeNodeId, TreeNodeStatus.valueOf(treeNodeStatus))); } @Given("A starting url {string}") @@ -103,7 +93,7 @@ public void setTimestampPath(String timestampPath) { @When("I create a Processor") public void iCreateAProcessor() { treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, - new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository), + ldesClientRepositories, requestExecutorFactory.createNoAuthExecutor(), timestampPath.isEmpty() ? new TimestampFromCurrentTimeExtractor() : new TimestampFromPathExtractor(createProperty(timestampPath)), clientStatusConsumer); @@ -115,34 +105,26 @@ public void memberIsProcessed(String memberId) { } @And("a StatePersistenceStrategy MEMORY") - public StatePersistence aMemoryStatePersistenceStrategy() { - memberRepository = MemberRepositoryFactory.getMemberRepository(StatePersistenceStrategy.MEMORY, null, - "instanceName"); - memberIdRepository = MemberIdRepositoryFactory.getMemberIdRepository(StatePersistenceStrategy.MEMORY, null, - "instanceName"); - treeNodeRecordRepository = TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(StatePersistenceStrategy.MEMORY, null, "instanceName"); - memberVersionRepository = MemberVersionRepositoryFactory.getMemberVersionRepositoryFactory(StatePersistenceStrategy.MEMORY, null, - "instanceName"); - return new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository); + public LdesClientRepositories aMemoryStatePersistenceStrategy() { + LdesClientRepositories clientRepositories = LdesClientRepositories.memoryBased(); + + ldesClientRepositories = new LdesClientRepositories(clientRepositories.memberRepository(), clientRepositories.memberIdRepository(), + clientRepositories.treeNodeRecordRepository(), clientRepositories.memberVersionRepository()); + return ldesClientRepositories; } @And("a StatePersistenceStrategy SQLITE") - public StatePersistence aSqliteStatePersistenceStrategy() { - final SqliteProperties sqliteProperties = new SqliteProperties("instanceName", false); - memberRepository = MemberRepositoryFactory.getMemberRepository(StatePersistenceStrategy.SQLITE, sqliteProperties, - "instanceName"); - memberIdRepository = MemberIdRepositoryFactory.getMemberIdRepository(StatePersistenceStrategy.SQLITE, sqliteProperties, - "instanceName"); - treeNodeRecordRepository = TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(StatePersistenceStrategy.SQLITE, sqliteProperties, "instanceName"); - memberVersionRepository = MemberVersionRepositoryFactory - .getMemberVersionRepositoryFactory(StatePersistenceStrategy.SQLITE, sqliteProperties, "instanceName"); - return new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository); + public LdesClientRepositories aSqliteStatePersistenceStrategy() { + final SqliteProperties sqliteProperties = new SqliteProperties("target", UUID.randomUUID().toString(), false); + LdesClientRepositories clientRepositories = LdesClientRepositories.sqlBased( + HibernateUtil.createEntityManagerFromProperties(sqliteProperties.getProperties())); + ldesClientRepositories = new LdesClientRepositories(clientRepositories.memberRepository(), clientRepositories.memberIdRepository(), + clientRepositories.treeNodeRecordRepository(), clientRepositories.memberVersionRepository()); + return ldesClientRepositories; } @And("a StatePersistenceStrategy POSTGRES") - public StatePersistence aPostgresStatePersistenceStrategy() { + public LdesClientRepositories aPostgresStatePersistenceStrategy() { postgreSQLContainer = new PostgreSQLContainer<>("postgres:11.1") .withDatabaseName("integration-test-client-persistence") .withUsername("sa") @@ -151,18 +133,12 @@ public StatePersistence aPostgresStatePersistenceStrategy() { PostgresProperties postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(), postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false); - memberRepository = MemberRepositoryFactory.getMemberRepository(StatePersistenceStrategy.POSTGRES, - postgresProperties, "instanceName"); - memberIdRepository = MemberIdRepositoryFactory.getMemberIdRepository(StatePersistenceStrategy.POSTGRES, - postgresProperties, "instanceName"); - treeNodeRecordRepository = TreeNodeRecordRepositoryFactory - .getTreeNodeRecordRepository(StatePersistenceStrategy.POSTGRES, postgresProperties, - "instanceName"); - memberVersionRepository = MemberVersionRepositoryFactory - .getMemberVersionRepositoryFactory(StatePersistenceStrategy.POSTGRES, postgresProperties, - "instanceName"); + LdesClientRepositories clientRepositories = LdesClientRepositories.sqlBased( + HibernateUtil.createEntityManagerFromProperties(postgresProperties.getProperties())); - return new StatePersistence(memberRepository, memberIdRepository, treeNodeRecordRepository, memberVersionRepository); + ldesClientRepositories = new LdesClientRepositories(clientRepositories.memberRepository(), clientRepositories.memberIdRepository(), + clientRepositories.treeNodeRecordRepository(), clientRepositories.memberVersionRepository()); + return ldesClientRepositories; } @Then("MemberSupplier is destroyed") @@ -187,11 +163,11 @@ public void iCreateAMemberSupplierWithoutState() { public void iCreateAMemberSupplierWithFilter() { final boolean keepState = false; memberSupplier = new FilteredMemberSupplier(new MemberSupplierImpl(treeNodeProcessor, keepState), - new ExactlyOnceFilter(memberIdRepository, keepState)); + new ExactlyOnceFilter(ldesClientRepositories.memberIdRepository(), keepState)); memberSupplier.init(); } - private StatePersistence defineStatePersistence(String persistenceStrategy) { + private LdesClientRepositories defineStatePersistence(String persistenceStrategy) { return switch (persistenceStrategy) { case "POSTGRES" -> aPostgresStatePersistenceStrategy(); case "SQLITE" -> aSqliteStatePersistenceStrategy(); @@ -243,7 +219,7 @@ public void membersuppliersAreDestroyed() { public void iCreateAMemberSupplierWithLatestStateFilter() { final boolean keepState = false; final MemberSupplierImpl baseMemberSupplier = new MemberSupplierImpl(treeNodeProcessor, keepState); - final MemberFilter latestStateFilter = new LatestStateFilter(memberVersionRepository, keepState, timestampPath, "http://purl.org/dc/terms/isVersionOf"); + final MemberFilter latestStateFilter = new LatestStateFilter(ldesClientRepositories.memberVersionRepository(), keepState, timestampPath, "http://purl.org/dc/terms/isVersionOf"); memberSupplier = new FilteredMemberSupplier(baseMemberSupplier, latestStateFilter); memberSupplier.init(); } diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistenceStrategyTest.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/valueobject/LdesClientRepositoriesStrategyTest.java similarity index 94% rename from ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistenceStrategyTest.java rename to ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/valueobject/LdesClientRepositoriesStrategyTest.java index bd0deddf9..773bd0569 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/valueobject/StatePersistenceStrategyTest.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/domain/valueobject/LdesClientRepositoriesStrategyTest.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; -class StatePersistenceStrategyTest { +class LdesClientRepositoriesStrategyTest { @ParameterizedTest @EnumSource(StatePersistenceStrategy.class) diff --git a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepositoryTest.java b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepositoryTest.java index d8e8dab0d..fcc5a2b48 100644 --- a/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepositoryTest.java +++ b/ldi-core/ldes-client/tree-node-supplier/src/test/java/ldes/client/treenodesupplier/repository/sql/SqlMemberVersionRepositoryTest.java @@ -1,8 +1,8 @@ package ldes.client.treenodesupplier.repository.sql; -import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.MemberVersionRecordEntity; import ldes.client.treenodesupplier.domain.entities.MemberVersionRecord; +import ldes.client.treenodesupplier.repository.MemberVersionRepository; import ldes.client.treenodesupplier.repository.mapper.MemberVersionRecordEntityMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -23,24 +23,18 @@ @ExtendWith(MockitoExtension.class) class SqlMemberVersionRepositoryTest { - @Mock - private EntityManagerFactory entityManagerFactory; - @Mock private EntityManager entityManager; - private final String instanceName = "testInstance"; - - private SqlMemberVersionRepository repository; + private MemberVersionRepository repository; @BeforeEach void setUp() { - repository = new SqlMemberVersionRepository(entityManagerFactory, instanceName); + repository = new SqlMemberVersionRepository(entityManager); } @Test void test_addMemberVersion() { - when(entityManagerFactory.getEntityManager()).thenReturn(entityManager); MemberVersionRecord actual = new MemberVersionRecord("versionOf", LocalDateTime.now()); MemberVersionRecordEntity entity = MemberVersionRecordEntityMapper.fromMemberVersionRecord(actual); @@ -51,7 +45,6 @@ void test_addMemberVersion() { @Test void whenNoOlderVersionPresent_isVersionAfterTimestampIsTrue() { - when(entityManagerFactory.getEntityManager()).thenReturn(entityManager); TypedQuery mockedQuery = mock(); when(entityManager.createNamedQuery("MemberVersion.findMemberVersionAfterTimestamp", MemberVersionRecordEntity.class)).thenReturn(mockedQuery); when(mockedQuery.setParameter(anyString(), any())).thenReturn(mockedQuery); @@ -65,7 +58,6 @@ void whenNoOlderVersionPresent_isVersionAfterTimestampIsTrue() { @Test void whenOlderVersionIsPresent_isVersionAfterTimestampIsFalse() { - when(entityManagerFactory.getEntityManager()).thenReturn(entityManager); final String versionOf = "versionOf"; MemberVersionRecordEntity olderVersion = new MemberVersionRecordEntity(versionOf, LocalDateTime.now().minusDays(1)); MemberVersionRecord memberVersion = new MemberVersionRecord(versionOf, LocalDateTime.now()); @@ -79,11 +71,4 @@ void whenOlderVersionIsPresent_isVersionAfterTimestampIsFalse() { assertThat(actual).isFalse(); } - - @Test - void test_destroyState() { - repository.destroyState(); - - verify(entityManagerFactory).destroyState(instanceName); - } } \ No newline at end of file diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index c4aa1970f..f47f92b2b 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT ldi-common diff --git a/ldi-core/ldi-infra-sql/pom.xml b/ldi-core/ldi-infra-sql/pom.xml index 8f6ab6389..ef8489ea6 100644 --- a/ldi-core/ldi-infra-sql/pom.xml +++ b/ldi-core/ldi-infra-sql/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT ldi-infra-sql diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/AbstractEntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/AbstractEntityManagerFactory.java deleted file mode 100644 index 655706ee7..000000000 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/AbstractEntityManagerFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi; - -import org.hibernate.Session; -import org.hibernate.StatelessSession; - -import javax.persistence.EntityManager; - -public abstract class AbstractEntityManagerFactory implements EntityManagerFactory { - private final javax.persistence.EntityManagerFactory entityManagerFactory; - private final EntityManager entityManager; - - protected AbstractEntityManagerFactory(javax.persistence.EntityManagerFactory entityManagerFactory) { - this.entityManagerFactory = entityManagerFactory; - this.entityManager = entityManagerFactory.createEntityManager(); - } - - @Override - public EntityManager getEntityManager() { - return entityManager; - } - - @Override - public int executeStatelessQuery(StatelessQueryExecutor queryExecutor) { - final Session session = entityManager.unwrap(Session.class); - return session.doReturningWork(connection -> { - try (final StatelessSession statelessSession = session.getSessionFactory().openStatelessSession(connection)) { - return queryExecutor.execute(statelessSession); - } - }); - } - - @Override - public void destroyState(String instanceName) { - entityManager.close(); - entityManagerFactory.close(); - } -} diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java deleted file mode 100644 index b9822568e..000000000 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/EntityManagerFactory.java +++ /dev/null @@ -1,22 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi; - -import javax.persistence.EntityManager; - -/** - * Custom interface to manage the EntityManager, EntityManagerFactory from the javax library and their lifecycle more easily - */ -public interface EntityManagerFactory { - /** - * @return the enityManager that is managed by the javax library - */ - EntityManager getEntityManager(); - - int executeStatelessQuery(StatelessQueryExecutor queryExecutor); - - /** - * Destroy the EntityManager and EntityManagerFactory from the javax library for the specified instanceName - * - * @param instanceName name of the instances from which the EntityManager and EntityManagerFactory must be destroyed - */ - void destroyState(String instanceName); -} diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java index 5fe148755..8d7655b2e 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateProperties.java @@ -7,6 +7,8 @@ public interface HibernateProperties { String HIBERNATE_CONNECTION_URL = "hibernate.connection.url"; String HIBERNATE_CONNECTION_USERNAME = "hibernate.connection.username"; String HIBERNATE_CONNECTION_PASSWORD = "hibernate.connection.password"; + String HIBERNATE_DATASOURCE = "hibernate.connection.datasource"; + String HIBERNATE_DIALECT = "hibernate.dialect"; String HIBERNATE_HBM_2_DDL_AUTO = "hibernate.hbm2ddl.auto"; String UPDATE = "update"; String CREATE_DROP = "create-drop"; diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateUtil.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateUtil.java new file mode 100644 index 000000000..0963f0c3c --- /dev/null +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/HibernateUtil.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; + +import javax.persistence.EntityManager; +import javax.persistence.Persistence; +import javax.sql.DataSource; +import java.util.Map; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties.*; + +public class HibernateUtil { + private HibernateUtil() { + } + + public static EntityManager createEntityManagerFromDatasource(DataSource dataSource, boolean keepState, StatePersistenceStrategy state) { + var entityManagerFactory = Persistence.createEntityManagerFactory("pu-sql-jpa", + Map.of(HIBERNATE_DATASOURCE, dataSource, + HIBERNATE_HBM_2_DDL_AUTO, keepState ? UPDATE : CREATE_DROP, + HIBERNATE_DIALECT, getHibernateDialect(state))); + return entityManagerFactory.createEntityManager(); + } + + public static EntityManager createEntityManagerFromProperties(Map properties) { + var entityManagerFactory = Persistence.createEntityManagerFactory("pu-sql-jpa", properties); + return entityManagerFactory.createEntityManager(); + } + + public static String getHibernateDialect(StatePersistenceStrategy state) { + return switch (state) { + case POSTGRES -> PostgresProperties.DIALECT; + case SQLITE -> SqliteProperties.DIALECT; + default -> ""; + }; + } +} diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/HashedStateMemberEntity.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/HashedStateMemberEntity.java index 57af6cf72..4b16756e0 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/HashedStateMemberEntity.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/HashedStateMemberEntity.java @@ -3,9 +3,9 @@ import javax.persistence.*; @Entity -@Table +@Table(name = "member_hashed") @NamedQuery(name = "HashedStateMember.findMember", query = "FROM HashedStateMemberEntity m WHERE m.id = :memberId AND m.hash = :memberHash") -@NamedNativeQuery(name = "HashedStateMember.insert", query = "INSERT INTO HashedStateMemberEntity(id, hash) VALUES (?, ?) ON CONFLICT DO NOTHING") +@NamedNativeQuery(name = "HashedStateMember.insert", query = "INSERT INTO member_hashed(id, hash) VALUES (?, ?) ON CONFLICT DO NOTHING") public class HashedStateMemberEntity { @Id private String id; diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberIdRecordEntity.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberIdEntity.java similarity index 51% rename from ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberIdRecordEntity.java rename to ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberIdEntity.java index 1812d3c5d..dba844699 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberIdRecordEntity.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberIdEntity.java @@ -6,9 +6,21 @@ import javax.persistence.Table; @Entity -@Table -@NamedNativeQuery(name = "MemberId.insert", query = "INSERT INTO MemberIdRecordEntity(id) VALUES (:memberId) ON CONFLICT DO NOTHING") -public class MemberIdRecordEntity { +@Table(name = "member_id") +@NamedNativeQuery(name = "MemberId.insert", query = "INSERT INTO member_id(id) VALUES (:memberId) ON CONFLICT DO NOTHING") +public class MemberIdEntity { @Id private String id; + + public MemberIdEntity() { + // Default constructor needed for JPA + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberRecordEntity.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberRecordEntity.java index 94359b4f0..e2a7c7d5b 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberRecordEntity.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberRecordEntity.java @@ -4,7 +4,7 @@ import java.time.LocalDateTime; @Entity -@Table(indexes = { +@Table(name = "member", indexes = { @Index(name = "idx_members_created_at", columnList = "createdAt"), @Index(name = "idx_members_member_id", columnList = "memberId") }) @@ -18,13 +18,19 @@ public class MemberRecordEntity { @Column(columnDefinition = "text", length = 10485760) private String memberId; private LocalDateTime createdAt; - @Column(name = "model", columnDefinition = "bytea", nullable = false) private byte[] bytes; public MemberRecordEntity() { } + public MemberRecordEntity(int id, String memberId, LocalDateTime dateCreated, byte[] bytes) { + this.id = id; + this.memberId = memberId; + this.createdAt = dateCreated; + this.bytes = bytes; + } + public MemberRecordEntity(String memberId, LocalDateTime dateCreated, byte[] bytes) { this.memberId = memberId; this.createdAt = dateCreated; @@ -35,6 +41,10 @@ public int getId() { return id; } + public void setId(int id) { + this.id = id; + } + public String getMemberId() { return memberId; } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberVersionRecordEntity.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberVersionRecordEntity.java index cecf59306..a2e0e6364 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberVersionRecordEntity.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/MemberVersionRecordEntity.java @@ -4,7 +4,7 @@ import java.time.LocalDateTime; @Entity -@Table() +@Table(name = "member_version") @NamedQuery(name = "MemberVersion.findMemberVersionAfterTimestamp", query = "SELECT m FROM MemberVersionRecordEntity m WHERE versionOf = :versionOf AND timestamp >= :timestamp") public class MemberVersionRecordEntity { @Id @@ -24,4 +24,16 @@ public MemberVersionRecordEntity(String versionOf, LocalDateTime timestamp) { public String getVersionOf() { return versionOf; } + + public void setVersionOf(String versionOf) { + this.versionOf = versionOf; + } + + public LocalDateTime getTimestamp() { + return timestamp; + } + + public void setTimestamp(LocalDateTime timestamp) { + this.timestamp = timestamp; + } } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/TreeNodeRecordEntity.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/TreeNodeRecordEntity.java index cd2c6e09d..1da3e5aec 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/TreeNodeRecordEntity.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/TreeNodeRecordEntity.java @@ -5,9 +5,9 @@ import java.util.List; @Entity -@Table(indexes = { - @Index(name="treenoderecordentity_treenodeurl_treenodestatus", columnList = "treeNodeUrl,treeNodeStatus"), - @Index(name="treenoderecordentity_treenodestatus_earliestnextvisit", columnList = "treeNodeStatus,earliestNextVisit") +@Table(name = "treenode", indexes = { + @Index(name = "treenoderecordentity_treenodeurl_treenodestatus", columnList = "treeNodeUrl,treeNodeStatus"), + @Index(name = "treenoderecordentity_treenodestatus_earliestnextvisit", columnList = "treeNodeStatus,earliestNextVisit") }) @NamedQuery(name = "TreeNode.getAll", query = "SELECT t FROM TreeNodeRecordEntity t") @NamedQuery(name = "TreeNode.getById", query = "SELECT t FROM TreeNodeRecordEntity t WHERE t.id = :id") @@ -21,7 +21,7 @@ public class TreeNodeRecordEntity { private String treeNodeStatus; private LocalDateTime earliestNextVisit; @Column - @ElementCollection(targetClass=String.class) + @ElementCollection(targetClass = String.class) private List members; public TreeNodeRecordEntity() { @@ -49,4 +49,20 @@ public LocalDateTime getEarliestNextVisit() { public List getMembers() { return members; } + + public void setTreeNodeUrl(String treeNodeUrl) { + this.treeNodeUrl = treeNodeUrl; + } + + public void setTreeNodeStatus(String treeNodeStatus) { + this.treeNodeStatus = treeNodeStatus; + } + + public void setEarliestNextVisit(LocalDateTime earliestNextVisit) { + this.earliestNextVisit = earliestNextVisit; + } + + public void setMembers(List members) { + this.members = members; + } } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java deleted file mode 100644 index 378c136c4..000000000 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.postgres; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.AbstractEntityManagerFactory; - -import javax.persistence.Persistence; -import java.util.HashMap; -import java.util.Map; - -public class PostgresEntityManagerFactory extends AbstractEntityManagerFactory { - public static final String PERSISTENCE_UNIT_NAME = "pu-postgres-jpa"; - private static final Map instances = new HashMap<>(); - - - private PostgresEntityManagerFactory(Map properties) { - super(Persistence.createEntityManagerFactory(PERSISTENCE_UNIT_NAME, properties)); - } - - public static synchronized PostgresEntityManagerFactory getInstance(String instanceName, Map properties) { - return instances.computeIfAbsent(instanceName, s -> new PostgresEntityManagerFactory(properties)); - } - - - @Override - public void destroyState(String instanceName) { - super.destroyState(instanceName); - instances.remove(instanceName); - // DELETE tables - } -} diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresProperties.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresProperties.java index c7cbe3ced..a6625d0d2 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresProperties.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresProperties.java @@ -5,6 +5,7 @@ import java.util.Map; public class PostgresProperties implements HibernateProperties { + public static final String DIALECT = "org.hibernate.dialect.PostgreSQLDialect"; private final String url; private final String username; private final String password; @@ -21,6 +22,7 @@ public Map getProperties() { return Map.of(HIBERNATE_CONNECTION_URL, url, HIBERNATE_CONNECTION_USERNAME, username, HIBERNATE_CONNECTION_PASSWORD, password, + HIBERNATE_DIALECT, DIALECT, HIBERNATE_HBM_2_DDL_AUTO, keepState ? UPDATE : CREATE_DROP); } diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/CreateDirectoryFailedException.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/CreateDirectoryFailedException.java deleted file mode 100644 index 4148da6b7..000000000 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/CreateDirectoryFailedException.java +++ /dev/null @@ -1,7 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite; - -public class CreateDirectoryFailedException extends RuntimeException { - public CreateDirectoryFailedException(Throwable cause) { - super(cause); - } -} diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java deleted file mode 100644 index f01ac2d43..000000000 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.AbstractEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import org.apache.commons.io.FileUtils; - -import javax.persistence.Persistence; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; - -public class SqliteEntityManagerFactory extends AbstractEntityManagerFactory { - public static final String PERSISTENCE_UNIT_NAME = "pu-sqlite-jpa"; - private final SqliteProperties properties; - private static final Map instances = new HashMap<>(); - - private SqliteEntityManagerFactory(SqliteProperties properties) { - super(Persistence.createEntityManagerFactory(PERSISTENCE_UNIT_NAME, properties.getProperties())); - this.properties = properties; - } - - public static synchronized SqliteEntityManagerFactory getInstance(HibernateProperties hibernateProperties) { - if (!(hibernateProperties instanceof SqliteProperties properties)) { - throw new IllegalArgumentException("Invalid properties for SqliteEntityManagerFactory provided"); - } - return instances.computeIfAbsent(properties.getInstanceName(), s -> { - try { - Files.createDirectories(Paths.get(properties.getDatabaseDirectory())); - } catch (IOException e) { - throw new CreateDirectoryFailedException(e); - } - return new SqliteEntityManagerFactory(properties); - }); - } - - @Override - public void destroyState(String instanceName) { - super.destroyState(instanceName); - instances.remove(instanceName); - FileUtils.deleteQuietly(new File(properties.getDatabaseDirectory(), properties.getDatabaseName())); - } -} diff --git a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteProperties.java b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteProperties.java index 08aa60a05..8db646868 100644 --- a/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteProperties.java +++ b/ldi-core/ldi-infra-sql/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteProperties.java @@ -5,8 +5,8 @@ import java.util.Map; public class SqliteProperties implements HibernateProperties { - public static final String DATABASE_DIRECTORY = "sqlite"; - + public static final String DATABASE_DIRECTORY = "."; + public static final String DIALECT = "org.sqlite.hibernate.dialect.SQLiteDialect"; private final String databaseDirectory; private final String instanceName; private final boolean keepState; @@ -37,6 +37,9 @@ public String getDatabaseName() { public Map getProperties() { return Map.of("javax.persistence.jdbc.url", "jdbc:sqlite:./%s/%s".formatted(databaseDirectory, getDatabaseName()), + HIBERNATE_DIALECT, DIALECT, + "javax.persistence.jdbc.driver", "org.sqlite.JDBC", + "hibernate.connection.provider_class", "com.zaxxer.hikari.hibernate.HikariConnectionProvider", HIBERNATE_HBM_2_DDL_AUTO, keepState ? UPDATE : CREATE_DROP); } } diff --git a/ldi-core/ldi-infra-sql/src/main/resources/META-INF/persistence.xml b/ldi-core/ldi-infra-sql/src/main/resources/META-INF/persistence.xml index 3da065eca..26bc20414 100644 --- a/ldi-core/ldi-infra-sql/src/main/resources/META-INF/persistence.xml +++ b/ldi-core/ldi-infra-sql/src/main/resources/META-INF/persistence.xml @@ -3,29 +3,11 @@ xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd" version="2.0"> - - - org.hibernate.ejb.HibernatePersistence - - - - - - - - - - - - - - - + org.hibernate.ejb.HibernatePersistence - - + diff --git a/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactoryTest.java b/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactoryTest.java deleted file mode 100644 index ce3b1e474..000000000 --- a/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/postgres/PostgresEntityManagerFactoryTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.postgres; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; - -import static org.assertj.core.api.Assertions.assertThat; - -class PostgresEntityManagerFactoryTest { - public static final String INSTANCE_NAME = "instance"; - private static PostgreSQLContainer postgreSQLContainer; - private static PostgresProperties postgresProperties; - - @BeforeAll - static void beforeAll() { - postgreSQLContainer = new PostgreSQLContainer<>("postgres:11.1") - .withDatabaseName("integration-change-detection-filter-persistence") - .withUsername("sa") - .withPassword("sa"); - postgreSQLContainer.start(); - - postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(), postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false); - } - - @AfterAll - static void afterAll() { - postgreSQLContainer.stop(); - } - - @AfterEach - void tearDown() { - PostgresEntityManagerFactory.getInstance(INSTANCE_NAME, postgresProperties.getProperties()).destroyState(INSTANCE_NAME); - } - - @Test - void when_GetInstance_then_SameSingletonInstanceIsReturned() { - final PostgresEntityManagerFactory firstInstance = PostgresEntityManagerFactory.getInstance(INSTANCE_NAME, postgresProperties.getProperties()); - final PostgresEntityManagerFactory secondInstance = PostgresEntityManagerFactory.getInstance(INSTANCE_NAME, postgresProperties.getProperties()); - - assertThat(firstInstance).isSameAs(secondInstance); - } - - @Test - void test_DestroyState() { - final PostgresEntityManagerFactory firstInstance = PostgresEntityManagerFactory.getInstance(INSTANCE_NAME, postgresProperties.getProperties()); - firstInstance.destroyState(INSTANCE_NAME); - - assertThat(firstInstance.getEntityManager().isOpen()).isFalse(); - - final PostgresEntityManagerFactory secondInstance = PostgresEntityManagerFactory.getInstance(INSTANCE_NAME, postgresProperties.getProperties()); - - assertThat(firstInstance).isNotSameAs(secondInstance); - } -} \ No newline at end of file diff --git a/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactoryTest.java b/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactoryTest.java deleted file mode 100644 index 8dbe86f19..000000000 --- a/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqliteEntityManagerFactoryTest.java +++ /dev/null @@ -1,41 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite; - -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class SqliteEntityManagerFactoryTest { - private static final SqliteProperties sqliteProperties = new SqliteProperties("instanceName", false); - - @Test - void given_InvalidProperties_when_GetInstance_then_ThrowException() { - assertThatThrownBy(() -> SqliteEntityManagerFactory.getInstance(Map::of)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid properties for SqliteEntityManagerFactory provided"); - } - - @Test - void when_GetInstance_then_SameSingletonInstanceIsReturned() { - final SqliteEntityManagerFactory firstInstance = SqliteEntityManagerFactory.getInstance(sqliteProperties); - final SqliteEntityManagerFactory secondInstance = SqliteEntityManagerFactory.getInstance(sqliteProperties); - - assertThat(firstInstance).isSameAs(secondInstance); - } - - @Test - void test_DestroyState() { - final SqliteEntityManagerFactory firstInstance = SqliteEntityManagerFactory.getInstance(sqliteProperties); - firstInstance.destroyState(sqliteProperties.getInstanceName()); - - assertThat(firstInstance.getEntityManager().isOpen()).isFalse(); - assertThat(new File(sqliteProperties.getDatabaseDirectory(), sqliteProperties.getDatabaseName())).doesNotExist(); - - final SqliteEntityManagerFactory secondInstance = SqliteEntityManagerFactory.getInstance(sqliteProperties); - - assertThat(firstInstance).isNotSameAs(secondInstance); - } -} \ No newline at end of file diff --git a/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqlitePropertiesTest.java b/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqlitePropertiesTest.java index e22f36c2a..389d8caae 100644 --- a/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqlitePropertiesTest.java +++ b/ldi-core/ldi-infra-sql/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/sqlite/SqlitePropertiesTest.java @@ -40,7 +40,7 @@ void test_GetProperties() { Map result = sqliteProperties.getProperties(); assertThat(result) - .containsEntry("javax.persistence.jdbc.url", "jdbc:sqlite:./sqlite/" + INSTANCE_NAME + ".db") + .containsEntry("javax.persistence.jdbc.url", "jdbc:sqlite:././" + INSTANCE_NAME + ".db") .containsEntry(SqliteProperties.HIBERNATE_HBM_2_DDL_AUTO, SqliteProperties.UPDATE); } } diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 60e69a6c8..4916865cf 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index aebca2025..509361483 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index cd5001b4d..4cb59ede5 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/repository-sink/pom.xml b/ldi-core/repository-sink/pom.xml index 8b387d4d6..6597eddf5 100644 --- a/ldi-core/repository-sink/pom.xml +++ b/ldi-core/repository-sink/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index f985acd85..48248fc92 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index cee6ba54a..495a19455 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/skolemisation-transformer/pom.xml b/ldi-core/skolemisation-transformer/pom.xml index 61a6c9d69..cc06c9e6c 100644 --- a/ldi-core/skolemisation-transformer/pom.xml +++ b/ldi-core/skolemisation-transformer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT skolemisation-transformer diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index f9ef86563..4d644d089 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index b1f2c2418..d5156cc00 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index 532e2dd2b..a255c9a33 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index 20542d040..7bb62d6c9 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 2.12.0 + 2.13.0-SNAPSHOT ldes-discoverer diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index b8a6c14c7..902881ef2 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.12.0 + 2.13.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index 32bce292b..27d4811a8 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,13 +3,34 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 ldi-nifi-common + + 2.1.0 + + + + org.apache.nifi + nifi-dbcp-service-api + ${nifi-dbcp.version} + + + org.apache.nifi + nifi-standard-services-api-nar + ${nifi-dbcp.version} + nar + + + org.apache.nifi + nifi-dbcp-service + ${nifi-dbcp.version} + + org.apache.jena jena-arq diff --git a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/PersistenceProperties.java b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/PersistenceProperties.java index 602f1309d..f0b9490a2 100644 --- a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/PersistenceProperties.java +++ b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/PersistenceProperties.java @@ -2,11 +2,11 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.dbcp.DBCPService; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; import java.util.Arrays; -import java.util.Optional; import java.util.stream.Collectors; import static java.lang.Boolean.FALSE; @@ -25,41 +25,12 @@ private PersistenceProperties() { .defaultValue(StatePersistenceStrategy.MEMORY.name()) .build(); - public static final PropertyDescriptor POSTGRES_URL = new PropertyDescriptor.Builder() - .name("POSTGRES_URL") - .displayName("Postgres database url") - .description("Postgres database url formatted as \"jdbc:postgresql://localhost:5432/postgres\"") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .dependsOn(STATE_PERSISTENCE_STRATEGY, StatePersistenceStrategy.POSTGRES.name()) - .build(); - - public static final PropertyDescriptor POSTGRES_USERNAME = new PropertyDescriptor.Builder() - .name("POSTGRES_USERNAME") - .displayName("Postgres database username") - .description("Username used to connect to the postgres database") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .dependsOn(STATE_PERSISTENCE_STRATEGY, StatePersistenceStrategy.POSTGRES.name()) - .build(); - - public static final PropertyDescriptor POSTGRES_PASSWORD = new PropertyDescriptor.Builder() - .name("POSTGRES_PASSWORD") - .displayName("Postgres database password") - .description("Password used to connect to the postgres database") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .sensitive(true) - .dependsOn(STATE_PERSISTENCE_STRATEGY, StatePersistenceStrategy.POSTGRES.name()) - .build(); - - public static final PropertyDescriptor SQLITE_DIRECTORY = new PropertyDescriptor.Builder() - .name("SQLITE_DIRECTORY") - .displayName("Sqlite database directory") - .description("Sqlite database directory where the '.db' file can be stored") - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .dependsOn(STATE_PERSISTENCE_STRATEGY, StatePersistenceStrategy.SQLITE.name()) + public static final PropertyDescriptor DBCP_SERVICE = new PropertyDescriptor.Builder() + .name("Database Connection Pooling Service") + .description("The Controller Service that is used to obtain connection to database") + .identifiesControllerService(DBCPService.class) + .required(true) + .dependsOn(STATE_PERSISTENCE_STRATEGY, StatePersistenceStrategy.POSTGRES.name(), StatePersistenceStrategy.SQLITE.name()) .build(); public static final PropertyDescriptor KEEP_STATE = new PropertyDescriptor.Builder() @@ -81,21 +52,4 @@ public static boolean stateKept(final ProcessContext context) { return TRUE.equals(context.getProperty(KEEP_STATE).asBoolean()); } - - public static String getPostgresUrl(final ProcessContext context) { - return context.getProperty(POSTGRES_URL).getValue(); - } - - public static String getPostgresUsername(final ProcessContext context) { - return context.getProperty(POSTGRES_USERNAME).getValue(); - } - - public static String getPostgresPassword(final ProcessContext context) { - return context.getProperty(POSTGRES_PASSWORD).getValue(); - } - - public static Optional getSqliteDirectory(final ProcessContext context) { - return Optional.ofNullable(context.getProperty(SQLITE_DIRECTORY).getValue()); - } - } diff --git a/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/NiFiDBCPDataSource.java b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/NiFiDBCPDataSource.java new file mode 100644 index 000000000..2767586da --- /dev/null +++ b/ldi-nifi/ldi-nifi-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/services/NiFiDBCPDataSource.java @@ -0,0 +1,64 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services; + +import org.apache.nifi.dbcp.DBCPService; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.logging.Logger; + +public class NiFiDBCPDataSource implements DataSource { + private static final String NOT_SUPPORTED = "Not supported"; + + private final DBCPService dbcpService; + + public NiFiDBCPDataSource(DBCPService dbcpService) { + this.dbcpService = dbcpService; + } + + @Override + public Connection getConnection() { + return dbcpService.getConnection(); + } + + @Override + public Connection getConnection(String username, String password) { + throw new UnsupportedOperationException("DBCPService does not support username/password connections"); + } + + @Override + public PrintWriter getLogWriter() { + throw new UnsupportedOperationException(NOT_SUPPORTED); + } + + @Override + public void setLogWriter(PrintWriter out) { + throw new UnsupportedOperationException(NOT_SUPPORTED); + } + + @Override + public int getLoginTimeout() { + throw new UnsupportedOperationException(NOT_SUPPORTED); + } + + @Override + public void setLoginTimeout(int seconds) { + throw new UnsupportedOperationException(NOT_SUPPORTED); + } + + @Override + public Logger getParentLogger() { + throw new UnsupportedOperationException(NOT_SUPPORTED); + } + + @Override + public T unwrap(Class iface) throws SQLException { + throw new SQLException("Not a wrapper"); + } + + @Override + public boolean isWrapperFor(Class iface) { + return false; + } +} \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index bd8ed33be..113b3277e 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index ed6d3df42..2973f1494 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml index cf013b261..b81d3bb20 100644 --- a/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/change-detection-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT change-detection-processor @@ -35,6 +35,11 @@ ${testcontainers-postgresql.version} test + + org.mockito + mockito-junit-jupiter + test + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessor.java b/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessor.java index b39d80bd0..76b5d3399 100644 --- a/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessor.java @@ -2,7 +2,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.ChangeDetectionFilter; -import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; import org.apache.jena.rdf.model.Model; @@ -44,10 +43,7 @@ protected List getSupportedPropertyDescriptors() { return List.of( DATA_SOURCE_FORMAT, STATE_PERSISTENCE_STRATEGY, - POSTGRES_URL, - POSTGRES_USERNAME, - POSTGRES_PASSWORD, - SQLITE_DIRECTORY, + DBCP_SERVICE, KEEP_STATE ); } @@ -55,8 +51,7 @@ protected List getSupportedPropertyDescriptors() { @OnScheduled public void onScheduled(final ProcessContext context) { final HashedStateMemberRepository repository = HashedMemberRepositoryFactory.getRepository(context); - final boolean keepState = PersistenceProperties.stateKept(context); - changeDetectionFilter = new ChangeDetectionFilter(repository, keepState); + changeDetectionFilter = new ChangeDetectionFilter(repository); } @Override @@ -85,6 +80,6 @@ public void onTrigger(final ProcessContext context, final ProcessSession session @OnRemoved public void onRemoved() { - changeDetectionFilter.destroyState(); + changeDetectionFilter.close(); } } diff --git a/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HashedMemberRepositoryFactory.java b/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HashedMemberRepositoryFactory.java index 8501bc7ef..f46c2462b 100644 --- a/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HashedMemberRepositoryFactory.java +++ b/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/HashedMemberRepositoryFactory.java @@ -1,18 +1,16 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateUtil; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.NiFiDBCPDataSource; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.inmemory.InMemoryHashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql.SqlHashedStateMemberRepository; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; +import org.apache.nifi.dbcp.DBCPService; import org.apache.nifi.processor.ProcessContext; -import java.util.Map; - +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.DBCP_SERVICE; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.getStatePersistenceStrategy; public class HashedMemberRepositoryFactory { @@ -21,34 +19,18 @@ private HashedMemberRepositoryFactory() { public static HashedStateMemberRepository getRepository(ProcessContext context) { StatePersistenceStrategy state = getStatePersistenceStrategy(context); - return switch (state) { - case POSTGRES -> createPostgresRepository(context); - case SQLITE -> createSqliteRepository(context); - case MEMORY -> new InMemoryHashedStateMemberRepository(); - }; - } - private static SqlHashedStateMemberRepository createPostgresRepository(ProcessContext context) { - final String instanceName = context.getName(); - final var entityManagerFactory = PostgresEntityManagerFactory.getInstance(instanceName, createPostgresProperties(context)); - return new SqlHashedStateMemberRepository(entityManagerFactory, instanceName); - } + return switch (state) { + case POSTGRES, SQLITE -> { + final DBCPService dbcpService = context.getProperty(DBCP_SERVICE).asControllerService(DBCPService.class); + final NiFiDBCPDataSource dataSource = new NiFiDBCPDataSource(dbcpService); + final boolean keepState = PersistenceProperties.stateKept(context); - private static Map createPostgresProperties(ProcessContext context) { - String url = PersistenceProperties.getPostgresUrl(context); - String username = PersistenceProperties.getPostgresUsername(context); - String password = PersistenceProperties.getPostgresPassword(context); - boolean keepState = PersistenceProperties.stateKept(context); - return new PostgresProperties(url, username, password, keepState).getProperties(); - } + var entityManager = HibernateUtil.createEntityManagerFromDatasource(dataSource, keepState, state); - private static SqlHashedStateMemberRepository createSqliteRepository(ProcessContext context) { - final String instanceName = context.getName(); - final boolean keepState = PersistenceProperties.stateKept(context); - final SqliteProperties sqliteProperties = PersistenceProperties.getSqliteDirectory(context) - .map(directory -> new SqliteProperties(directory, instanceName, keepState)) - .orElseGet(() -> new SqliteProperties(instanceName, keepState)); - final var entityManagerFactory = SqliteEntityManagerFactory.getInstance(sqliteProperties); - return new SqlHashedStateMemberRepository(entityManagerFactory, instanceName); + yield new SqlHashedStateMemberRepository(entityManager); + } + case MEMORY -> new InMemoryHashedStateMemberRepository(); + }; } } diff --git a/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessorTest.java b/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessorTest.java index 729ad5322..f7403ae12 100644 --- a/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessorTest.java +++ b/ldi-nifi/ldi-nifi-processors/change-detection-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/ChangeDetectionFilterProcessorTest.java @@ -1,6 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.dbcp.DBCPConnectionPool; +import org.apache.nifi.dbcp.utils.DBCPProperties; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; import org.junit.jupiter.api.AfterAll; @@ -54,9 +57,11 @@ void tearDown() { @ParameterizedTest @ArgumentsSource(PropertiesProvider.class) - void test_Filter(Map properties) { + void test_Filter(Map properties) throws InitializationException { properties.forEach(testRunner::setProperty); + setupDBCPforState(properties.get(STATE_PERSISTENCE_STRATEGY)); + testRunner.enqueue(createInputStreamForFile("members/state-member.nq")); testRunner.enqueue(createInputStreamForFile("members/state-member.ttl"), Map.of("mime.type", "text/turtle")); testRunner.enqueue(createInputStreamForFile("members/changed-state-member.nq")); @@ -73,6 +78,36 @@ void test_Filter(Map properties) { .isEqualTo(1); } + private void setupDBCPforState(String state) throws InitializationException { + // Initialize the DBCPService with H2 in-memory database + DBCPConnectionPool dbcpService; + + switch (state) { + case "POSTGRES": + dbcpService = new DBCPConnectionPool(); + testRunner.addControllerService("dbcpService", dbcpService); + // Set the DBCPService properties for Postgres database + testRunner.setProperty(dbcpService, DBCPProperties.DATABASE_URL, postgreSQLContainer.getJdbcUrl()); + testRunner.setProperty(dbcpService, DBCPProperties.DB_USER, postgreSQLContainer.getUsername()); + testRunner.setProperty(dbcpService, DBCPProperties.DB_PASSWORD, postgreSQLContainer.getPassword()); + testRunner.setProperty(dbcpService, DBCPProperties.DB_DRIVERNAME, "org.postgresql.Driver"); + testRunner.enableControllerService(dbcpService); + testRunner.setProperty(DBCP_SERVICE, "dbcpService"); + break; + case "SQLITE": + dbcpService = new DBCPConnectionPool(); + testRunner.addControllerService("dbcpService", dbcpService); + // Set the DBCPService properties for SQLite database + testRunner.setProperty(dbcpService, DBCPProperties.DATABASE_URL, "jdbc:sqlite:./test.db"); + testRunner.setProperty(dbcpService, DBCPProperties.DB_DRIVERNAME, "org.sqlite.JDBC"); + testRunner.enableControllerService(dbcpService); + testRunner.setProperty(DBCP_SERVICE, "dbcpService"); + break; + default: + // No need to setup DBCPService for MEMORY state + } + } + static class PropertiesProvider implements ArgumentsProvider { @Override public Stream provideArguments(ExtensionContext extensionContext) { @@ -80,12 +115,8 @@ public Stream provideArguments(ExtensionContext extensionContext) { Map.of(STATE_PERSISTENCE_STRATEGY, "MEMORY", KEEP_STATE, "false"), Map.of(STATE_PERSISTENCE_STRATEGY, "SQLITE", - SQLITE_DIRECTORY, "change-detection-filter", KEEP_STATE, "false"), Map.of(STATE_PERSISTENCE_STRATEGY, "POSTGRES", - POSTGRES_URL, postgreSQLContainer.getJdbcUrl(), - POSTGRES_USERNAME, postgreSQLContainer.getUsername(), - POSTGRES_PASSWORD, postgreSQLContainer.getPassword(), KEEP_STATE, "false") ).map(Arguments::of); } diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index 4c34aec4a..9aeabc91d 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index b9f2dea17..d865de374 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml index e19f1dca5..511452890 100644 --- a/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/http-sparql-out-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT http-sparql-out-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index aa42fd8ac..9ecaddda1 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 8ad9e7b21..27ab2bc76 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -1,9 +1,10 @@ - + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 @@ -26,10 +27,6 @@ be.vlaanderen.informatievlaanderen.ldes.client event-stream-properties-fetcher - - com.github.tomakehurst - wiremock-jre8 - be.vlaanderen.informatievlaanderen.ldes.client tree-node-supplier @@ -46,6 +43,11 @@ ${testcontainers-postgresql.version} test + + com.github.tomakehurst + wiremock-jre8 + test + diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java index 8986af2cc..366182d77 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessor.java @@ -1,12 +1,16 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; +import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateUtil; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.FlowManager; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.NiFiDBCPDataSource; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.services.RequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers.MemberSupplierWrappersBuilder; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.timestampextractor.TimestampFromPathExtractor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import ldes.client.eventstreamproperties.EventStreamPropertiesFetcher; import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; @@ -24,6 +28,7 @@ import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.state.Scope; +import org.apache.nifi.dbcp.DBCPService; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; @@ -66,6 +71,7 @@ public final List getSupportedPropertyDescriptors() { DATA_SOURCE_FORMAT, DATA_DESTINATION_FORMAT, STATE_PERSISTENCE_STRATEGY, + DBCP_SERVICE, KEEP_STATE, USE_EXACTLY_ONCE_FILTER, USE_VERSION_MATERIALISATION, @@ -80,9 +86,6 @@ public final List getSupportedPropertyDescriptors() { RETRIES_ENABLED, MAX_RETRIES, STATUSES_TO_RETRY, - POSTGRES_URL, - POSTGRES_USERNAME, - POSTGRES_PASSWORD, RESTRICT_TO_MEMBERS, STREAM_TIMESTAMP_PATH_PROPERTY, STREAM_VERSION_OF_PROPERTY, @@ -97,17 +100,20 @@ public void onScheduled(final ProcessContext context) { final RequestExecutorSupplier requestExecutorSupplier = new RequestExecutorSupplier(); final RequestExecutor requestExecutor = requestExecutorSupplier.getRequestExecutor(context); LdesMetaData ldesMetaData = new LdesMetaData(dataSourceUrls, dataSourceFormat); - StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(context); + LdesClientRepositories ldesClientRepositories = getClientRepositories(context); + eventStreamProperties = fetchEventStreamProperties(ldesMetaData, requestExecutor); TimestampExtractor timestampExtractor = new TimestampFromPathExtractor(createProperty(eventStreamProperties.getTimestampPath())); - TreeNodeProcessor treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, + TreeNodeProcessor treeNodeProcessor = new TreeNodeProcessor(ldesMetaData, ldesClientRepositories, requestExecutor, timestampExtractor, clientStatusConsumer()); final MemberSupplier baseMemberSupplier = new MemberSupplierImpl(treeNodeProcessor, stateKept(context)); memberSupplier = new MemberSupplierWrappersBuilder() .withContext(context) .withEventStreamProperties(eventStreamProperties) + .withClientRepositories(ldesClientRepositories) .build() .wrapMemberSupplier(baseMemberSupplier); + memberSupplier.init(); LOGGER.info("LDES Client processor {} configured to follow (sub)streams {} (expected LDES source format: {})", context.getName(), dataSourceUrls, dataSourceFormat); @@ -167,4 +173,18 @@ private Consumer clientStatusConsumer() { return status -> LOGGER.info("LDES Client is now {}", status); } + private LdesClientRepositories getClientRepositories(ProcessContext context) { + final DBCPService dbcpService = context.getProperty(DBCP_SERVICE).asControllerService(DBCPService.class); + final NiFiDBCPDataSource dataSource = new NiFiDBCPDataSource(dbcpService); + final StatePersistenceStrategy state = getStatePersistenceStrategy(context); + + return switch (state) { + case MEMORY -> LdesClientRepositories.memoryBased(); + case SQLITE, POSTGRES -> { + var keepState = PersistenceProperties.stateKept(context); + var entityManager = HibernateUtil.createEntityManagerFromDatasource(dataSource, keepState, state); + yield LdesClientRepositories.sqlBased(entityManager); + } + }; + } } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/StatePersistenceFactory.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/StatePersistenceFactory.java deleted file mode 100644 index 4d9e03603..000000000 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/StatePersistenceFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldi.processors; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; -import org.apache.nifi.processor.ProcessContext; - -import java.util.Map; - -import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.getStatePersistenceStrategy; - -public class StatePersistenceFactory { - - public StatePersistence getStatePersistence(ProcessContext context) { - StatePersistenceStrategy state = getStatePersistenceStrategy(context); - HibernateProperties properties = switch (state) { - case POSTGRES -> createPostgresProperties(context); - case SQLITE -> createSqliteProperties(context); - default -> Map::of; - }; - return StatePersistence.from(state, properties, context.getName()); - } - - private PostgresProperties createPostgresProperties(ProcessContext context) { - String url = PersistenceProperties.getPostgresUrl(context); - String username = PersistenceProperties.getPostgresUsername(context); - String password = PersistenceProperties.getPostgresPassword(context); - boolean keepState = PersistenceProperties.stateKept(context); - return new PostgresProperties(url, username, password, keepState); - } - - private SqliteProperties createSqliteProperties(ProcessContext context) { - final boolean keepState = PersistenceProperties.stateKept(context); - return PersistenceProperties.getSqliteDirectory(context) - .map(directory -> new SqliteProperties(directory, context.getName(), keepState)) - .orElseGet(() -> new SqliteProperties(context.getName(), keepState)); - } -} diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java index 7a6e8599f..6080ab474 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapper.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers; -import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.StatePersistenceFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; @@ -15,9 +14,11 @@ public class ExactlyOnceMemberSupplierWrapper extends MemberSupplierWrapper { private final ProcessContext context; + private final MemberIdRepository memberIdRepository; - public ExactlyOnceMemberSupplierWrapper(ProcessContext context) { + public ExactlyOnceMemberSupplierWrapper(ProcessContext context, MemberIdRepository memberIdRepository) { this.context = context; + this.memberIdRepository = memberIdRepository; } @Override @@ -31,9 +32,6 @@ protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSuppli } private MemberFilter createMemberFilter() { - final MemberIdRepository memberIdRepository = new StatePersistenceFactory() - .getStatePersistence(context) - .getMemberIdRepository(); final boolean keepState = PersistenceProperties.stateKept(context); return new ExactlyOnceFilter(memberIdRepository, keepState); } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java index dbbeeb675..e90318d0e 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapper.java @@ -1,6 +1,5 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers; -import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.StatePersistenceFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; @@ -16,10 +15,12 @@ public class LatestStateMemberSupplierWrapper extends MemberSupplierWrapper { private final ProcessContext context; + private final MemberVersionRepository memberVersionRepository; private final EventStreamProperties eventStreamProperties; - public LatestStateMemberSupplierWrapper(ProcessContext context, EventStreamProperties eventStreamProperties) { + public LatestStateMemberSupplierWrapper(ProcessContext context, MemberVersionRepository memberVersionRepository, EventStreamProperties eventStreamProperties) { this.context = context; + this.memberVersionRepository = memberVersionRepository; this.eventStreamProperties = eventStreamProperties; } @@ -34,9 +35,6 @@ protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSuppli } private MemberFilter createMemberFilter() { - final MemberVersionRepository memberVersionRepository = new StatePersistenceFactory() - .getStatePersistence(context) - .getMemberVersionRepository(); final boolean keepState = PersistenceProperties.stateKept(context); return new LatestStateFilter(memberVersionRepository, keepState, eventStreamProperties.getTimestampPath(), eventStreamProperties.getVersionOfPath()); } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java index 3b440717a..53d018fcd 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/MemberSupplierWrappersBuilder.java @@ -2,12 +2,14 @@ import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.treenodesupplier.domain.services.MemberSupplierWrappers; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; import org.apache.nifi.processor.ProcessContext; import java.util.List; public class MemberSupplierWrappersBuilder implements MemberSupplierWrappers.Builder { private ProcessContext context; + private LdesClientRepositories clientRepositories; private EventStreamProperties eventStreamProperties; public MemberSupplierWrappersBuilder withContext(ProcessContext context) { @@ -15,6 +17,11 @@ public MemberSupplierWrappersBuilder withContext(ProcessContext context) { return this; } + public MemberSupplierWrappers.Builder withClientRepositories(LdesClientRepositories clientRepositories) { + this.clientRepositories = clientRepositories; + return this; + } + public MemberSupplierWrappersBuilder withEventStreamProperties(EventStreamProperties eventStreamProperties) { this.eventStreamProperties = eventStreamProperties; return this; @@ -23,8 +30,8 @@ public MemberSupplierWrappersBuilder withEventStreamProperties(EventStreamProper @Override public MemberSupplierWrappers build() { return new MemberSupplierWrappers(List.of( - new ExactlyOnceMemberSupplierWrapper(context), - new LatestStateMemberSupplierWrapper(context, eventStreamProperties), + new ExactlyOnceMemberSupplierWrapper(context, clientRepositories.memberIdRepository()), + new LatestStateMemberSupplierWrapper(context, clientRepositories.memberVersionRepository(), eventStreamProperties), new VersionMaterialisedMemberSupplierWrapper(context, eventStreamProperties.getVersionOfPath()) )); } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java index 754a3c276..9d32d2ba1 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/LdesClientProcessorTest.java @@ -7,6 +7,9 @@ import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFParser; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.dbcp.DBCPConnectionPool; +import org.apache.nifi.dbcp.utils.DBCPProperties; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; @@ -24,7 +27,8 @@ import java.util.stream.Stream; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorRelationships.DATA_RELATIONSHIP; -import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.*; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.DBCP_SERVICE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties.STATE_PERSISTENCE_STRATEGY; import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; @@ -67,12 +71,14 @@ void tearDown() { @ParameterizedTest @ArgumentsSource(MatchNumberOfFlowFilesArgumentsProvider.class) - void shouldMatchNumberOfFlowFiles(String dataSourceUrl, int numberOfRuns) { + void shouldMatchNumberOfFlowFiles(String dataSourceUrl, int numberOfRuns) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", dataSourceUrl); testRunner.setProperty(STATE_PERSISTENCE_STRATEGY, "SQLITE"); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); + setupDBCPforState("SQLITE"); + testRunner.run(numberOfRuns); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -82,13 +88,13 @@ void shouldMatchNumberOfFlowFiles(String dataSourceUrl, int numberOfRuns) { @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) - void when_NecessaryPropertiesAreSet_then_statePersistenceCanBeCreated(Map properties) { + void when_NecessaryPropertiesAreSet_then_statePersistenceCanBeCreated(Map properties) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z"); properties.forEach((key, value) -> testRunner.setProperty(key, value)); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); - + setupDBCPforState(properties.get(STATE_PERSISTENCE_STRATEGY)); testRunner.run(); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -105,7 +111,7 @@ void when_NecessaryPropertiesAreSet_then_statePersistenceCanBeCreated(Map statePersistenceProps) { + void shouldSupportRedirectLogic(Map statePersistenceProps) throws InitializationException { testRunner.setProperty("AUTHORIZATION_STRATEGY", "NO_AUTH"); testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/200-response-with-indirect-url"); statePersistenceProps.forEach(testRunner::setProperty); @@ -114,6 +120,7 @@ void shouldSupportRedirectLogic(Map statePersistence testRunner.setProperty("STREAM_VERSION_OF_PROPERTY", Boolean.FALSE.toString()); testRunner.setProperty("DATA_SOURCE_FORMAT", "application/ld+json"); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); testRunner.run(); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -123,7 +130,7 @@ void shouldSupportRedirectLogic(Map statePersistence @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) - void shouldBeAbleToEndGracefully(Map statePersistenceProps) { + void shouldBeAbleToEndGracefully(Map statePersistenceProps) throws InitializationException { // This is an immutable fragment with 1 member and no relations. We reach the // end of the ldes after 1 run. testRunner.setProperty("DATA_SOURCE_URLS", @@ -135,6 +142,7 @@ void shouldBeAbleToEndGracefully(Map statePersistenc testRunner.setProperty("STREAM_VERSION_OF_PROPERTY", Boolean.FALSE.toString()); testRunner.setProperty("DATA_SOURCE_FORMAT", "application/ld+json"); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); testRunner.run(5); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -160,7 +168,7 @@ void shouldSupportDifferentHttpRequestExecutors(Map properties) @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) - void shouldSupportRetry(Map statePersistenceProps) { + void shouldSupportRetry(Map statePersistenceProps) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/retry"); statePersistenceProps.forEach(testRunner::setProperty); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); @@ -168,6 +176,8 @@ void shouldSupportRetry(Map statePersistenceProps) { testRunner.setProperty("MAX_RETRIES", "3"); testRunner.setProperty("STATUSES_TO_RETRY", "418"); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); + testRunner.run(6); WireMock.verify(3, getRequestedFor(urlEqualTo("/retry"))); @@ -179,7 +189,7 @@ void shouldSupportRetry(Map statePersistenceProps) { @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) - void shouldSupportVersionMaterialisation(Map statePersistenceProps) { + void shouldSupportVersionMaterialisation(Map statePersistenceProps) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z"); statePersistenceProps.forEach(testRunner::setProperty); @@ -188,6 +198,7 @@ void shouldSupportVersionMaterialisation(Map statePe testRunner.setProperty("USE_LATEST_STATE_FILTER", Boolean.FALSE.toString()); testRunner.setProperty("RESTRICT_TO_MEMBERS", Boolean.FALSE.toString()); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); testRunner.run(); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -243,13 +254,15 @@ void when_DataSourceUrlsIsEmpty_then_ThrowException() { @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) - void shouldSupportOnlyOnceFilter(Map statePersistenceProps) { + void shouldSupportOnlyOnceFilter(Map statePersistenceProps) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/duplicate-members?pageNumber=1"); statePersistenceProps.forEach(testRunner::setProperty); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); testRunner.setProperty("USE_EXACTLY_ONCE_FILTER", Boolean.TRUE.toString()); testRunner.setProperty("RESTRICT_TO_MEMBERS", Boolean.FALSE.toString()); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); + testRunner.run(4); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -267,13 +280,14 @@ void shouldSupportOnlyOnceFilter(Map statePersistenc @ParameterizedTest @ArgumentsSource(StatePersistenceArgumentsProvider.class) - void shouldSupportDisableOfOnlyOnceFilter(Map statePersistenceProps) { + void shouldSupportDisableOfOnlyOnceFilter(Map statePersistenceProps) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/duplicate-members?pageNumber=1"); statePersistenceProps.forEach(testRunner::setProperty); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); testRunner.setProperty("USE_EXACTLY_ONCE_FILTER", Boolean.FALSE.toString()); testRunner.setProperty("RESTRICT_TO_MEMBERS", Boolean.FALSE.toString()); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); testRunner.run(4); List dataFlowfiles = testRunner.getFlowFilesForRelationship(DATA_RELATIONSHIP); @@ -318,10 +332,11 @@ void shouldSupportVersionMaterialisationWithLatestStateFilter(Map statePersistenceProps) { + void shouldNotSupportOnlyOnceFilterWhenVersionMaterialiserIsActive(Map statePersistenceProps) throws InitializationException { testRunner.setProperty("DATA_SOURCE_URLS", "http://localhost:10101/exampleData?generatedAtTime=2022-05-03T00:00:00.000Z"); statePersistenceProps.forEach(testRunner::setProperty); + setupDBCPforState(statePersistenceProps.get(STATE_PERSISTENCE_STRATEGY)); testRunner.setProperty("KEEP_STATE", Boolean.FALSE.toString()); testRunner.setProperty("USE_VERSION_MATERIALISATION", Boolean.TRUE.toString()); testRunner.setProperty("USE_EXACTLY_ONCE_FILTER", Boolean.TRUE.toString()); @@ -342,6 +357,36 @@ void shouldNotSupportOnlyOnceFilterWhenVersionMaterialiserIsActive(Map provideArguments(ExtensionContext context) { @@ -360,10 +405,7 @@ public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of(Map.of(STATE_PERSISTENCE_STRATEGY, "MEMORY")), Arguments.of(Map.of(STATE_PERSISTENCE_STRATEGY, "SQLITE")), - Arguments.of(Map.of(STATE_PERSISTENCE_STRATEGY, "POSTGRES", - POSTGRES_URL, postgreSQLContainer.getJdbcUrl(), - POSTGRES_USERNAME, postgreSQLContainer.getUsername(), - POSTGRES_PASSWORD, postgreSQLContainer.getPassword())) + Arguments.of(Map.of(STATE_PERSISTENCE_STRATEGY, "POSTGRES")) ); } } diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java index 16bc04fcf..6681f2340 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/LdesProcessorPropertiesTest.java @@ -1,16 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config; -import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.mock.MockProcessContext; -import org.apache.nifi.util.MockPropertyValue; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.LdesClientProcessor; +import org.apache.nifi.util.MockProcessContext; import org.junit.jupiter.api.Test; import java.util.List; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.DATA_SOURCE_URLS; import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties.getDataSourceUrl; -import static org.assertj.core.api.Assertions.assertThat; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RequestExecutorProperties.RETRIES_ENABLED; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.RequestExecutorProperties.STATUSES_TO_RETRY; import static org.junit.jupiter.api.Assertions.*; class LdesProcessorPropertiesTest { @@ -18,17 +17,15 @@ class LdesProcessorPropertiesTest { @Test void test_retriesEnabled() { // default is true - assertTrue(RequestExecutorProperties.retriesEnabled(getMockContext(null))); - assertTrue(RequestExecutorProperties.retriesEnabled(getMockContext("true"))); - assertTrue(RequestExecutorProperties.retriesEnabled(getMockContext("trUe"))); - assertFalse(RequestExecutorProperties.retriesEnabled(getMockContext("false"))); - assertFalse(RequestExecutorProperties.retriesEnabled(getMockContext("FALSE"))); + assertTrue(RequestExecutorProperties.retriesEnabled(getMockContextRetriesEnabled("true"))); + assertTrue(RequestExecutorProperties.retriesEnabled(getMockContextRetriesEnabled("trUe"))); + assertFalse(RequestExecutorProperties.retriesEnabled(getMockContextRetriesEnabled("false"))); + assertFalse(RequestExecutorProperties.retriesEnabled(getMockContextRetriesEnabled("FALSE"))); } @Test void test_getStatusesToRetry() { - assertTrue(RequestExecutorProperties.getStatusesToRetry(getMockContext(null)).isEmpty()); - List statusesToRetry = RequestExecutorProperties.getStatusesToRetry(getMockContext("200, 204")); + List statusesToRetry = RequestExecutorProperties.getStatusesToRetry(getMockContextStatuses("200, 204")); assertTrue(statusesToRetry.contains(200)); assertTrue(statusesToRetry.contains(204)); assertFalse(statusesToRetry.contains(500)); @@ -36,31 +33,33 @@ void test_getStatusesToRetry() { @Test void test_getDatasourceUrl() { - assertDoesNotThrow(() -> getMockContext("http://localhost/endpoint")); - assertDoesNotThrow(() -> getMockContext("http://localhost/endpoint,http://localhost/other")); - var singleInvalidUri = getMockContext("inv alid"); + assertDoesNotThrow(() -> getMockContextDatasource("http://localhost/endpoint")); + assertDoesNotThrow(() -> getMockContextDatasource("http://localhost/endpoint,http://localhost/other")); + var singleInvalidUri = getMockContextDatasource("inv alid"); assertThrows(IllegalArgumentException.class, () -> getDataSourceUrl(singleInvalidUri)); - var multiInvalidUri = getMockContext("inv alid,http://localhost/other"); + var multiInvalidUri = getMockContextDatasource("inv alid,http://localhost/other"); assertThrows(IllegalArgumentException.class, () -> getDataSourceUrl(multiInvalidUri)); } - @Test - void test_getSqliteConfig() { - assertThat(PersistenceProperties.getStatePersistenceStrategy(getMockContext("SQLITE"))) - .isEqualTo(StatePersistenceStrategy.SQLITE); - assertThat(PersistenceProperties.getSqliteDirectory(getMockContext("/ldio/sqlite"))) - .contains("/ldio/sqlite"); - assertThat(PersistenceProperties.getSqliteDirectory(getMockContext(null))) - .isEmpty(); + private MockProcessContext getMockContextRetriesEnabled(String retriesEnabled) { + // Create a MockProcessContext with the TestRunner + MockProcessContext context = new MockProcessContext(new LdesClientProcessor()); + context.setProperty(RETRIES_ENABLED, retriesEnabled); + return context; + } + + private MockProcessContext getMockContextStatuses(String val) { + // Create a MockProcessContext with the TestRunner + MockProcessContext context = new MockProcessContext(new LdesClientProcessor()); + context.setProperty(STATUSES_TO_RETRY, val); + return context; } - private static MockProcessContext getMockContext(String value) { - return new MockProcessContext() { - @Override - public PropertyValue getProperty(PropertyDescriptor descriptor) { - return new MockPropertyValue(value); - } - }; + private MockProcessContext getMockContextDatasource(String val) { + // Create a MockProcessContext with the TestRunner + MockProcessContext context = new MockProcessContext(new LdesClientProcessor()); + context.setProperty(DATA_SOURCE_URLS, val); + return context; } } \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapperTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapperTest.java index 784d26422..9a5a6ab4e 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapperTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/ExactlyOnceMemberSupplierWrapperTest.java @@ -16,7 +16,7 @@ class ExactlyOnceMemberSupplierWrapperTest { @Test void given_ExactlyOnceEnabled_when_wrap_then_ReturnFilteredMemberSupplier() { final ProcessContext context = new TestProcessContext(true); - wrapper = new ExactlyOnceMemberSupplierWrapper(context); + wrapper = new ExactlyOnceMemberSupplierWrapper(context, null); final MemberSupplier memberSupplier = wrapper.wrapMemberSupplier(baseSupplier); @@ -26,7 +26,7 @@ void given_ExactlyOnceEnabled_when_wrap_then_ReturnFilteredMemberSupplier() { @Test void given_ExactlyOnceAndVersionMaterialisationEnabled_when_wrap_then_ReturnBaseMemberSupplier() { final ProcessContext context = new TestProcessContext(true, true, true); - wrapper = new ExactlyOnceMemberSupplierWrapper(context); + wrapper = new ExactlyOnceMemberSupplierWrapper(context, null); final MemberSupplier memberSupplier = wrapper.wrapMemberSupplier(baseSupplier); @@ -36,7 +36,7 @@ void given_ExactlyOnceAndVersionMaterialisationEnabled_when_wrap_then_ReturnBase @Test void given_ExactlyOnceDisabled_when_wrap_then_ReturnBaseMemberSupplier() { final ProcessContext context = new TestProcessContext(false); - wrapper = new ExactlyOnceMemberSupplierWrapper(context); + wrapper = new ExactlyOnceMemberSupplierWrapper(context, null); final MemberSupplier memberSupplier = wrapper.wrapMemberSupplier(baseSupplier); diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapperTest.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapperTest.java index 613339e18..c0412d254 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapperTest.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/LatestStateMemberSupplierWrapperTest.java @@ -18,7 +18,7 @@ class LatestStateMemberSupplierWrapperTest { @Test void given_LatestStateEnabled_when_wrap_then_ReturnFilteredMemberSupplier() { final ProcessContext context = new TestProcessContext(true, true); - wrapper = new LatestStateMemberSupplierWrapper(context, eventStreamProperties); + wrapper = new LatestStateMemberSupplierWrapper(context, null, eventStreamProperties); final MemberSupplier memberSupplier = wrapper.wrapMemberSupplier(baseSupplier); @@ -28,7 +28,7 @@ void given_LatestStateEnabled_when_wrap_then_ReturnFilteredMemberSupplier() { @Test void given_VersionMaterialisationDisabled_when_wrap_then_ReturnBaseSupplier() { final ProcessContext context = new TestProcessContext(false, true); - wrapper = new LatestStateMemberSupplierWrapper(context, eventStreamProperties); + wrapper = new LatestStateMemberSupplierWrapper(context, null, eventStreamProperties); final MemberSupplier memberSupplier = wrapper.wrapMemberSupplier(baseSupplier); @@ -38,7 +38,7 @@ void given_VersionMaterialisationDisabled_when_wrap_then_ReturnBaseSupplier() { @Test void given_LatestStateDisabled_when_wrap_then_ReturnBaseSupplier() { final ProcessContext context = new TestProcessContext(true, false); - wrapper = new LatestStateMemberSupplierWrapper(context, eventStreamProperties); + wrapper = new LatestStateMemberSupplierWrapper(context, null, eventStreamProperties); final MemberSupplier memberSupplier = wrapper.wrapMemberSupplier(baseSupplier); diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/testutils/TestProcessContext.java b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/testutils/TestProcessContext.java index 6e78f2e13..da5002bd7 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/testutils/TestProcessContext.java +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/wrappers/testutils/TestProcessContext.java @@ -1,26 +1,19 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.processors.wrappers.testutils; +import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.LdesClientProcessor; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.LdesProcessorProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.processors.config.PersistenceProperties; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.mock.MockProcessContext; -import org.apache.nifi.util.MockPropertyValue; - -import java.util.Map; +import org.apache.nifi.util.MockProcessContext; public class TestProcessContext extends MockProcessContext { - private final Map properties; - public TestProcessContext(boolean useExactlyOnceFilter, boolean useVersionMaterialisation, boolean useLatestStateFilter) { - properties = Map.of( - LdesProcessorProperties.USE_EXACTLY_ONCE_FILTER, String.valueOf(useExactlyOnceFilter), - LdesProcessorProperties.USE_VERSION_MATERIALISATION, String.valueOf(useVersionMaterialisation), - LdesProcessorProperties.USE_LATEST_STATE_FILTER, String.valueOf(useLatestStateFilter), - PersistenceProperties.KEEP_STATE, String.valueOf(false), - PersistenceProperties.STATE_PERSISTENCE_STRATEGY, "MEMORY" - ); + super(new LdesClientProcessor()); + this.setProperty(LdesProcessorProperties.USE_EXACTLY_ONCE_FILTER, String.valueOf(useExactlyOnceFilter)); + this.setProperty(LdesProcessorProperties.USE_VERSION_MATERIALISATION, String.valueOf(useVersionMaterialisation)); + this.setProperty(LdesProcessorProperties.USE_LATEST_STATE_FILTER, String.valueOf(useLatestStateFilter)); + this.setProperty(PersistenceProperties.KEEP_STATE, String.valueOf(false)); + this.setProperty(PersistenceProperties.STATE_PERSISTENCE_STRATEGY, "MEMORY"); } public TestProcessContext(boolean useExactlyOnce) { @@ -30,9 +23,4 @@ public TestProcessContext(boolean useExactlyOnce) { public TestProcessContext(boolean useVersionMaterialisation, boolean useLatestStateFilter) { this(false, useVersionMaterialisation, useLatestStateFilter); } - - @Override - public PropertyValue getProperty(PropertyDescriptor descriptor) { - return new MockPropertyValue(properties.get(descriptor)); - } } diff --git a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml index 3b163d9ad..05374b08a 100644 --- a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT ldi-processors-bundle diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index ed051774d..12d0ae29b 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 2.12.0 + 2.13.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 6043269e7..68ba95c71 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml index 9c77d434a..ba23b622b 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-sink-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT rdf4j-repository-sink-processor diff --git a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml index 831f1d386..347d51589 100644 --- a/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rml-adapter-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT rml-adapter-processor diff --git a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml index 35a936680..e9c58e888 100644 --- a/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/skolemisation-transformer-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT skolemisation-transformer-processor diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index 2577ec7fc..bc6be070e 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index 33d1a5b9e..25443a460 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index e8964b091..41d812d29 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 pom @@ -17,7 +17,7 @@ - 1.24.0 + 2.0.0 diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index 890b3938c..97f92c988 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index 0834d76cd..3f316534b 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index 23ff958ee..312e5b67b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index afeb4e0a9..d781afea8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index b8f8ec85c..a7b1724da 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml index 30d6a545f..ba4e2c2ab 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-change-detection-filter diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilter.java b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilter.java index 0a879178f..1267c5be8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilter.java +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilter.java @@ -27,7 +27,7 @@ public void apply(Model model) { } public void shutdown() { - changeDetectionFilter.destroyState(); + changeDetectionFilter.close(); } private String extractSubjectFromModel(Model model) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HasedStateMemberRepositoryFactory.java b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HasedStateMemberRepositoryFactory.java index ebeb5b923..ea53d25ca 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HasedStateMemberRepositoryFactory.java +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/HasedStateMemberRepositoryFactory.java @@ -1,59 +1,56 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateUtil; import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.inmemory.InMemoryHashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql.SqlHashedStateMemberRepository; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties; -import java.util.Map; - public class HasedStateMemberRepositoryFactory { public static final StatePersistenceStrategy DEFAULT_PERSISTENCE_STRATEGY = StatePersistenceStrategy.MEMORY; public static final boolean DEFAULT_KEEP_STATE = false; - private final ComponentProperties properties; - public HasedStateMemberRepositoryFactory(ComponentProperties properties) { - this.properties = properties; + private HasedStateMemberRepositoryFactory() { } - public HashedStateMemberRepository getHashedStateMemberRepository() { + public static HashedStateMemberRepository getHashedStateMemberRepository(ComponentProperties properties) { final StatePersistenceStrategy persistenceStrategy = properties.getOptionalProperty(PersistenceProperties.STATE) .flatMap(StatePersistenceStrategy::from) .orElse(DEFAULT_PERSISTENCE_STRATEGY); + return switch (persistenceStrategy) { - case POSTGRES -> createPostgresRepository(); - case SQLITE -> createSqliteRepository(); + case POSTGRES -> { + var hibernateProperties = createPostgresProperties(properties); + var entityManager = HibernateUtil.createEntityManagerFromProperties(hibernateProperties.getProperties()); + yield new SqlHashedStateMemberRepository(entityManager); + } + case SQLITE -> { + var hibernateProperties = createSqliteProperties(properties); + var entityManager = HibernateUtil.createEntityManagerFromProperties(hibernateProperties.getProperties()); + yield new SqlHashedStateMemberRepository(entityManager); + } case MEMORY -> new InMemoryHashedStateMemberRepository(); }; } - private SqlHashedStateMemberRepository createPostgresRepository() { - final String pipelineName = properties.getPipelineName(); - final var entityManagerFactory = PostgresEntityManagerFactory.getInstance(pipelineName, - createPostgresProperties()); - return new SqlHashedStateMemberRepository(entityManagerFactory, pipelineName); - } - - private Map createPostgresProperties() { + private static PostgresProperties createPostgresProperties(ComponentProperties properties) { String url = properties.getProperty(PersistenceProperties.POSTGRES_URL); String username = properties.getProperty(PersistenceProperties.POSTGRES_USERNAME); String password = properties.getProperty(PersistenceProperties.POSTGRES_PASSWORD); boolean keepState = properties.getOptionalBoolean(PersistenceProperties.KEEP_STATE) .orElse(DEFAULT_KEEP_STATE); - return new PostgresProperties(url, username, password, keepState).getProperties(); + return new PostgresProperties(url, username, password, keepState); } - private SqlHashedStateMemberRepository createSqliteRepository() { + private static SqliteProperties createSqliteProperties(ComponentProperties properties) { final String pipelineName = properties.getPipelineName(); final boolean keepState = properties.getOptionalBoolean(PersistenceProperties.KEEP_STATE).orElse(DEFAULT_KEEP_STATE); - final SqliteProperties sqliteProperties = new SqliteProperties(pipelineName, keepState); - final var entityManagerFactory = SqliteEntityManagerFactory.getInstance(sqliteProperties); - return new SqlHashedStateMemberRepository(entityManagerFactory, pipelineName); + return properties.getOptionalProperty(PersistenceProperties.SQLITE_DIRECTORY) + .map(directory -> new SqliteProperties(directory, pipelineName, keepState)) + .orElse(new SqliteProperties(pipelineName, keepState)); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioChangeDetectionFilterAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioChangeDetectionFilterAutoConfig.java index 8705743e7..6341a6968 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioChangeDetectionFilterAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioChangeDetectionFilterAutoConfig.java @@ -3,10 +3,9 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.ChangeDetectionFilter; import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioChangeDetectionFilter; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformerConfigurator; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -24,9 +23,8 @@ public LdioTransformerConfigurator ldioChangeDetectionFilterConfigurator() { public static class LdioChangeDetectionFilterConfigurator implements LdioTransformerConfigurator { @Override public LdioTransformer configure(ComponentProperties properties) { - final HashedStateMemberRepository repository = new HasedStateMemberRepositoryFactory(properties).getHashedStateMemberRepository(); - final boolean keepState = properties.getOptionalBoolean(PersistenceProperties.KEEP_STATE).orElse(false); - final ChangeDetectionFilter changeDetectionFilter = new ChangeDetectionFilter(repository, keepState); + final HashedStateMemberRepository repository = HasedStateMemberRepositoryFactory.getHashedStateMemberRepository(properties); + final ChangeDetectionFilter changeDetectionFilter = new ChangeDetectionFilter(repository); return new LdioChangeDetectionFilter(changeDetectionFilter); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionEventsListenerTest.java b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionEventsListenerTest.java index 405a3585b..f542577f8 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionEventsListenerTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionEventsListenerTest.java @@ -51,7 +51,7 @@ void when_PipelineDeletedEventIsPublished_then_DestroyFilter() { eventPublisher.publishEvent(new PipelineDeletedEvent(pipelineName)); verify(pipelineEventsListener).handlePipelineDeletedEvent(new PipelineDeletedEvent(pipelineName)); - verify(filter).destroyState(); + verify(filter).close(); } @Test diff --git a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilterSteps.java b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilterSteps.java index 696fc38e4..773831369 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilterSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-change-detection-filter/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioChangeDetectionFilterSteps.java @@ -1,9 +1,10 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioChangeDetectionFilterAutoConfig; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioTransformerConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.LdioVaultTransformer; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -65,9 +66,12 @@ public void anLdioChangeDetectionFilterWithAStatePersistenceStrategy(String pers final var postgresProperties = startPostgresContainer(); properties.putAll(postgresProperties); } + if (persistenceStrategy.equals("SQLITE")) { + properties.put(PersistenceProperties.SQLITE_DIRECTORY, "target"); + } vault = new LdioVaultTransformer(); final LdioTransformerConfigurator configurator = new LdioChangeDetectionFilterAutoConfig.LdioChangeDetectionFilterConfigurator(); - final ComponentProperties componentProperties = new ComponentProperties(PIPELINE_NAME, "Ldio:ChangeDetectionFitler", properties); + final ComponentProperties componentProperties = new ComponentProperties(PIPELINE_NAME, "Ldio:ChangeDetectionFilter", properties); ldioChangeDetectionFilter = (LdioChangeDetectionFilter) link(configurator.configure(componentProperties), List.of(vault)); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 056cd4974..8a017583b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index 6931bc6f0..53b728382 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index a1f62e0ed..53a78e7b1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index a51faaf20..8bf3b2398 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index a4ad719de..bd8599dae 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index cab196d91..1e55d4a8d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index e007c2e2d..407eabb80 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml index 3e2af4787..189ab333f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-sparql-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-http-sparql-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index 9e360de6e..da1de9095 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 6bb00f160..20fd21a28 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index 04db98c04..3515f4318 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 93edf40cd..42f6e23bc 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientRepositoriesFactory.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientRepositoriesFactory.java new file mode 100644 index 000000000..9ab84ce92 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientRepositoriesFactory.java @@ -0,0 +1,55 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateUtil; +import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; +import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties.*; + +public class LdesClientRepositoriesFactory { + public static final StatePersistenceStrategy DEFAULT_STATE_PERSISTENCE_STRATEGY = StatePersistenceStrategy.MEMORY; + public static final boolean DEFAULT_KEEP_STATE = false; + + private LdesClientRepositoriesFactory() { + } + + public static LdesClientRepositories getLdesClientRepositories(ComponentProperties properties) { + StatePersistenceStrategy state = properties.getOptionalProperty(STATE) + .flatMap(StatePersistenceStrategy::from) + .orElse(DEFAULT_STATE_PERSISTENCE_STRATEGY); + return switch (state) { + case POSTGRES -> { + var hibernateProperties = createPostgresProperties(properties); + var entityManager = HibernateUtil.createEntityManagerFromProperties(hibernateProperties.getProperties()); + yield LdesClientRepositories.sqlBased(entityManager); + } + case SQLITE -> { + var hibernateProperties = createSqliteProperties(properties); + var entityManager = HibernateUtil.createEntityManagerFromProperties(hibernateProperties.getProperties()); + yield LdesClientRepositories.sqlBased(entityManager); + } + case MEMORY -> LdesClientRepositories.memoryBased(); + }; + } + + private static PostgresProperties createPostgresProperties(ComponentProperties properties) { + String url = properties.getProperty(POSTGRES_URL); + String username = properties.getProperty(POSTGRES_USERNAME); + String password = properties.getProperty(POSTGRES_PASSWORD); + boolean keepState = properties.getOptionalBoolean(KEEP_STATE) + .orElse(DEFAULT_KEEP_STATE); + return new PostgresProperties(url, username, password, keepState); + } + + private static SqliteProperties createSqliteProperties(ComponentProperties properties) { + final String pipelineName = properties.getPipelineName(); + boolean keepState = properties.getOptionalBoolean(KEEP_STATE) + .orElse(DEFAULT_KEEP_STATE); + return properties.getOptionalProperty(SQLITE_DIRECTORY) + .map(directory -> new SqliteProperties(directory, pipelineName, keepState)) + .orElseGet(() -> new SqliteProperties(pipelineName, keepState)); + } +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java index f1de9ca78..ae6411213 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/MemberSupplierFactory.java @@ -10,8 +10,8 @@ import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.eventstreamproperties.valueobjects.PropertiesRequest; import ldes.client.treenodesupplier.TreeNodeProcessor; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; import ldes.client.treenodesupplier.domain.valueobject.LdesMetaData; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; import ldes.client.treenodesupplier.membersuppliers.MemberSupplier; import ldes.client.treenodesupplier.membersuppliers.MemberSupplierImpl; import org.slf4j.Logger; @@ -51,10 +51,10 @@ public MemberSupplier getMemberSupplier() { } private TreeNodeProcessor getTreeNodeProcessor(EventStreamProperties eventStreamProperties) { - final StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(clientProperties.getProperties()); + final LdesClientRepositories ldesClientRepositories = LdesClientRepositoriesFactory.getLdesClientRepositories(clientProperties.getProperties()); LdesMetaData ldesMetaData = new LdesMetaData(clientProperties.getUrls(), clientProperties.getSourceFormat()); TimestampExtractor timestampExtractor = new TimestampFromPathExtractor(createProperty(eventStreamProperties.getTimestampPath())); - return new TreeNodeProcessor(ldesMetaData, statePersistence, requestExecutor, timestampExtractor, clientStatusConsumer); + return new TreeNodeProcessor(ldesMetaData, ldesClientRepositories, requestExecutor, timestampExtractor, clientStatusConsumer); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/StatePersistenceFactory.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/StatePersistenceFactory.java deleted file mode 100644 index 25c4dee70..000000000 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/StatePersistenceFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -package be.vlaanderen.informatievlaanderen.ldes.ldio.config; - -import be.vlaanderen.informatievlaanderen.ldes.ldi.HibernateProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.StatePersistenceStrategy; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.persistence.PersistenceProperties; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; - -import java.util.Map; - -public class StatePersistenceFactory { - - public static final StatePersistenceStrategy DEFAULT_STATE_PERSISTENCE_STRATEGY = StatePersistenceStrategy.MEMORY; - public static final boolean DEFAULT_KEEP_STATE = false; - - public StatePersistence getStatePersistence(ComponentProperties properties) { - StatePersistenceStrategy state = properties.getOptionalProperty(PersistenceProperties.STATE) - .flatMap(StatePersistenceStrategy::from) - .orElse(DEFAULT_STATE_PERSISTENCE_STRATEGY); - HibernateProperties hibernateProperties = switch (state) { - case POSTGRES -> createPostgresProperties(properties); - case SQLITE -> createSqliteProperties(properties); - default -> Map::of; - }; - return StatePersistence.from(state, hibernateProperties, properties.getPipelineName()); - } - - private PostgresProperties createPostgresProperties(ComponentProperties properties) { - String url = properties.getProperty(PersistenceProperties.POSTGRES_URL); - String username = properties.getProperty(PersistenceProperties.POSTGRES_USERNAME); - String password = properties.getProperty(PersistenceProperties.POSTGRES_PASSWORD); - boolean keepState = properties.getOptionalBoolean(PersistenceProperties.KEEP_STATE) - .orElse(DEFAULT_KEEP_STATE); - return new PostgresProperties(url, username, password, keepState); - } - - private SqliteProperties createSqliteProperties(ComponentProperties properties) { - final String pipelineName = properties.getPipelineName(); - boolean keepState = properties.getOptionalBoolean(PersistenceProperties.KEEP_STATE) - .orElse(DEFAULT_KEEP_STATE); - return properties.getOptionalProperty(PersistenceProperties.SQLITE_DIRECTORY) - .map(directory -> new SqliteProperties(directory, pipelineName, keepState)) - .orElseGet(() -> new SqliteProperties(pipelineName, keepState)); - } -} diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java index 530795812..67aaad2e7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/ExactlyOnceMemberSupplierWrapper.java @@ -1,9 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.StatePersistenceFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdesClientRepositoriesFactory; import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; import ldes.client.treenodesupplier.filters.ExactlyOnceFilter; import ldes.client.treenodesupplier.filters.MemberFilter; import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; @@ -27,7 +27,7 @@ protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSuppli } private MemberFilter createExactlyOnceFilter() { - final StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(properties.getProperties()); - return new ExactlyOnceFilter(statePersistence.getMemberIdRepository(), properties.isKeepStateEnabled()); + final LdesClientRepositories ldesClientRepositories = LdesClientRepositoriesFactory.getLdesClientRepositories(properties.getProperties()); + return new ExactlyOnceFilter(ldesClientRepositories.memberIdRepository(), properties.isKeepStateEnabled()); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java index 19ab7864e..45dd69218 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/wrappers/LatestStateMemberSupplierWrapper.java @@ -1,10 +1,10 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config.wrappers; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioLdesClientProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.StatePersistenceFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdesClientRepositoriesFactory; import ldes.client.eventstreamproperties.valueobjects.EventStreamProperties; import ldes.client.treenodesupplier.domain.services.MemberSupplierWrapper; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; import ldes.client.treenodesupplier.filters.LatestStateFilter; import ldes.client.treenodesupplier.filters.MemberFilter; import ldes.client.treenodesupplier.membersuppliers.FilteredMemberSupplier; @@ -30,7 +30,7 @@ protected MemberSupplier createWrappedMemberSupplier(MemberSupplier memberSuppli } private MemberFilter createLatestStateFilter() { - final StatePersistence statePersistence = new StatePersistenceFactory().getStatePersistence(ldioLdesClientProperties.getProperties()); - return new LatestStateFilter(statePersistence.getMemberVersionRepository(), ldioLdesClientProperties.isKeepStateEnabled(), eventStreamProperties.getTimestampPath(), eventStreamProperties.getVersionOfPath()); + final LdesClientRepositories ldesClientRepositories = LdesClientRepositoriesFactory.getLdesClientRepositories(ldioLdesClientProperties.getProperties()); + return new LatestStateFilter(ldesClientRepositories.memberVersionRepository(), ldioLdesClientProperties.isKeepStateEnabled(), eventStreamProperties.getTimestampPath(), eventStreamProperties.getVersionOfPath()); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/StatePersistenceFactoryTest.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientRepositoriesFactoryTest.java similarity index 85% rename from ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/StatePersistenceFactoryTest.java rename to ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientRepositoriesFactoryTest.java index b43a44edf..3474b9cd1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/StatePersistenceFactoryTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdesClientRepositoriesFactoryTest.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.creation.valueobjects.ComponentProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.pipeline.exception.ConfigPropertyMissingException; -import ldes.client.treenodesupplier.domain.valueobject.StatePersistence; +import ldes.client.treenodesupplier.domain.valueobject.LdesClientRepositories; import ldes.client.treenodesupplier.repository.MemberRepository; import ldes.client.treenodesupplier.repository.TreeNodeRecordRepository; import ldes.client.treenodesupplier.repository.inmemory.InMemoryMemberRepository; @@ -26,9 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -class StatePersistenceFactoryTest { - - private final StatePersistenceFactory statePersistenceFactory = new StatePersistenceFactory(); +class LdesClientRepositoriesFactoryTest { private static PostgreSQLContainer postgreSQLContainer; @BeforeAll @@ -50,11 +48,11 @@ static void afterAll() { void when_stateIsAllowedValue_then_StatePersistenceIsCreated(ComponentProperties componentProperties, Class expectedMemberRepositoryClass, Class expectedTreeNodeRecordRepositoryClass) { - StatePersistence statePersistence = statePersistenceFactory.getStatePersistence(componentProperties); + LdesClientRepositories ldesClientRepositories = LdesClientRepositoriesFactory.getLdesClientRepositories(componentProperties); - MemberRepository memberRepository = statePersistence.getMemberRepository(); + MemberRepository memberRepository = ldesClientRepositories.memberRepository(); assertEquals(expectedMemberRepositoryClass, memberRepository.getClass()); - TreeNodeRecordRepository treeNodeRecordRepository = statePersistence.getTreeNodeRecordRepository(); + TreeNodeRecordRepository treeNodeRecordRepository = ldesClientRepositories.treeNodeRecordRepository(); assertEquals(expectedTreeNodeRecordRepositoryClass, treeNodeRecordRepository.getClass()); memberRepository.destroyState(); @@ -65,7 +63,7 @@ void when_stateIsAllowedValue_then_StatePersistenceIsCreated(ComponentProperties void when_stateIsPostgres_and_additionalPropertiesAreMissing_then_throwException() { ComponentProperties props = new ComponentProperties("pipelineName", "", Map.of(STATE, "postgres")); - assertThrows(ConfigPropertyMissingException.class, () -> statePersistenceFactory.getStatePersistence(props)); + assertThrows(ConfigPropertyMissingException.class, () -> LdesClientRepositoriesFactory.getLdesClientRepositories(props)); } private static class ComponentPropertiesArgumentsProvider implements ArgumentsProvider { @@ -75,7 +73,8 @@ public Stream provideArguments(ExtensionContext extensionContext) { Arguments.of(new ComponentProperties("pipelineName", "", Map.of(STATE, "memory")), InMemoryMemberRepository.class, InMemoryTreeNodeRecordRepository.class), - Arguments.of(new ComponentProperties("pipelineName", "", Map.of(STATE, "sqlite")), + Arguments.of(new ComponentProperties("pipelineName", "", Map.of(STATE, "sqlite", + SQLITE_DIRECTORY, "target")), SqlMemberRepository.class, SqlTreeNodeRepository.class), Arguments.of( diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index 3fd32b036..f8776ea2d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter diff --git a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml index 9f53b28f3..2bc21eaa1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml @@ -4,7 +4,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index 89d66428b..891d30d37 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml index 359040161..d5d3beb3d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-sink/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-repository-sink diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index 96b1b8240..bccb5531e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index 8215a0dd2..75623d89b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml index c788d298b..a622137e1 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-skolemisation-transformer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT ldio-skolemisation-transformer diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index 71d954b53..e46d5cd1b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index fb7ef6dd5..7711b0180 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index 0c821d69b..3cbb60f33 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index 779a9540f..3b0a74126 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-instrumentation/pom.xml b/ldi-orchestrator/ldio-instrumentation/pom.xml index 06b8f7f28..039459210 100644 --- a/ldi-orchestrator/ldio-instrumentation/pom.xml +++ b/ldi-orchestrator/ldio-instrumentation/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index e33c9a90c..eba713844 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.12.0 + 2.13.0-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 8f7efd2f6..c23553751 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 2.12.0 + 2.13.0-SNAPSHOT ldi-api @@ -81,7 +81,7 @@ 3.24.2 5.10.2 1.10.2 - 4.2.0 + 4.2.2 1.14.12 2.35.1 5.14.2