ci: finalise 2.13.0 (#750)

Informatievlaanderen · Jan 24, 2025 · 085817f · 085817f
2 parents a1eac53 + 8597a93
commit 085817f
Show file tree

Hide file tree

Showing 144 changed files with 867 additions and 1,077 deletions.
diff --git a/.github/workflows/deploy-documentation-old.yml b/.github/workflows/deploy-documentation-old.yml
diff --git a/.github/workflows/deploy-documentation.yml b/.github/workflows/deploy-documentation.yml
@@ -34,7 +34,7 @@ jobs:
           export VERSION=$(mvn help:evaluate -Dexpression="project.version" -q -DforceStdout)
           echo "version=$VERSION" >> $>> $GITHUB_OUTPUT
       - name: Setup Ruby
-        uses: ruby/setup-ruby@0a29871fe2b0200a17a4497bae54fe5df0d973aa
+        uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.0' # Not needed with a .ruby-version file
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically

diff --git a/docs/_ldi-nifi/index.md b/docs/_ldi-nifi/index.md
@@ -15,28 +15,61 @@ The processors can be imported into a NiFi docker instance via volume binding:
 1. Create a `docker-compose.yml` file with the following content in a new directory
     ````yaml
     services:
-      nifi:
-        image: apache/nifi:2.0.0
-        environment:
-          SINGLE_USER_CREDENTIALS_USERNAME: admin
-          SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB
-        ports:
-          - 8443:8443
-        volumes:
-          - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw
+        nifi:
+    image: apache/nifi:2.0.0
+    environment:
+      SINGLE_USER_CREDENTIALS_USERNAME: admin
+      SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB
+    ports:
+      - 8443:8443
+    volumes:
+      - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw
+      - ./nifi-drivers:/opt/nifi/nifi-current/lib_temp:rw
+    entrypoint: [ "/bin/bash",
+                  "-c",
+                  "cp -r /opt/nifi/nifi-current/lib_temp/* /opt/nifi/nifi-current/lib/ && exec /opt/nifi/scripts/start.sh" ]
     ````
 2. Create a directory `nifi-ext` in your current directory.
 3. Download either the `...-nar-bundle.jar` and unpack this or download the individual required processors (.nar extension) from the [nexus repository].
    Next, place the required processors in the `nifi-ext` directory.
-4. Finally, start your instance.
+4. Additionally, for using any processor with a persistence layer, you will need to download the needed drivers.
+   Place these in the `nifi-drivers` directory.
+   Possible drivers are:
+   - [PostgreSQL driver](https://jdbc.postgresql.org/download.html)
+   - [SQLite driver](https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc)
+5. Finally, start your instance.
     ````shell
     docker compose up
     ````
-5. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1
-6. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group.
+6. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1
+7. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group.
 
 {: .note }
 All documentation and notes about configuration are available in the NiFi component itself.
 
+## Setting up Database Connection Pools
+
+As of LDI 2.13.0, some processors require a database connection pool to be set up.
+Although this is a pure NiFi configuration, a minimal explanation will be mentioned here. \
+For more details, please refer to
+the [NiFi documentation on DBCP](https://nifi.apache.org/components/org.apache.nifi.dbcp.DBCPConnectionPool/).
+This can be done by following these steps:
+
+1. Go to the `Controller Services` tab in the NiFi UI.
+2. Click on the `Create new service` button.
+3. Select the `DBCPConnectionPool` service.
+4. Configure the service with the required properties:
+   - `Database Connection URL`,
+      - `jdbc:sqlite:/path/to/database.db` for SQLite
+      - `jdbc:postgresql://localhost:5432/database` for PostgreSQL,
+   - `Database Driver Class Name`
+      - `org.sqlite.JDBC` for SQLite
+      - `org.postgresql.Driver` for PostgreSQL,
+   - `Database User`,
+   - `Password`,
+5. Save the service.
+6. Enable the service.
+7. Use the service in the processors that require it.
+
 [Apache NiFi]: https://nifi.apache.org/
 [nexus repository]: https://s01.oss.sonatype.org/#nexus-search;quick~be.vlaanderen.informatievlaanderen.ldes.ldi.nifi
diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes</groupId>
         <artifactId>linked-data-interactions</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/ldi-core/change-detection-filter/pom.xml b/ldi-core/change-detection-filter/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>change-detection-filter</artifactId>

diff --git a/...lter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java b/...lter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java
@@ -27,11 +27,9 @@ public class ChangeDetectionFilter implements LdiOneToOneTransformer {
 	private static final Lang NORMALIZING_LANG = Lang.NQUADS;
 	private static final MediaType NORMALIZING_MEDIA_TYPE = MediaType.N_QUADS;
 	private final HashedStateMemberRepository hashedStateMemberRepository;
-	private final boolean keepState;
 
-	public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository, boolean keepState) {
+	public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository) {
 		this.hashedStateMemberRepository = hashedStateMemberRepository;
-		this.keepState = keepState;
 	}
 
 	/**
@@ -54,16 +52,10 @@ public Model transform(Model model) {
 		return model;
 	}
 
-	/**
-	 * Clean up the resources, should be called when the component is not used anymore
-	 */
-	public void destroyState() {
-		if (!keepState) {
-			hashedStateMemberRepository.destroyState();
-		}
+	public void close() {
+		hashedStateMemberRepository.close();
 	}
 
-
 	private Resource getSingleNamedNodeFromStateObject(Model model) {
 		final List<Resource> namedNodes = model.listSubjects().filterDrop(RDFNode::isAnon).toList();
 		if (namedNodes.size() != 1) {

diff --git a/...be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java b/...be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java
@@ -18,5 +18,5 @@ public interface HashedStateMemberRepository {
 	/**
 	 * Clean up the resources, should be called when the repository is not used anymore
 	 */
-	void destroyState();
+	void close();
 }
diff --git a/...ormatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java b/...ormatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java
@@ -3,7 +3,8 @@
 import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
 
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
 
 public class InMemoryHashedStateMemberRepository implements HashedStateMemberRepository {
 	private final Map<String, HashedStateMember> members;
@@ -24,7 +25,7 @@ public void saveHashedStateMember(HashedStateMember hashedStateMember) {
 	}
 
 	@Override
-	public void destroyState() {
+	public void close() {
 		members.clear();
 	}
 }
diff --git a/...nderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java b/...nderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java
@@ -1,6 +1,5 @@
 package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql;
 
-import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMemberEntity;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
@@ -9,14 +8,10 @@
 import javax.persistence.EntityManager;
 
 public class SqlHashedStateMemberRepository implements HashedStateMemberRepository {
-	private final EntityManagerFactory entityManagerFactory;
 	private final EntityManager entityManager;
-	private final String instanceName;
 
-	public SqlHashedStateMemberRepository(EntityManagerFactory entityManagerFactory, String instanceName) {
-		this.entityManagerFactory = entityManagerFactory;
-		this.entityManager = entityManagerFactory.getEntityManager();
-		this.instanceName = instanceName;
+	public SqlHashedStateMemberRepository(EntityManager entityManager) {
+		this.entityManager = entityManager;
 	}
 
 	@Override
@@ -38,7 +33,7 @@ public void saveHashedStateMember(HashedStateMember hashedStateMember) {
 	}
 
 	@Override
-	public void destroyState() {
-		entityManagerFactory.destroyState(instanceName);
+	public void close() {
+		entityManager.close();
 	}
 }
diff --git a/...src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterSteps.java b/...src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterSteps.java
@@ -1,11 +1,9 @@
 package be.vlaanderen.informatievlaanderen.ldes.ldi;
 
-import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresEntityManagerFactory;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.postgres.PostgresProperties;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.inmemory.InMemoryHashedStateMemberRepository;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql.SqlHashedStateMemberRepository;
-import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteEntityManagerFactory;
 import be.vlaanderen.informatievlaanderen.ldes.ldi.sqlite.SqliteProperties;
 import io.cucumber.java.en.Given;
 import io.cucumber.java.en.Then;
@@ -34,14 +32,14 @@ public void teardown() {
 	@Given("A ChangeDetectionFilter with state persistence strategy MEMORY")
 	public void aChangeDetectionFilterWithStatePersistenceStrategyMEMORY() {
 		HashedStateMemberRepository hashedStateMemberRepository = new InMemoryHashedStateMemberRepository();
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Given("A ChangeDetectionFilter with state persistence strategy SQLITE")
 	public void aChangeDetectionFilterWithStatePersistenceStrategySQLITE() {
-		final var emf = SqliteEntityManagerFactory.getInstance(new SqliteProperties(DATABASE_INSTANCE_NAME, false));
-		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(emf, DATABASE_INSTANCE_NAME);
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		var entityManager = HibernateUtil.createEntityManagerFromProperties(new SqliteProperties("target", DATABASE_INSTANCE_NAME, false).getProperties());
+		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(entityManager);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Given("A ChangeDetectionFilter with state persistence strategy POSTGRES")
@@ -55,9 +53,9 @@ public void aChangeDetectionFilterWithStatePersistenceStrategyPOSTGRES() {
 		PostgresProperties postgresProperties = new PostgresProperties(postgreSQLContainer.getJdbcUrl(),
 				postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword(), false);
 
-		final var emf = PostgresEntityManagerFactory.getInstance(DATABASE_INSTANCE_NAME, postgresProperties.getProperties());
-		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(emf, DATABASE_INSTANCE_NAME);
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		var entityManager = HibernateUtil.createEntityManagerFromProperties(postgresProperties.getProperties());
+		HashedStateMemberRepository hashedStateMemberRepository = new SqlHashedStateMemberRepository(entityManager);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Then("The filtered member is not empty")
@@ -85,6 +83,6 @@ public void theFilteredMemberIsIsomorphicWith(String fileName) {
 
 	@Then("The filter is destroyed")
 	public void theFilterIsDestroyed() {
-		changeDetectionFilter.destroyState();
+		changeDetectionFilter.close();
 	}
 }
diff --git a/.../src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterTest.java b/.../src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilterTest.java
@@ -11,7 +11,6 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
-import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
@@ -20,7 +19,7 @@
 import static org.assertj.core.api.Assertions.*;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.argThat;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
 class ChangeDetectionFilterTest {
@@ -31,7 +30,7 @@ class ChangeDetectionFilterTest {
 
 	@BeforeEach
 	void setUp() {
-		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository, false);
+		changeDetectionFilter = new ChangeDetectionFilter(hashedStateMemberRepository);
 	}
 
 	@Test
@@ -71,18 +70,6 @@ void given_EmptyRepository_when_FilterModel_then_ReturnExactModel() {
 		assertThat(filteredModel).isSameAs(modelToFilter);
 	}
 
-	@ParameterizedTest
-	@ValueSource(booleans = {true, false})
-	void given_KeepState_when_DestroyFilterState_then_VerifyRepositoryDestroy(boolean keepState) {
-		final int expectedDestroyRepositoryInvocations = keepState ? 0 : 1;
-		final var filter = new ChangeDetectionFilter(hashedStateMemberRepository, keepState);
-
-		filter.destroyState();
-
-		verify(hashedStateMemberRepository, times(expectedDestroyRepositoryInvocations)).destroyState();
-	}
-
-
 	private static Stream<Arguments> supplyInvalidTriples() {
 		final String twoStateObjects = """
 				<http://test-data/mobility-hindrance/1/2> <http://purl.org/dc/terms/isVersionOf> <http://test-data/mobility-hindrance/1> .

diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>file-archiving</artifactId>

diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>geojson-to-wkt</artifactId>

diff --git a/ldi-core/http-sparql-out/pom.xml b/ldi-core/http-sparql-out/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>http-sparql-out</artifactId>

diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>json-to-ld-adapter</artifactId>

diff --git a/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml b/ldi-core/ldes-client/event-stream-properties-fetcher/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.client</groupId>
         <artifactId>ldes-client</artifactId>
-        <version>2.12.0</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>event-stream-properties-fetcher</artifactId>