diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dd01eb67..8f974708 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,30 +56,26 @@ default:
General:
stage: unittests
image: maven:3.5.0-jdk-8
- retry: 2
services:
- - postgres:10.4
- - name: mcr.microsoft.com/mssql/server:latest
- alias: sqlserver
+ - name: docker:19.03.12-dind
+ # explicitly disable tls to avoid docker startup interruption
+ command: ["--tls=false"]
+ variables:
+ # Instruct Testcontainers to use the daemon of DinD.
+ DOCKER_HOST: "tcp://docker:2375"
+ # Instruct Docker not to start over TLS.
+ DOCKER_TLS_CERTDIR: ""
+ # Improve performance with overlayfs.
+ DOCKER_DRIVER: overlay2
script:
- 'mvn $MAVEN_CLI_OPTS -Dtest=$TEST test'
parallel:
matrix:
- - TEST: [Arguments_Test, Mapper_CSV_Test, Mapper_JSON_Test, Mapper_MySQL_Test, Mapper_Postgres_R2RML_Test, Mapper_WoT_Test, Arguments_Test_MySQL, Mapper_CSVW_Test, Mapper_LDES_Test, Mapper_ODS_Test, Mapper_Postgres_XML_Test, Mapper_XML_Test, Custom_RML_FnO_Mapper_CSV_Test, Mapper_EXCEL_Test, Mapper_MappingFile_URL_Test, Mapper_SPARQL_Test, Metadata_Test, Custom_RML_FnO_Mapper_JSON_Test, Mapper_HTML_Test, Mapper_MySQL_R2RML_Test, Mapper_Postgres_CSV_Test, Mapper_SQLServer_Test, Optimizations_Test, R2RMLConverterTest, Quad_Test, ReadmeTest, ReadmeFunctionTest]
+ - TEST: [Arguments_Test, Mapper_CSV_Test, Mapper_JSON_Test, Mapper_MySQL_Test, Mapper_Postgres_R2RML_Test, Mapper_WoT_Test, Arguments_Test_MySQL, Mapper_CSVW_Test, Mapper_LDES_Test, Mapper_ODS_Test, Mapper_OracleDB_Test, Mapper_Postgres_XML_Test, Mapper_XML_Test, Custom_RML_FnO_Mapper_CSV_Test, Mapper_EXCEL_Test, Mapper_MappingFile_URL_Test, Mapper_SPARQL_Test, Metadata_Test, Custom_RML_FnO_Mapper_JSON_Test, Mapper_HTML_Test, Mapper_MySQL_R2RML_Test, Mapper_Postgres_CSV_Test, Mapper_SQLServer_Test, Optimizations_Test, R2RMLConverterTest, Quad_Test, ReadmeTest, ReadmeFunctionTest]
except:
- master
- development
-Oracle DB:
- stage: unittests
- image: gitlab.ilabt.imec.be:4567/rml/util/mvn-oracle-docker:latest
- script:
- - '/entrypoint.sh'
- except:
- - master
- - development
- - tags # Gitlab CI bot cannot access Docker images
-
Docker Build:
stage: unittests
image: docker:latest
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1381ff18..e8b2257f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
## Unreleased
### Changed
+- Database tests are executed with a fresh instance of the required database in a Docker container
+- Database tests are now executed with JUnit5
+
+### Fixed
+- Dropped dependency on com.spotify.docker-client ([issue 231](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/231))
+- Running multiple pipelines should no longer interfere with each other ([issue 245](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/245))
+
+### Added
+- pom.xml: Added Testcontainers library dependencies for databases we test on
+- pom.xml: Added JUnit5 dependencies
- Check for changelog changes in a separate lint stage during CI.
## [6.0.0] - 2022-07-04
diff --git a/README.md b/README.md
index 72efdf43..a7a1c45f 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
[](https://search.maven.org/search?q=g:%22be.ugent.rml%22%20AND%20a:%22rmlmapper%22)
-The RMLMapper execute RML rules to generate Linked Data.
+The RMLMapper executes RML rules to generate Linked Data.
It is a Java library, which is available via the command line ([API docs online](https://javadoc.io/doc/be.ugent.rml/rmlmapper)).
The RMLMapper loads all data in memory, so be aware when working with big datasets.
@@ -78,7 +78,7 @@ This is the recommended way to get started with RMLMapper.
Do you want to build from source yourself? Check [Build](#build).
## Build
-The RMLMapper is build using Maven.
+The RMLMapper is built using Maven.
As it is also tested against Oracle (check [here](#accessing-oracle-database) for details),
it needs a specific set-up to run all tests.
That's why we recommend to build without testing: `mvn install -DskipTests=true`.
@@ -269,13 +269,18 @@ and up to which level metadata should be stored (dataset, triple, or term level
## Testing
-Run the tests via `test.sh`.
+### Command line
+Run the tests via `test.sh`.
+
+### IntelliJ
+Right-click `src/test/java` directory and select "Run 'All tests'".
#### Derived tests
Some tests (Excel, ODS) are derived from other tests (CSV) using a script (`./generate_spreadsheet_test_cases.sh`)
### RDBs
-Make sure you have [Docker](https://www.docker.com) running.
+Make sure you have [Docker](https://www.docker.com) running. On Unix, others read-write permission (006) is required on `/var/run/docker.sock` in order to run the tests.
+The tests will fail otherwise, as Testcontainers can't spin up the container.
#### Problems
* A problem with Docker (can't start the container) causes the SQLServer tests to fail locally. These tests will always succeed locally.
@@ -290,7 +295,10 @@ Make sure you have [Docker](https://www.docker.com) running.
| com.opencsv opencsv | Apache License 2.0 |
| commons-cli commons-cli | Apache License 2.0 |
| org.eclipse.rdf4j rdf4j-runtime | Eclipse Public License 1.0 |
-| junit junit | Eclipse Public License 1.0 |
+| org.junit.jupiter junit-jupiter-engine | Eclipse Public License v2.0 |
+| org.junit.jupiter junit-jupiter-api | Eclipse Public License v2.0 |
+| org.junit.jupiter junit-jupiter-params | Eclipse Public License v2.0 |
+| org.junit.vintage junit-vintage-engine | Eclipse Public License v2.0 |
| com.jayway.jsonpath json-path | Apache License 2.0 |
| javax.xml.parsers jaxp-api | Apache License 2.0 |
| org.jsoup | MIT |
@@ -298,7 +306,6 @@ Make sure you have [Docker](https://www.docker.com) running.
| ch.vorbuger.mariaDB4j mariaDB4j | Apache License 2.0 |
| postgresql postgresql | BSD |
| com.microsoft.sqlserver mssql-jdbc | MIT |
-| com.spotify docker-client | Apache License 2.0 |
| com.fasterxml.jackson.core jackson-core | Apache License 2.0 |
| org.eclipse.jetty jetty-server | Eclipse Public License 1.0 & Apache License 2.0 |
| org.eclipse.jetty jetty-security | Eclipse Public License 1.0 & Apache License 2.0 |
@@ -310,6 +317,11 @@ Make sure you have [Docker](https://www.docker.com) running.
| com.github.fnoio function-agent-java | MIT |
| com.github.fnoio idlab-functions-java | MIT |
| net.sf.saxon | Mozilla Public License version 2.0 |
+| org.mybatis mybatis | Apache License 2.0 |
+| org.testcontainers postgresql | MIT |
+| org.testcontainers mysql | MIT |
+| org.testcontainers mssqlserver | MIT |
+| org.testcontainers oracle-xe | MIT |
## Commercial Support
diff --git a/pom.xml b/pom.xml
index d9b0ecc5..2e535dce 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@
UTF-8
- 4.13.2
+ 5.8.2
8
8
@@ -143,8 +143,26 @@
2.5.5
- junit
- junit
+ org.junit.jupiter
+ junit-jupiter-engine
+ ${junit.version}
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ ${junit.version}
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-params
+ ${junit.version}
+ test
+
+
+ org.junit.vintage
+ junit-vintage-engine
${junit.version}
test
@@ -194,10 +212,9 @@
test
- com.spotify
- docker-client
- 8.16.0
- test
+ com.oracle.database.jdbc
+ ojdbc8
+ 21.6.0.0.1
com.fasterxml.jackson.core
@@ -313,7 +330,44 @@
1.0-1
test
-
+
+
+ org.testcontainers
+ testcontainers
+ 1.17.2
+ test
+
+
+ org.testcontainers
+ postgresql
+ 1.17.2
+ test
+
+
+ org.testcontainers
+ mysql
+ 1.17.2
+ test
+
+
+ org.testcontainers
+ mssqlserver
+ 1.17.2
+ test
+
+
+ org.testcontainers
+ oracle-xe
+ 1.17.2
+ test
+
+
+
+
+ org.mybatis
+ mybatis
+ 3.5.10
+
diff --git a/src/main/java/be/ugent/rml/access/AccessFactory.java b/src/main/java/be/ugent/rml/access/AccessFactory.java
index b0facaef..0ff5009c 100644
--- a/src/main/java/be/ugent/rml/access/AccessFactory.java
+++ b/src/main/java/be/ugent/rml/access/AccessFactory.java
@@ -251,7 +251,6 @@ private RDBAccess getRDBAccess(QuadStore rmlStore, Term source, Term logicalSour
}
String dsn = dsnObject.get(0).getValue();
- dsn = dsn.substring(dsn.indexOf("//") + 2);
// - SQL query
String query;
@@ -291,13 +290,14 @@ private RDBAccess getRDBAccess(QuadStore rmlStore, Term source, Term logicalSour
// - ContentType
List contentType = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalSource, new NamedNode(NAMESPACES.RML + "referenceFormulation"), null));
- return new RDBAccess(dsn, database, username, password, query, (contentType.isEmpty()? "text/csv" : contentType.get(0).getValue()));
+
+ return new RDBAccess(dsn, database, username, password, query, (contentType.isEmpty() ? "text/csv" : contentType.get(0).getValue()));
}
/**
* This method returns a SPARQLResultFormat based on the result formats and reference formulations.
* @param resultFormats the result formats used to determine the SPARQLResultFormat.
- * @param referenceFormulations the reference formulations used to to determine the SPARQLResultFormat.
+ * @param referenceFormulations the reference formulations used to determine the SPARQLResultFormat.
* @return a SPARQLResultFormat.
*/
private SPARQLResultFormat getSPARQLResultFormat(List resultFormats, List referenceFormulations) {
diff --git a/src/main/java/be/ugent/rml/access/RDBAccess.java b/src/main/java/be/ugent/rml/access/RDBAccess.java
index 4af8d351..ca4fb5e4 100644
--- a/src/main/java/be/ugent/rml/access/RDBAccess.java
+++ b/src/main/java/be/ugent/rml/access/RDBAccess.java
@@ -47,7 +47,7 @@ public class RDBAccess implements Access {
/**
- * This constructor takes as arguments the dsn, database, username, password, query, and content type.
+ * This constructor takes as arguments the dsn, database, username, password, query, content type
*
* @param dsn the data source name.
* @param databaseType the database type.
@@ -76,59 +76,44 @@ public InputStream getInputStream() throws IOException, SQLException, ClassNotFo
// JDBC objects
Connection connection = null;
Statement statement = null;
- String jdbcDriver = databaseType.getDriver();
- String jdbcDSN = "jdbc:" + databaseType.getJDBCPrefix() + "//" + dsn;
- InputStream inputStream = null;
+ InputStream inputStream;
try {
- // Register JDBC driver
- Class.forName(jdbcDriver);
-
// Open connection
- String connectionString = jdbcDSN;
- boolean alreadySomeQueryParametersPresent = false;
-
- if (username != null && !username.equals("") && password != null && !password.equals("")) {
- if (databaseType == DatabaseType.ORACLE) {
- connectionString = connectionString.replace(":@", ":" + username + "/" + password + "@");
- } else if (!connectionString.contains("user=")) {
- connectionString += "?user=" + username + "&password=" + password;
- alreadySomeQueryParametersPresent = true;
- }
- }
+ boolean alreadySomeQueryParametersPresent = dsn.contains("?");
+
if (databaseType == DatabaseType.MYSQL) {
+ StringBuilder parametersSB = new StringBuilder();
+
if (alreadySomeQueryParametersPresent) {
- connectionString += "&";
+ parametersSB.append("&");
} else {
- connectionString += "?";
+ parametersSB.append("?");
}
- connectionString += "serverTimezone=UTC&useSSL=false";
+ parametersSB.append("serverTimezone=UTC&useSSL=false");
+
+ dsn += parametersSB;
}
if (databaseType == DatabaseType.SQL_SERVER) {
- connectionString = connectionString.replaceAll("\\?|&", ";");
-
- if (!connectionString.endsWith(";")) {
- connectionString += ";";
- }
+ dsn = dsn.replaceAll("[?&]", ";");
}
- connection = DriverManager.getConnection(connectionString);
+
+ connection = DriverManager.getConnection(dsn, username, password);
// Execute query
statement = connection.createStatement();
+
ResultSet rs = statement.executeQuery(query);
- switch (contentType) {
- case NAMESPACES.QL + "XPath" :
- inputStream = getXMLInputStream(rs);
- break;
- default:
- inputStream = getCSVInputStream(rs);
+ if ((NAMESPACES.QL + "XPath").equals(contentType)) {
+ inputStream = getXMLInputStream(rs);
+ } else {
+ inputStream = getCSVInputStream(rs);
}
-
// Clean-up environment
rs.close();
statement.close();
diff --git a/src/test/java/be/ugent/rml/Arguments_Test_MySQL.java b/src/test/java/be/ugent/rml/Arguments_Test_MySQL.java
index 2c03f5f3..27cda092 100644
--- a/src/test/java/be/ugent/rml/Arguments_Test_MySQL.java
+++ b/src/test/java/be/ugent/rml/Arguments_Test_MySQL.java
@@ -1,49 +1,32 @@
package be.ugent.rml;
import be.ugent.rml.cli.Main;
-import ch.vorburger.exec.ManagedProcessException;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.LoggerFactory;
import java.io.File;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
public class Arguments_Test_MySQL extends MySQLTestCore {
- private static String CONNECTIONSTRING;
-
@BeforeClass
- public static void before() throws Exception {
- int portNumber = Utils.getFreePortNumber();
- CONNECTIONSTRING = getConnectionString(portNumber);
- mysqlDB = setUpMySQLDBInstance(portNumber);
- }
-
- @AfterClass
- public static void after() throws ManagedProcessException {
- stopDBs();
+ public static void beforeClass() {
+ logger = LoggerFactory.getLogger(Arguments_Test_MySQL.class);
}
@Test
public void executeR2RML() throws Exception {
- String cwd = Utils.getFile( "argument/r2rml").getAbsolutePath();
+ String cwd = (new File("./src/test/resources/argument/r2rml")).getAbsolutePath();
String mappingFilePath = (new File(cwd, "mapping.r2rml.ttl")).getAbsolutePath();
String actualPath = (new File("./generated_output.nq")).getAbsolutePath();
- String expectedPath = (new File( cwd, "output.nq")).getAbsolutePath();
- String resourcePath = "argument/r2rml/resource.sql";
+ String expectedPath = (new File(cwd, "output.nq")).getAbsolutePath();
+ String resourcePath = "src/test/resources/argument/r2rml/resource.sql";
- // Get SQL resource
- try {
- mysqlDB.source(resourcePath);
- } catch (ManagedProcessException e) {
- e.printStackTrace();
- fail();
- }
+ prepareDatabase(resourcePath, "root", "");
- Main.main(("-m " + mappingFilePath + " -o " + actualPath + " --r2rml-jdbcDSN " + CONNECTIONSTRING + " --r2rml-username root -v").split(" "), cwd);
+ Main.main(("-m " + mappingFilePath + " -o " + actualPath + " --r2rml-jdbcDSN " + dbURL + " --r2rml-username root -v").split(" "), cwd);
compareFiles(
expectedPath,
actualPath,
diff --git a/src/test/java/be/ugent/rml/DBTestCore.java b/src/test/java/be/ugent/rml/DBTestCore.java
index d4759fa9..a0a56cba 100644
--- a/src/test/java/be/ugent/rml/DBTestCore.java
+++ b/src/test/java/be/ugent/rml/DBTestCore.java
@@ -1,20 +1,70 @@
package be.ugent.rml;
-import java.io.File;
-import java.io.IOException;
+import org.apache.ibatis.jdbc.ScriptRunner;
+import org.junit.AfterClass;
+import org.slf4j.Logger;
+import org.testcontainers.containers.JdbcDatabaseContainer;
+
+import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Random;
public abstract class DBTestCore extends TestCore {
-
+ protected static Logger logger;
protected static HashSet tempFiles = new HashSet<>();
- /*protected static String replaceDSNInMappingFile(String path, String connectionString) {
+ // Testcontainers library uses SELF-typing, which will be removed in later versions. That's why >.
+ // omitting > causes compiler to complain
+
+ // This class has no information or way of knowing which specific JDBC container is required.
+ // It is the child's responsibility to initialize this field in its constructor,
+ // as only the child knows what container is required
+ protected JdbcDatabaseContainer> container;
+ protected String dbURL;
+
+ protected final String USERNAME;
+ protected final String PASSWORD;
+ protected final String DOCKER_TAG;
+
+ protected DBTestCore(String username, String password, String dockerTag) {
+ this.USERNAME = username;
+ this.PASSWORD = password;
+ this.DOCKER_TAG = dockerTag;
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ // Make sure all tempFiles are removed
+ int counter = 0;
+ while (!tempFiles.isEmpty()) {
+ for (Iterator i = tempFiles.iterator(); i.hasNext(); ) {
+ try {
+ if (new File(i.next()).delete()) {
+ i.remove();
+ }
+ } catch (Exception ex) {
+ counter++;
+ ex.printStackTrace();
+ // Prevent infinity loops
+ if (counter > 100) {
+ throw new Error("Could not remove all temp mapping files.");
+ }
+ }
+ }
+ }
+ }
+
+ protected static String replaceDSNInMappingFile(String path, String connectionString) {
try {
// Read mapping file
String mapping = new String(Files.readAllBytes(Paths.get(Utils.getFile(path, null).getAbsolutePath())), StandardCharsets.UTF_8);
@@ -36,7 +86,7 @@ public abstract class DBTestCore extends TestCore {
} catch (IOException ex) {
throw new Error(ex.getMessage());
}
- }*/
+ }
protected static String createTempMappingFile(String path) {
try {
@@ -77,7 +127,9 @@ protected static void deleteTempMappingFile(String absolutePath) {
private static String writeMappingFile(String mapping, String path) {
try {
- String fileName = Integer.toString(Math.abs(mapping.hashCode())) + "tempMapping.ttl";
+ // when multiple tests are running with the same mapping, they can remove each other's mapping files
+ // adding a random number prevents this behaviour
+ String fileName = Math.abs(new Random().nextInt()) + mapping.hashCode() + "tempMapping.ttl";
Path file = Paths.get(fileName);
Files.write(file, Arrays.asList(mapping.split("\n")));
@@ -90,4 +142,15 @@ private static String writeMappingFile(String mapping, String path) {
}
}
+ protected void prepareDatabase(String path, String username, String password) {
+ try (Connection conn = DriverManager.getConnection(dbURL, username, password)) {
+ ScriptRunner runner = new ScriptRunner(conn);
+ Reader reader = new BufferedReader(new FileReader(path));
+ runner.setLogWriter(null); // ScriptRunner will output the contents of the SQL file to System.out by default
+
+ runner.runScript(reader);
+ } catch (SQLException | FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/src/test/java/be/ugent/rml/Mapper_MySQL_R2RML_Test.java b/src/test/java/be/ugent/rml/Mapper_MySQL_R2RML_Test.java
index 6f505a06..2034e55e 100644
--- a/src/test/java/be/ugent/rml/Mapper_MySQL_R2RML_Test.java
+++ b/src/test/java/be/ugent/rml/Mapper_MySQL_R2RML_Test.java
@@ -1,13 +1,11 @@
package be.ugent.rml;
-import ch.vorburger.exec.ManagedProcessException;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
@@ -16,42 +14,13 @@
import static be.ugent.rml.TestStrictMode.*;
// Adapted from Mapper_MySQL_Test to include connection options for R2RML mapping files.
-@RunWith(Parameterized.class)
-public class Mapper_MySQL_R2RML_Test extends MySQLTestCore{
-
- private static String CONNECTIONSTRING;
-
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class Mapper_MySQL_R2RML_Test extends MySQLTestCore {
@BeforeClass
- public static void before() throws Exception {
- int portNumber = Utils.getFreePortNumber();
- CONNECTIONSTRING = getConnectionString(portNumber);
- mysqlDB = setUpMySQLDBInstance(portNumber);
-
- // add RDB connection options
- mappingOptions = new HashMap<>();
- mappingOptions.put("jdbcDSN", CONNECTIONSTRING);
- mappingOptions.put("username", "root");
- mappingOptions.put("password", "");
+ public static void beforeClass() {
+ logger = LoggerFactory.getLogger(Mapper_MySQL_R2RML_Test.class);
}
- @AfterClass
- public static void after() throws ManagedProcessException {
- stopDBs();
- }
-
- @Parameterized.Parameter(0)
- public String testCaseName;
-
- @Parameterized.Parameter(1)
- public Class extends Exception> expectedException;
-
- @Parameterized.Parameter(2)
- public TestStrictMode testStrictMode;
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Parameterized.Parameters(name = "{index}: mySQL_{0}")
public static Iterable