tests: refactor RDB tests to use Docker

Dockerize all tests for RDBs to make test more reliable and easier to run

.gitlab-ci.yml

@@ -56,30 +56,26 @@ default:
 General:
   stage: unittests
   image: maven:3.5.0-jdk-8
-  retry: 2
   services:
-    - postgres:10.4
-    - name: mcr.microsoft.com/mssql/server:latest
-      alias: sqlserver
+    - name: docker:19.03.12-dind
+      # explicitly disable tls to avoid docker startup interruption
+      command: ["--tls=false"]
+  variables:
+    # Instruct Testcontainers to use the daemon of DinD.
+    DOCKER_HOST: "tcp://docker:2375"
+    # Instruct Docker not to start over TLS.
+    DOCKER_TLS_CERTDIR: ""
+    # Improve performance with overlayfs.
+    DOCKER_DRIVER: overlay2
   script:
     - 'mvn $MAVEN_CLI_OPTS -Dtest=$TEST test'
   parallel:
     matrix:
-      - TEST: [Arguments_Test, Mapper_CSV_Test, Mapper_JSON_Test, Mapper_MySQL_Test, Mapper_Postgres_R2RML_Test, Mapper_WoT_Test, Arguments_Test_MySQL, Mapper_CSVW_Test, Mapper_LDES_Test, Mapper_ODS_Test, Mapper_Postgres_XML_Test, Mapper_XML_Test, Custom_RML_FnO_Mapper_CSV_Test, Mapper_EXCEL_Test, Mapper_MappingFile_URL_Test, Mapper_SPARQL_Test, Metadata_Test, Custom_RML_FnO_Mapper_JSON_Test, Mapper_HTML_Test, Mapper_MySQL_R2RML_Test, Mapper_Postgres_CSV_Test, Mapper_SQLServer_Test, Optimizations_Test, R2RMLConverterTest, Quad_Test, ReadmeTest, ReadmeFunctionTest]
+      - TEST: [Arguments_Test, Mapper_CSV_Test, Mapper_JSON_Test, Mapper_MySQL_Test, Mapper_Postgres_R2RML_Test, Mapper_WoT_Test, Arguments_Test_MySQL, Mapper_CSVW_Test, Mapper_LDES_Test, Mapper_ODS_Test, Mapper_OracleDB_Test, Mapper_Postgres_XML_Test, Mapper_XML_Test, Custom_RML_FnO_Mapper_CSV_Test, Mapper_EXCEL_Test, Mapper_MappingFile_URL_Test, Mapper_SPARQL_Test, Metadata_Test, Custom_RML_FnO_Mapper_JSON_Test, Mapper_HTML_Test, Mapper_MySQL_R2RML_Test, Mapper_Postgres_CSV_Test, Mapper_SQLServer_Test, Optimizations_Test, R2RMLConverterTest, Quad_Test, ReadmeTest, ReadmeFunctionTest]
   except:
     - master
     - development
 
-Oracle DB:
-  stage: unittests
-  image: gitlab.ilabt.imec.be:4567/rml/util/mvn-oracle-docker:latest
-  script:
-    - '/entrypoint.sh'
-  except:
-    - master
-    - development
-    - tags # Gitlab CI bot cannot access Docker images
-
 Docker Build:
   stage: unittests
   image: docker:latest

CHANGELOG.md

@@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## Unreleased
 
 ### Changed
+- Database tests are executed with a fresh instance of the required database in a Docker container
+- Database tests are now executed with JUnit5
+
+### Fixed
+- Dropped dependency on com.spotify.docker-client ([issue 231](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/231))
+- Running multiple pipelines should no longer interfere with each other ([issue 245](https://gitlab.ilabt.imec.be/rml/proc/rmlmapper-java/-/issues/245))
+
+### Added
+- pom.xml: Added Testcontainers library dependencies for databases we test on
+- pom.xml: Added JUnit5 dependencies
 - Check for changelog changes in a separate lint stage during CI.
 
 ## [6.0.0] - 2022-07-04

README.md

@@ -2,7 +2,7 @@
 
 [![Maven Central](https://img.shields.io/maven-central/v/be.ugent.rml/rmlmapper.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22be.ugent.rml%22%20AND%20a:%22rmlmapper%22)
 
-The RMLMapper execute RML rules to generate Linked Data.
+The RMLMapper executes RML rules to generate Linked Data.
 It is a Java library, which is available via the command line ([API docs online](https://javadoc.io/doc/be.ugent.rml/rmlmapper)).
 The RMLMapper loads all data in memory, so be aware when working with big datasets.
 
@@ -78,7 +78,7 @@ This is the recommended way to get started with RMLMapper.
 Do you want to build from source yourself? Check [Build](#build).
 
 ## Build
-The RMLMapper is build using Maven.
+The RMLMapper is built using Maven.
 As it is also tested against Oracle (check [here](#accessing-oracle-database) for details),
 it needs a specific set-up to run all tests.
 That's why we recommend to build without testing: `mvn install -DskipTests=true`.
@@ -269,13 +269,18 @@ and up to which level metadata should be stored (dataset, triple, or term level
 
 ## Testing
 
-Run the tests via `test.sh`.
+### Command line
+Run the tests via `test.sh`. 
+
+### IntelliJ
+Right-click `src/test/java` directory and select "Run 'All tests'". 
 
 #### Derived tests
 Some tests (Excel, ODS) are derived from other tests (CSV) using a script (`./generate_spreadsheet_test_cases.sh`)
 
 ### RDBs
-Make sure you have [Docker](https://www.docker.com) running.
+Make sure you have [Docker](https://www.docker.com) running. On Unix, others read-write permission (006) is required on `/var/run/docker.sock` in order to run the tests.
+The tests will fail otherwise, as Testcontainers can't spin up the container. 
 
 #### Problems
 * A problem with Docker (can't start the container) causes the SQLServer tests to fail locally. These tests will always succeed locally.
@@ -290,15 +295,17 @@ Make sure you have [Docker](https://www.docker.com) running.
 |           com.opencsv opencsv           | Apache License 2.0                                                 |
 |         commons-cli commons-cli         | Apache License 2.0                                                 |
 |     org.eclipse.rdf4j rdf4j-runtime     | Eclipse Public License 1.0                                         |
-|               junit junit               | Eclipse Public License 1.0                                         |
+| org.junit.jupiter junit-jupiter-engine  | Eclipse Public License v2.0                                        |
+|   org.junit.jupiter junit-jupiter-api   | Eclipse Public License v2.0                                        |
+| org.junit.jupiter junit-jupiter-params  | Eclipse Public License v2.0                                        |
+| org.junit.vintage junit-vintage-engine  | Eclipse Public License v2.0                                        |
 |      com.jayway.jsonpath json-path      | Apache License 2.0                                                 |
 |       javax.xml.parsers jaxp-api        | Apache License 2.0                                                 |
 |                org.jsoup                | MIT                                                                |
 |       mysql mysql-connector-java        | GNU General Public License v2.0                                    |
 |     ch.vorbuger.mariaDB4j mariaDB4j     | Apache License 2.0                                                 |
 |          postgresql postgresql          | BSD                                                                |
 |   com.microsoft.sqlserver mssql-jdbc    | MIT                                                                |
-|        com.spotify docker-client        | Apache License 2.0                                                 |
 | com.fasterxml.jackson.core jackson-core | Apache License 2.0                                                 |
 |     org.eclipse.jetty jetty-server      | Eclipse Public License 1.0 & Apache License 2.0                    |
 |    org.eclipse.jetty jetty-security     | Eclipse Public License 1.0 & Apache License 2.0                    |
@@ -310,6 +317,11 @@ Make sure you have [Docker](https://www.docker.com) running.
 |  com.github.fnoio function-agent-java   | MIT                                                                |
 |  com.github.fnoio idlab-functions-java  | MIT                                                                |
 |              net.sf.saxon               | Mozilla Public License version 2.0                                 |
+|           org.mybatis mybatis           | Apache License 2.0                                                 |
+|      org.testcontainers postgresql      | MIT                                                                |
+|        org.testcontainers mysql         | MIT                                                                |
+|     org.testcontainers mssqlserver      | MIT                                                                |
+|      org.testcontainers oracle-xe       | MIT                                                                |
 
 ## Commercial Support
 

pom.xml

@@ -35,7 +35,7 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <junit.version>4.13.2</junit.version>
+        <junit.version>5.8.2</junit.version>
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
     </properties>
@@ -143,8 +143,26 @@
             <version>2.5.5</version>
         </dependency>
         <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.vintage</groupId>
+            <artifactId>junit-vintage-engine</artifactId>
             <version>${junit.version}</version>
             <scope>test</scope>
         </dependency>
@@ -194,10 +212,9 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.spotify</groupId>
-            <artifactId>docker-client</artifactId>
-            <version>8.16.0</version>
-            <scope>test</scope>
+            <groupId>com.oracle.database.jdbc</groupId>
+            <artifactId>ojdbc8</artifactId>
+            <version>21.6.0.0.1</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
@@ -313,7 +330,44 @@
             <version>1.0-1</version>
             <scope>test</scope>
         </dependency>
-
+        <!--< testcontainer dependencies > -->
+        <dependency>
+            <groupId>org.testcontainers</groupId>  <!-- for running tests with dockerized databases -->
+            <artifactId>testcontainers</artifactId>
+            <version>1.17.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>1.17.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mysql</artifactId>
+            <version>1.17.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>mssqlserver</artifactId>
+            <version>1.17.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>oracle-xe</artifactId>
+            <version>1.17.2</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- < testcontainer dependencies /> -->
+        <!-- For easy parsing and execution of SQL files -->
+        <dependency>
+            <groupId>org.mybatis</groupId>
+            <artifactId>mybatis</artifactId>
+            <version>3.5.10</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/be/ugent/rml/access/AccessFactory.java

@@ -251,7 +251,6 @@ private RDBAccess getRDBAccess(QuadStore rmlStore, Term source, Term logicalSour
         }
 
         String dsn = dsnObject.get(0).getValue();
-        dsn = dsn.substring(dsn.indexOf("//") + 2);
 
         // - SQL query
         String query;
@@ -291,13 +290,14 @@ private RDBAccess getRDBAccess(QuadStore rmlStore, Term source, Term logicalSour
         // - ContentType
         List<Term> contentType = Utils.getObjectsFromQuads(rmlStore.getQuads(logicalSource, new NamedNode(NAMESPACES.RML + "referenceFormulation"), null));
 
-        return new RDBAccess(dsn, database, username, password, query, (contentType.isEmpty()? "text/csv" : contentType.get(0).getValue()));
+
+        return new RDBAccess(dsn, database, username, password, query, (contentType.isEmpty() ? "text/csv" : contentType.get(0).getValue()));
     }
 
     /**
      * This method returns a SPARQLResultFormat based on the result formats and reference formulations.
      * @param resultFormats the result formats used to determine the SPARQLResultFormat.
-     * @param referenceFormulations the reference formulations used to to determine the SPARQLResultFormat.
+     * @param referenceFormulations the reference formulations used to determine the SPARQLResultFormat.
      * @return a SPARQLResultFormat.
      */
     private SPARQLResultFormat getSPARQLResultFormat(List<Term> resultFormats, List<Term> referenceFormulations) {

src/main/java/be/ugent/rml/access/RDBAccess.java

@@ -47,7 +47,7 @@ public class RDBAccess implements Access {
 
 
     /**
-     * This constructor takes as arguments the dsn, database, username, password, query, and content type.
+     * This constructor takes as arguments the dsn, database, username, password, query, content type
      *
      * @param dsn          the data source name.
      * @param databaseType the database type.
@@ -76,59 +76,44 @@ public InputStream getInputStream() throws IOException, SQLException, ClassNotFo
         // JDBC objects
         Connection connection = null;
         Statement statement = null;
-        String jdbcDriver = databaseType.getDriver();
-        String jdbcDSN = "jdbc:" + databaseType.getJDBCPrefix() + "//" + dsn;
-        InputStream inputStream = null;
+        InputStream inputStream;
 
         try {
-            // Register JDBC driver
-            Class.forName(jdbcDriver);
-
             // Open connection
-            String connectionString = jdbcDSN;
-            boolean alreadySomeQueryParametersPresent = false;
-
-            if (username != null && !username.equals("") && password != null && !password.equals("")) {
-                if (databaseType == DatabaseType.ORACLE) {
-                    connectionString = connectionString.replace(":@", ":" + username + "/" + password + "@");
-                } else if (!connectionString.contains("user=")) {
-                    connectionString += "?user=" + username + "&password=" + password;
-                    alreadySomeQueryParametersPresent = true;
-                }
-            }
+            boolean alreadySomeQueryParametersPresent = dsn.contains("?");
+
 
             if (databaseType == DatabaseType.MYSQL) {
+                StringBuilder parametersSB = new StringBuilder();
+
                 if (alreadySomeQueryParametersPresent) {
-                    connectionString += "&";
+                    parametersSB.append("&");
                 } else {
-                    connectionString += "?";
+                    parametersSB.append("?");
                 }
 
-                connectionString += "serverTimezone=UTC&useSSL=false";
+                parametersSB.append("serverTimezone=UTC&useSSL=false");
+
+                dsn += parametersSB;
             }
 
             if (databaseType == DatabaseType.SQL_SERVER) {
-                connectionString = connectionString.replaceAll("\\?|&", ";");
-
-                if (!connectionString.endsWith(";")) {
-                    connectionString += ";";
-                }
+                dsn = dsn.replaceAll("[?&]", ";");
             }
-            connection = DriverManager.getConnection(connectionString);
+
+            connection = DriverManager.getConnection(dsn, username, password);
 
             // Execute query
             statement = connection.createStatement();
+
             ResultSet rs = statement.executeQuery(query);
 
-            switch (contentType) {
-                case NAMESPACES.QL + "XPath" :
-                    inputStream = getXMLInputStream(rs);
-                    break;
-                default:
-                    inputStream = getCSVInputStream(rs);
+            if ((NAMESPACES.QL + "XPath").equals(contentType)) {
+                inputStream = getXMLInputStream(rs);
+            } else {
+                inputStream = getCSVInputStream(rs);
             }
 
-
             // Clean-up environment
             rs.close();
             statement.close();

src/test/java/be/ugent/rml/Arguments_Test_MySQL.java

@@ -1,49 +1,32 @@
 package be.ugent.rml;
 
 import be.ugent.rml.cli.Main;
-import ch.vorburger.exec.ManagedProcessException;
-import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 public class Arguments_Test_MySQL extends MySQLTestCore {
 
-    private static String CONNECTIONSTRING;
-
     @BeforeClass
-    public static void before() throws Exception {
-        int portNumber = Utils.getFreePortNumber();
-        CONNECTIONSTRING = getConnectionString(portNumber);
-        mysqlDB = setUpMySQLDBInstance(portNumber);
-    }
-
-    @AfterClass
-    public static void after() throws ManagedProcessException {
-        stopDBs();
+    public static void beforeClass() {
+        logger = LoggerFactory.getLogger(Arguments_Test_MySQL.class);
     }
 
     @Test
     public void executeR2RML() throws Exception {
-        String cwd = Utils.getFile( "argument/r2rml").getAbsolutePath();
+        String cwd = (new File("./src/test/resources/argument/r2rml")).getAbsolutePath();
         String mappingFilePath = (new File(cwd, "mapping.r2rml.ttl")).getAbsolutePath();
         String actualPath = (new File("./generated_output.nq")).getAbsolutePath();
-        String expectedPath = (new File( cwd, "output.nq")).getAbsolutePath();
-        String resourcePath = "argument/r2rml/resource.sql";
+        String expectedPath = (new File(cwd, "output.nq")).getAbsolutePath();
+        String resourcePath = "src/test/resources/argument/r2rml/resource.sql";
 
-        // Get SQL resource
-        try {
-            mysqlDB.source(resourcePath);
-        } catch (ManagedProcessException e) {
-            e.printStackTrace();
-            fail();
-        }
+        prepareDatabase(resourcePath, "root", "");
 
-        Main.main(("-m " + mappingFilePath + " -o " + actualPath + " --r2rml-jdbcDSN " + CONNECTIONSTRING + " --r2rml-username root -v").split(" "), cwd);
+        Main.main(("-m " + mappingFilePath + " -o " + actualPath + " --r2rml-jdbcDSN " + dbURL + " --r2rml-username root -v").split(" "), cwd);
         compareFiles(
                 expectedPath,
                 actualPath,