CI: automated test reports and Maven Central deploy

RMLio · Jan 13, 2022 · 441b899 · 441b899
1 parent 5516c2a
commit 441b899
Show file tree

Hide file tree

Showing 4 changed files with 163 additions and 6 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,13 +1,14 @@
 stages:
   - unittests
   - mirror
+  - release
   - deploy
 
 variables:
   # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log.
   # `showDateTime` will show the passed time in milliseconds.
   MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true"
-  MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
+  MAVEN_CLI_OPTS: "-s $CI_PROJECT_DIR/.m2/settings.xml --batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true"
   # Postgres
   POSTGRES_DB: postgres
   POSTGRES_USER: postgres
@@ -36,12 +37,12 @@ default:
 General:
   stage: unittests
   image: maven:3.5.0-jdk-8
+  retry: 2
   services:
     - postgres:10.4
     - name: mcr.microsoft.com/mssql/server:latest
       alias: sqlserver
   script:
-    - echo "SKIP"
     - 'mvn $MAVEN_CLI_OPTS -Dtest=!Mapper_OracleDB_Test test'
   except:
     - master
@@ -111,10 +112,10 @@ Create Release:
   image:
     entrypoint: ['']
     name: alpine/git:${GIT_VERSION}
-  stage: deploy
+  stage: release
   before_script:
     # Clone the repository via HTTPS inside a new directory
-    - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}"
+    - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}"
 
     # Set the displayed user with the commits that are about to be made
     - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}"
@@ -200,7 +201,12 @@ Github Release:
   stage: deploy
   image: maven:3.5.0-jdk-8
   before_script:
-    - apk add github-cli
+    # Clone the repository via HTTPS inside a new directory
+    - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}"
+
+    # Set the displayed user with the commits that are about to be made
+    - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}"
+    - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}"
   script:
     # Build fat jar for Github Release
     - mvn install -DskipTests=true
@@ -212,6 +218,107 @@ Github Release:
   only:
     - tags
 
+# Create a Maven Central release on new tags
+# Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/
+# Define the following CI variables:
+# - MAVEN_REPO_USER: Maven user for deployment
+# - MAVEN_REPO_PASSWORD: Maven password for deployment
+# - GPG_SECRET_KEY: file with your secret key to sign things
+# You can get these credentials by creating an user access token on oss.sonatype.org > Profile > User Token
+Maven Central:
+  stage: deploy
+  image: maven:3.5.0-jdk-8
+  before_script:
+    - gpg --version
+    - gpg --import --batch --yes $GPG_SECRET_KEY
+  script:
+    - mvn $MAVEN_CLI_OPTS clean deploy -P release -DskipTests=true
+  only:
+    - tags
+
+# Generate R2RML test report and send MR to rml.io website
+# Push options: https://docs.gitlab.com/ee/user/project/push_options.html
+R2RML Test Report:
+  stage: deploy
+  image: docker:latest
+  services:
+    - docker:19.03.12-dind
+  before_script:
+    # Dependencies
+    - apk add git python3 python3-dev py3-pip postgresql-dev postgresql libpq gcc musl-dev docker docker-compose java-jre-headless maven java-jdk
+
+    # Clone the repository via HTTPS inside a new directory
+    - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_WEBSITE}@${CI_SERVER_HOST}/rml/doc/rmlio-website.git"
+    - git clone "https://github.com/DylanVanAssche/r2rml-test-cases-support" # TODO: change this when patch is merged
+
+    # Set the displayed user with the commits that are about to be made
+    - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}"
+    - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}"
+
+    # Show docker
+    - docker info
+  script:
+    # Variables
+    - RELEASE_TAG_NAME=$(git tag -l "v*" --sort=-creatordate | head -n1)
+    - RELEASE_TAG_NAME=$(echo "$RELEASE_TAG_NAME" | cut -c2-)
+    - TODAY=$(date +"%Y-%m-%d")
+
+    # Build RMLMapper jar
+    - mvn install -DskipTests=true
+
+    # Setup R2RML test cases configs
+    - cp data/r2rml-test-cases-config-postgresql.ini r2rml-test-cases-support/config-postgresql.ini
+    - cp data/r2rml-test-cases-config-mysql.ini r2rml-test-cases-support/config-mysql.ini
+    - cd r2rml-test-cases-support
+    - echo "Generating test report for release v${RELEASE_TAG_NAME} on ${TODAY}"
+    - sed -i "s/VERSION/${RELEASE_TAG_NAME}/" config-postgresql.ini
+    - sed -i "s/VERSION/${RELEASE_TAG_NAME}/" config-mysql.ini
+    - sed -i "s/DATE/${TODAY}/" config-postgresql.ini
+    - sed -i "s/DATE/${TODAY}/" config-mysql.ini
+
+    # Install R2RML test cases dependencies
+    - python3 -m pip install -r requirements.txt
+
+    # Execute R2RML test cases
+    - HOST=docker python3 test.py config-postgresql.ini
+    - HOST=$(getent hosts docker | cut -f1 -d " ") python3 test.py config-mysql.ini # MySQL wants an IP address
+
+    # Prepare MR for rml.io website
+    - cp results.ttl ../rmlio-website/tools/rmlmapper/r2rml-test-cases-results.ttl
+    - cd ..
+  after_script:
+    - RELEASE_TAG_NAME=$(git tag -l "v*" --sort=-creatordate | head -n1)
+    - RELEASE_TAG_NAME=$(echo "$RELEASE_TAG_NAME" | cut -c2-)
+
+    # Go to the new directory
+    - cd rmlio-website
+
+    # Add all generated files to Git
+    - git add .
+    - |-
+      # Check if we have modifications to commit
+      CHANGES=$(git status --porcelain | wc -l)
+      BRANCH="rmlmapper/${RELEASE_TAG_NAME}"
+
+      if [ "$CHANGES" -gt "0" ]; then
+        # Show the status of files that are about to be created, updated or deleted
+        git diff --cached --shortstat
+
+        # Create new branch
+        git checkout -b "${BRANCH}"
+
+        # Commit all changes
+        COMMIT_MESSAGE="tools/rmlmapper: update R2RML test report for RMLMapper ${RELEASE_TAG_NAME}"
+        echo "Commit message: ${COMMIT_MESSAGE}"
+        git commit -m "${COMMIT_MESSAGE}"
+
+        # Update the repository and make sure to skip the pipeline create for this commit
+        echo "Creating Merge Request of branch ${BRANCH}"
+        git push origin "${BRANCH}" -o merge_request.create -o merge_request.remove_source_branch -o merge_request.title="${COMMIT_MESSAGE}"
+      fi
+  only:
+    - tags
+
 ###############################
 #                             #
 #     Automated Mirroring     #
@@ -226,7 +333,6 @@ Github:
     entrypoint: ['']
     name: alpine/git:${GIT_VERSION}
   before_script:
-    - apk add git
     # Set the displayed user with the commits that are about to be made
     - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}"
     - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}"

diff --git a/.m2/settings.xml b/.m2/settings.xml
@@ -0,0 +1,15 @@
+<settings xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd"
+xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <servers>
+        <server>
+            <id>central</id>
+            <username>${env.MAVEN_REPO_USER}</username>
+            <password>${env.MAVEN_REPO_PASS}</password>
+        </server>
+        <server>
+            <id>ossrh</id>
+            <username>${env.MAVEN_REPO_USER}</username>
+            <password>${env.MAVEN_REPO_PASS}</password>
+        </server>
+    </servers>
+</settings>
diff --git a/data/r2rml-test-cases-config-mysql.ini b/data/r2rml-test-cases-config-mysql.ini
@@ -0,0 +1,18 @@
+[tester]
+tester_name: Dylan Van Assche
+tester_url: https://rml.io/
+tester_contact: dylan.vanassche@ugent.be
+
+[engine]
+test_date: DATE
+engine_version: VERSION
+engine_name: RMLMapper
+engine_created: 2018-04-18
+engine_url: https://github.com/rmlio/rmlmapper-java
+
+[properties]
+database_system: mysql
+tests: all
+output_results: ./output.nq
+output_format: nquads
+engine_command: java -jar ../target/rmlmapper-*-all.jar -d -s nquads -m ./r2rml.ttl -dsn jdbc:mysql://docker:3306/r2rml -u r2rml -p r2rml -o ./output.nq
diff --git a/data/r2rml-test-cases-config-postgresql.ini b/data/r2rml-test-cases-config-postgresql.ini
@@ -0,0 +1,18 @@
+[tester]
+tester_name: Dylan Van Assche
+tester_url: https://rml.io/
+tester_contact: dylan.vanassche@ugent.be
+
+[engine]
+test_date: DATE
+engine_version: VERSION
+engine_name: RMLMapper
+engine_created: 2018-04-18
+engine_url: https://github.com/rmlio/rmlmapper-java
+
+[properties]
+database_system: postgresql
+tests: all
+output_results: ./output.nq
+output_format: nquads
+engine_command: java -jar ../target/rmlmapper-*-all.jar -d -s nquads -m ./r2rml.ttl -dsn jdbc:postgresql://docker:5432/r2rml -u r2rml -p r2rml -o ./output.nq