diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14dd6588..8c047104 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,14 @@ stages: - unittests - mirror + - release - deploy variables: # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. # `showDateTime` will show the passed time in milliseconds. MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" - MAVEN_CLI_OPTS: "--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" + MAVEN_CLI_OPTS: "-s $CI_PROJECT_DIR/.m2/settings.xml --batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true" # Postgres POSTGRES_DB: postgres POSTGRES_USER: postgres @@ -36,12 +37,12 @@ default: General: stage: unittests image: maven:3.5.0-jdk-8 + retry: 2 services: - postgres:10.4 - name: mcr.microsoft.com/mssql/server:latest alias: sqlserver script: - - echo "SKIP" - 'mvn $MAVEN_CLI_OPTS -Dtest=!Mapper_OracleDB_Test test' except: - master @@ -111,10 +112,10 @@ Create Release: image: entrypoint: [''] name: alpine/git:${GIT_VERSION} - stage: deploy + stage: release before_script: # Clone the repository via HTTPS inside a new directory - - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}" + - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}" # Set the displayed user with the commits that are about to be made - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" @@ -200,7 +201,12 @@ Github Release: stage: deploy image: maven:3.5.0-jdk-8 before_script: - - apk add github-cli + # Clone the repository via HTTPS inside a new directory + - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_RMLMAPPER}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git" "${CI_COMMIT_SHA}" + + # Set the displayed user with the commits that are about to be made + - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" + - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" script: # Build fat jar for Github Release - mvn install -DskipTests=true @@ -212,6 +218,107 @@ Github Release: only: - tags +# Create a Maven Central release on new tags +# Thanks to https://1337codersblog.wordpress.com/2019/09/18/automatical-artifact-deployment-to-maven-central-using-gitlab-ci-part-1/ +# Define the following CI variables: +# - MAVEN_REPO_USER: Maven user for deployment +# - MAVEN_REPO_PASSWORD: Maven password for deployment +# - GPG_SECRET_KEY: file with your secret key to sign things +# You can get these credentials by creating an user access token on oss.sonatype.org > Profile > User Token +Maven Central: + stage: deploy + image: maven:3.5.0-jdk-8 + before_script: + - gpg --version + - gpg --import --batch --yes $GPG_SECRET_KEY + script: + - mvn $MAVEN_CLI_OPTS clean deploy -P release -DskipTests=true + only: + - tags + +# Generate R2RML test report and send MR to rml.io website +# Push options: https://docs.gitlab.com/ee/user/project/push_options.html +R2RML Test Report: + stage: deploy + image: docker:latest + services: + - docker:19.03.12-dind + before_script: + # Dependencies + - apk add git python3 python3-dev py3-pip postgresql-dev postgresql libpq gcc musl-dev docker docker-compose java-jre-headless maven java-jdk + + # Clone the repository via HTTPS inside a new directory + - git clone "https://${GITLAB_USERNAME}:${GITLAB_TOKEN_WEBSITE}@${CI_SERVER_HOST}/rml/doc/rmlio-website.git" + - git clone "https://github.com/DylanVanAssche/r2rml-test-cases-support" # TODO: change this when patch is merged + + # Set the displayed user with the commits that are about to be made + - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" + - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" + + # Show docker + - docker info + script: + # Variables + - RELEASE_TAG_NAME=$(git tag -l "v*" --sort=-creatordate | head -n1) + - RELEASE_TAG_NAME=$(echo "$RELEASE_TAG_NAME" | cut -c2-) + - TODAY=$(date +"%Y-%m-%d") + + # Build RMLMapper jar + - mvn install -DskipTests=true + + # Setup R2RML test cases configs + - cp data/r2rml-test-cases-config-postgresql.ini r2rml-test-cases-support/config-postgresql.ini + - cp data/r2rml-test-cases-config-mysql.ini r2rml-test-cases-support/config-mysql.ini + - cd r2rml-test-cases-support + - echo "Generating test report for release v${RELEASE_TAG_NAME} on ${TODAY}" + - sed -i "s/VERSION/${RELEASE_TAG_NAME}/" config-postgresql.ini + - sed -i "s/VERSION/${RELEASE_TAG_NAME}/" config-mysql.ini + - sed -i "s/DATE/${TODAY}/" config-postgresql.ini + - sed -i "s/DATE/${TODAY}/" config-mysql.ini + + # Install R2RML test cases dependencies + - python3 -m pip install -r requirements.txt + + # Execute R2RML test cases + - HOST=docker python3 test.py config-postgresql.ini + - HOST=$(getent hosts docker | cut -f1 -d " ") python3 test.py config-mysql.ini # MySQL wants an IP address + + # Prepare MR for rml.io website + - cp results.ttl ../rmlio-website/tools/rmlmapper/r2rml-test-cases-results.ttl + - cd .. + after_script: + - RELEASE_TAG_NAME=$(git tag -l "v*" --sort=-creatordate | head -n1) + - RELEASE_TAG_NAME=$(echo "$RELEASE_TAG_NAME" | cut -c2-) + + # Go to the new directory + - cd rmlio-website + + # Add all generated files to Git + - git add . + - |- + # Check if we have modifications to commit + CHANGES=$(git status --porcelain | wc -l) + BRANCH="rmlmapper/${RELEASE_TAG_NAME}" + + if [ "$CHANGES" -gt "0" ]; then + # Show the status of files that are about to be created, updated or deleted + git diff --cached --shortstat + + # Create new branch + git checkout -b "${BRANCH}" + + # Commit all changes + COMMIT_MESSAGE="tools/rmlmapper: update R2RML test report for RMLMapper ${RELEASE_TAG_NAME}" + echo "Commit message: ${COMMIT_MESSAGE}" + git commit -m "${COMMIT_MESSAGE}" + + # Update the repository and make sure to skip the pipeline create for this commit + echo "Creating Merge Request of branch ${BRANCH}" + git push origin "${BRANCH}" -o merge_request.create -o merge_request.remove_source_branch -o merge_request.title="${COMMIT_MESSAGE}" + fi + only: + - tags + ############################### # # # Automated Mirroring # @@ -226,7 +333,6 @@ Github: entrypoint: [''] name: alpine/git:${GIT_VERSION} before_script: - - apk add git # Set the displayed user with the commits that are about to be made - git config --global user.email "${GIT_USER_EMAIL:-$GITLAB_USER_EMAIL}" - git config --global user.name "${GIT_USER_NAME:-$GITLAB_USER_NAME}" diff --git a/.m2/settings.xml b/.m2/settings.xml new file mode 100644 index 00000000..f01b50ab --- /dev/null +++ b/.m2/settings.xml @@ -0,0 +1,15 @@ + + + + central + ${env.MAVEN_REPO_USER} + ${env.MAVEN_REPO_PASS} + + + ossrh + ${env.MAVEN_REPO_USER} + ${env.MAVEN_REPO_PASS} + + + diff --git a/data/r2rml-test-cases-config-mysql.ini b/data/r2rml-test-cases-config-mysql.ini new file mode 100644 index 00000000..10325c35 --- /dev/null +++ b/data/r2rml-test-cases-config-mysql.ini @@ -0,0 +1,18 @@ +[tester] +tester_name: Dylan Van Assche +tester_url: https://rml.io/ +tester_contact: dylan.vanassche@ugent.be + +[engine] +test_date: DATE +engine_version: VERSION +engine_name: RMLMapper +engine_created: 2018-04-18 +engine_url: https://github.com/rmlio/rmlmapper-java + +[properties] +database_system: mysql +tests: all +output_results: ./output.nq +output_format: nquads +engine_command: java -jar ../target/rmlmapper-*-all.jar -d -s nquads -m ./r2rml.ttl -dsn jdbc:mysql://docker:3306/r2rml -u r2rml -p r2rml -o ./output.nq diff --git a/data/r2rml-test-cases-config-postgresql.ini b/data/r2rml-test-cases-config-postgresql.ini new file mode 100644 index 00000000..15a50075 --- /dev/null +++ b/data/r2rml-test-cases-config-postgresql.ini @@ -0,0 +1,18 @@ +[tester] +tester_name: Dylan Van Assche +tester_url: https://rml.io/ +tester_contact: dylan.vanassche@ugent.be + +[engine] +test_date: DATE +engine_version: VERSION +engine_name: RMLMapper +engine_created: 2018-04-18 +engine_url: https://github.com/rmlio/rmlmapper-java + +[properties] +database_system: postgresql +tests: all +output_results: ./output.nq +output_format: nquads +engine_command: java -jar ../target/rmlmapper-*-all.jar -d -s nquads -m ./r2rml.ttl -dsn jdbc:postgresql://docker:5432/r2rml -u r2rml -p r2rml -o ./output.nq