Merge branch 'branch-0.2' into test-pr

.github/workflows/auto-merge.yml

@@ -0,0 +1,41 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to keep BASE branch up-to-date from HEAD branch
+name: auto-merge HEAD to BASE
+
+on:
+  pull_request_target:
+    branches:
+    - branch-0.2
+    types: [closed]
+
+jobs:
+  auto-merge:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: branch-0.2 # force to fetch from latest upstream instead of PR ref
+
+      - name: auto-merge job
+        uses: ./.github/workflows/auto-merge
+        env:
+          OWNER: NVIDIA
+          REPO_NAME: spark-rapids
+          HEAD: branch-0.2
+          BASE: branch-0.3
+          AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR

.github/workflows/auto-merge/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:alpine
+
+WORKDIR /
+COPY automerge .
+RUN pip install requests && chmod +x /automerge
+
+# require envs: OWNER,REPO_NAME,HEAD,BASE,GITHUB_TOKEN
+ENTRYPOINT ["/automerge"]

.github/workflows/auto-merge/action.yml

@@ -0,0 +1,20 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'auto-merge action'
+description: 'auto-merge HEAD to BASE'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+

.github/workflows/auto-merge/automerge

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A auto-merge tool
+
+Create a PR to merge HEAD to BASE branch.
+NOTE:
+    The generated PR should be automatically merged if no conflict. Otherwise, manual operation will be required.
+"""
+
+import os
+import sys
+import time
+
+import requests
+
+# ENV
+OWNER = os.environ.get('OWNER')
+assert OWNER, 'env OWNER should not be empty'
+REPO_NAME = os.environ.get('REPO_NAME')
+assert REPO_NAME, 'env REPO_NAME should not be empty'
+HEAD = os.environ.get('HEAD')
+assert HEAD, 'env HEAD should not be empty'
+BASE = os.environ.get('BASE')
+assert BASE, 'env BASE should not be empty'
+AUTOMERGE_TOKEN = os.environ.get('AUTOMERGE_TOKEN')
+assert AUTOMERGE_TOKEN, 'env AUTOMERGE_TOKEN should not be empty'
+# static
+API_URL = 'https://api.github.com'
+AUTH_HEADERS = {
+    'Authorization': 'token ' + AUTOMERGE_TOKEN
+}
+
+
+def create():
+    url = f'{API_URL}/repos/{OWNER}/{REPO_NAME}/pulls'
+    params = {
+        'title': f'[auto-merge] {HEAD} to {BASE} [skip ci] [bot]',
+        'head': HEAD,
+        'base': BASE,
+        'body': f'auto-merge triggered by github actions on `{HEAD}` to create a PR keeping `{BASE}` up-to-date. If '
+                'this PR is unable to be merged due to conflicts, it will remain open until manually fix.',
+        'maintainer_can_modify': True
+    }
+    r = requests.post(url, headers=AUTH_HEADERS, json=params)
+    if r.status_code == 201:
+        print('SUCCESS - create PR')
+        pull = r.json()
+        number = str(pull['number'])
+        sha = str(pull['head']['sha'])
+        return number, sha, False
+    if r.status_code == 422:  # early-terminate if no commits between HEAD and BASE
+        print('SUCCESS - No commits')
+        print(r.json())
+        return '', '', True
+    # FAILURE
+    print('FAILURE - create PR')
+    print(f'status code: {r.status_code}')
+    print(r.json())
+    sys.exit(1)
+
+
+def auto_merge(number, sha):
+    url = f'{API_URL}/repos/{OWNER}/{REPO_NAME}/pulls/{number}/merge'
+    params = {
+        'sha': sha,
+        'merge_method': 'merge'
+    }
+    r = requests.put(url, headers=AUTH_HEADERS, json=params)
+    if r.status_code == 200:
+        comment(number, '**SUCCESS** - auto-merge')
+        print('SUCCESS - auto-merge')
+        sys.exit(0)
+    else:
+        print('FAILURE - auto-merge')
+        comment(number=number, content=f"""**FAILURE** - Unable to auto-merge. Manual operation is required.
+```
+{r.json()}
+```
+""")
+        print(f'status code: {r.status_code}')
+        print(r.json())
+        sys.exit(1)
+
+
+def comment(number, content):
+    url = f'{API_URL}/repos/{OWNER}/{REPO_NAME}/issues/{number}/comments'
+    params = {
+        'body': content
+    }
+    r = requests.post(url, headers=AUTH_HEADERS, json=params)
+    if r.status_code == 201:
+        print('SUCCESS - create comment')
+    else:
+        print('FAILURE - create comment')
+        print(f'status code: {r.status_code}')
+        print(r.json())
+
+
+def main():
+    number, sha, term = create()
+    if term:
+        sys.exit(0)
+
+    time.sleep(10)  # sleep 10s, then try auto-merge
+    auto_merge(number, sha)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/blossom-ci.yml

@@ -0,0 +1,114 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to trigger blossom-CI on self-hosted runner
+name: Blossom-CI
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  authorization:
+    name: Authorization
+    # trigger on pre-defined text
+    if: github.event.comment.body == 'build'
+    runs-on: [self-hosted, linux, blossom]
+    steps:
+      - name: Check if comment is issued by authorized person
+        run: blossom-ci
+        env:
+          OPERATION: 'AUTH'
+          VERSION: '1'
+
+  vulnerability-scan-job:
+    name: Vulnerability scan job
+    needs: [authorization]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get pull request data
+        id: pull_request_data
+        uses: octokit/request-action@v2.x
+        with:
+          route: 'GET /repos/:repository/pulls/:issue_id'
+          repository: ${{ github.repository }}
+          issue_id: ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Set blackduck project version
+        id: blackduck-project-version
+        run: echo "${{ fromJson(steps.pull_request_data.outputs.data).head.ref }}-${{ github.run_id }}"
+
+      - name: Update status
+        uses: octokit/request-action@v2.x
+        with:
+          route: 'POST /repos/:repository/statuses/:sha'
+          repository: ${{ github.repository }}
+          sha: ${{ fromJson(steps.pull_request_data.outputs.data).head.sha }}
+          target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          description: "vulnerability scan running"
+          state: "pending"
+          context: "blossom-ci"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ fromJson(steps.pull_request_data.outputs.data).head.repo.full_name }}
+          ref: ${{ fromJson(steps.pull_request_data.outputs.data).head.ref }}
+          lfs: 'true'
+
+      - name: Setup java
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: Get project data (maven)
+        run: |
+          echo ::set-env name=projects::$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g')
+
+      - name: Add mask
+        run: echo "::add-mask::${{ secrets.BLACKDUCK_URL }}"
+
+      - name: Run synopsys detect
+        id: scan_result
+        uses: blackducksoftware/github-action@2.0.1
+        env:
+          PROJECTS: ${{ env.projects }}
+        with:
+          args: >
+            --blackduck.url="https://${{ secrets.BLACKDUCK_URL }}"
+            --blackduck.api.token="${{ secrets.BLACKDUCK_API_TOKEN }}"
+            --detect.maven.build.command="-pl='$PROJECTS -am'"
+            --detect.force.success=false
+            --detect.parallel.processors=0
+            --detect.project.name="${{ github.repository }}"
+            --detect.project.version.name="${{ github.run_id }}"
+
+  vulnerability-check-trigger:
+    name: Vulnerability check & start ci job
+    needs: [vulnerability-scan-job]
+    runs-on: [self-hosted, linux, blossom]
+    steps:
+      - name: Check for new issue in vulnerability scan & start ci job
+        run: blossom-ci
+        env:
+          OPERATION: 'SCAN-CHECK-CI-JOB-START'
+          VERSION: '1'
+          BLACKDUCK_TOKEN: "${{ secrets.BLACKDUCK_API_TOKEN }}"
+          BLACKDUCK_URL: "${{ secrets.BLACKDUCK_URL }}"
+          BLACKDUCK_PROJECT_VERSION: "${{ github.run_id }}"
+          CI_SERVER: ${{ secrets.CI_SERVER }}
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/signoff-check.yml

@@ -0,0 +1,34 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A workflow to check if PR got sign-off
+name: signoff check
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  signoff-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: sigoff-check job
+        uses: ./.github/workflows/signoff-check
+        env:
+          OWNER: NVIDIA
+          REPO_NAME: spark-rapids
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PULL_NUMBER: ${{ github.event.number }}

.github/workflows/signoff-check/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:alpine
+
+WORKDIR /
+COPY signoff-check .
+RUN pip install PyGithub && chmod +x /signoff-check
+
+# require envs: OWNER,REPO_NAME,GITHUB_TOKEN,PULL_NUMBER
+ENTRYPOINT ["/signoff-check"]