-
Notifications
You must be signed in to change notification settings - Fork 5
177 lines (177 loc) · 7.37 KB
/
int-and-connected-test-run.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
name: Unit, Smoke, Connected and Integration tests
on:
workflow_dispatch: {}
pull_request:
schedule:
- cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST.
# do not allow concurrent runs of this workflow on the same branch
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test_unit:
name: Unit tests
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: Run unit tests
env:
GOOGLE_APPLICATION_CREDENTIALS: 'jade-dev-account.json'
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# extract service account credentials
base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS}
# assemble code, run unit tests, and generate scan
./gradlew assemble
./gradlew check --scan jacocoTestReport sonar
test_connected:
name: Connected tests
runs-on: ubuntu-latest
timeout-minutes: 180
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: Run connected tests
env:
# required for azure tests
AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571
AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408
AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }}
# required for synapse tests
AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }}
AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }}
AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net
# required for connected tests
GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json
# required data project for snapshotTest
GOOGLE_CLOUD_DATA_PROJECT: broad-jade-integration-data
# required for testAzureBillingProfile
# uses an azure marketplace app with this hardcoded deployment email
JADE_USER_EMAIL: connected-tdr-user@notarealemail.org
# required for rbs tests
RBS_CLIENT_CREDENTIAL_FILE_PATH: rbs-tools-sa.json
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# extract service account credentials
base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS}
base64 --decode <<< ${{ secrets.B64_RBS_APPLICATION_CREDENTIALS }} > ${RBS_CLIENT_CREDENTIAL_FILE_PATH}
# assemble code and run connected tests
./gradlew assemble
./gradlew testConnected --scan
test_integration:
name: Integration tests
runs-on: ubuntu-latest
timeout-minutes: 300
services:
postgres:
image: postgres:11
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
env:
# This must be defined for the bash redirection
GOOGLE_APPLICATION_CREDENTIALS: 'jade-dev-account.json'
# This must be defined for the bash redirection
GOOGLE_SA_CERT: 'jade-dev-account.pem'
# Required for locking and deployment to integration namespace
K8_CLUSTER: 'integration-master'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: Whitelist Runner IP
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
with:
actions_subcommand: 'gcp_whitelist'
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Check for an available namespace to deploy API to and set state lock
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
with:
actions_subcommand: 'k8_checknamespace'
k8_namespaces: 'integration-1,integration-2,integration-3,integration-6'
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Build docker container via Gradle
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
env:
# Unset the GitHub Action default JAVA_HOME to build with JDK 17
JAVA_HOME:
with:
actions_subcommand: 'gradlebuild' # creates gcr build with git_hash tag
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Deploy to cluster with Helm
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
with:
actions_subcommand: 'helmdeploy'
helm_create_secret_manager_secret_version: 0.0.8
helm_datarepo_api_chart_version: 0.0.744
helm_datarepo_ui_chart_version: 0.0.362
helm_gcloud_sqlproxy_chart_version: 0.19.13
helm_oidc_proxy_chart_version: 0.0.44
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Fetch gitHash for deployed integration version
id: configuration
run: |
git_hash=$(git rev-parse --short HEAD)
echo "git_hash=${git_hash}" >> $GITHUB_OUTPUT
echo "Latest git hash for this branch: $git_hash"
- name: Wait for deployment to come back online
uses: broadinstitute/datarepo-actions/actions/wait-for-deployment@0.74.0
timeout-minutes: 20
env:
DESIRED_GITHASH: ${{ steps.configuration.outputs.git_hash }}
DEPLOYMENT_TYPE: 'api'
- name: Run test runner smoke tests via Gradle
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
with:
actions_subcommand: 'gradletestrunnersmoketest'
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Run integration tests via Gradle
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
env:
AZURE_CREDENTIALS_APPLICATIONID: ${{ env.AZURE_CREDENTIALS_APPLICATIONID }}
AZURE_CREDENTIALS_HOMETENANTID: ${{ env.AZURE_CREDENTIALS_HOMETENANTID }}
TDR_LOG_APPENDER: 'Console-Standard'
with:
actions_subcommand: 'gradleinttest'
pgport: ${{ job.services.postgres.ports[5432] }}
test_to_run: 'testIntegration'
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Clean state lock from used Namespace on API deploy
if: always()
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
with:
actions_subcommand: 'k8_checknamespace_clean'
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}
- name: Clean whitelisted Runner IP
if: always()
uses: broadinstitute/datarepo-actions/actions/main@0.74.0
with:
actions_subcommand: 'gcp_whitelist_clean'
sa_b64_credentials: ${{ secrets.SA_B64_CREDENTIALS }}