-
Notifications
You must be signed in to change notification settings - Fork 5
174 lines (174 loc) · 6.84 KB
/
int-and-connected-test-run.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
name: Unit, Smoke, Connected and Integration tests
on:
workflow_dispatch: {}
pull_request:
schedule:
- cron: '0 4 * * *' # run at 4 AM UTC, 12PM EST.
# do not allow concurrent runs of this workflow on the same branch
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test_unit:
name: Unit tests
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: Run unit tests
env:
# required for unit tests
GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json
# required for sonarqube reports
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# extract service account credentials
base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS}
# assemble code, run unit tests, and generate scan
./gradlew --scan assemble check jacocoTestReport sonar
test_connected:
name: Connected tests
runs-on: ubuntu-latest
timeout-minutes: 180
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: Run connected tests
env:
# required for azure tests
AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571
AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408
AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }}
# required for synapse tests
AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }}
AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }}
AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net
# required for connected tests
GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json
# required for testAzureBillingProfile
# uses an azure marketplace app with this hardcoded deployment email
JADE_USER_EMAIL: connected-tdr-user@notarealemail.org
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# extract service account credentials
base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS}
# run connected tests
./gradlew --scan --warn testConnected
test_integration:
name: Integration tests
runs-on: ubuntu-latest
timeout-minutes: 300
services:
postgres:
image: postgres:16.4
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'gradle'
- name: Run integration tests
env:
# required for azure tests
AZURE_CREDENTIALS_APPLICATIONID: 0e29ec36-04e8-44d5-ae7c-50dc15135571
AZURE_CREDENTIALS_HOMETENANTID: fad90753-2022-4456-9b0a-c7e5b934e408
AZURE_CREDENTIALS_SECRET: ${{ secrets.AZURE_CREDENTIALS_SECRET }}
AZURE_SYNAPSE_SQLADMINUSER: ${{ secrets.AZURE_SYNAPSE_SQLADMINUSER }}
AZURE_SYNAPSE_SQLADMINPASSWORD: ${{ secrets.AZURE_SYNAPSE_SQLADMINPASSWORD }}
AZURE_SYNAPSE_WORKSPACENAME: tdr-snps-int-east-us-ondemand.sql.azuresynapse.net
# required for integration tests
GOOGLE_APPLICATION_CREDENTIALS: jade-dev-account.json
GOOGLE_SA_CERT: jade-dev-account.pem
IT_JADE_API_URL: http://localhost:8080
# postgres connection details
PGHOST: 127.0.0.1
PGPASSWORD: postgres
# required for integration tests
RBS_INSTANCEURL: https://buffer.tools.integ.envs.broadinstitute.org
RBS_POOLID: datarepo_v1
# output plain logs instead of json
TDR_LOG_APPENDER: 'Console-Standard'
run: |
# wait for postgres to be ready
pg_isready -h ${PGHOST} -t 30
# create the datarepo and stairway databases
psql -U postgres -f ./scripts/init-db/postgres-init.sql
# extract service account credentials
base64 --decode <<< ${{ secrets.SA_B64_CREDENTIALS }} > ${GOOGLE_APPLICATION_CREDENTIALS}
jq -r .private_key ${GOOGLE_APPLICATION_CREDENTIALS} > ${GOOGLE_SA_CERT}
chmod 644 ${GOOGLE_SA_CERT}
# build code and run local api
./gradlew bootJar
export DATA_REPO_JAR=$(find . -type f -name jade-data-repo-*-SNAPSHOT.jar)
java -jar ${DATA_REPO_JAR} > local-api-output.log &
# wait until api is ready
timeout 30 bash -c 'until curl -s ${IT_JADE_API_URL}/status; do sleep 1; done'
# run integration tests
./gradlew --scan --warn testIntegration
- name: Upload API logs
if: always()
uses: actions/upload-artifact@v4
with:
name: integration-test-api-logs
path: local-api-output.log
retention-days: 7
git_hash:
name: Extract git hash
runs-on: ubuntu-latest
outputs:
version: ${{ steps.config.outputs.GIT_HASH }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Get the latest git hash
id: config
run: |
GIT_HASH=$(git rev-parse --short HEAD)
echo "GIT_HASH=${GIT_HASH}" >> $GITHUB_OUTPUT
echo "Latest git hash in branch is ${GIT_HASH}"
report-workflow:
if: github.ref_name == github.event.repository.default_branch
uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main
# dependency is not required but makes the action run page more readable
needs: git_hash
with:
relates-to-chart-releases: 'datarepo-dev'
notify-slack-channels-upon-workflow-failure: ${{ vars.SLACK_NOTIFICATION_CHANNELS }}
notify-slack-channels-upon-workflow-retry: ${{ vars.SLACK_NOTIFICATION_CHANNELS }}
permissions:
id-token: write