-
Notifications
You must be signed in to change notification settings - Fork 27
156 lines (154 loc) · 6.64 KB
/
publish-contracts-and-assets.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: Gable Publish Contracts & Assets
run-name: ${{ github.actor }} - Gable Publish Contracts & Assets
# Run on push to main branch
on:
push:
branches:
- 'main'
- 'pyspark_cicd'
permissions:
# Required to checkout the contracts from the repo
contents: read
jobs:
validate-publish-contracts:
runs-on: ubuntu-latest
name: 'Validate & Publish Contracts'
needs: [register-data-assets]
steps:
- name: Check out repository code
uses: actions/checkout@v3
# By default, the Gable CLI will exit with a non-zero exit code if no contracts are found when calling the
# validate or publish command. For this tutorial, there won't be any contracts in the repo to start, so we
# need this extra step to check if we should skip validation
- shell: bash
run: |
set +e
sh -c "ls ./contracts/*.yaml"
if [ $? -eq 0 ]; then
echo "Found contract files, publishing contracts..."
else
echo "No contract files found, skipping contract validation..."
echo SKIP_VALIDATION=true >> $GITHUB_ENV
fi
- name: Validate Contracts
if: ${{ env.SKIP_VALIDATION != 'true' }}
uses: gabledata/cicd/github-actions/validate-contracts@latest
with:
# Provide API key and endpoint secrets
gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE}}
gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE}}
# List of paths to contract files that should be validated, with support for glob syntax.
# Can either be specified as a space separated list ('contract1.yml contract2.yml'), or
# a multiline string
contract-paths: |
./contracts/*.yaml
- name: Publish Contracts
if: ${{ env.SKIP_VALIDATION != 'true' }}
uses: gabledata/cicd/github-actions/publish-contracts@latest
with:
# Provide API key and endpoint secrets
gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE}}
gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE}}
# List of paths to contract files that should be published, with support for glob syntax.
# Can either be specified as a space separated list ('contract1.yml contract2.yml'), or
# a multiline string
contract-paths: |
./contracts/*.yaml
register-data-assets:
runs-on: ubuntu-latest
name: 'Register Data Assets'
services:
postgres:
image: postgres:14
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: tutorial
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: Check out repository code
uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.10.11
- name: Install Poetry
run: curl -sSL https://install.python-poetry.org | python3 -
shell: bash
- name: Install dependencies
shell: bash
run: |
# Install root level dependencies
poetry install
# Create a virtual environment for the PySpark project, install dependencies
python3 -m venv "pyspark/.venv"
pyspark/.venv/bin/pip install -r pyspark/requirements.txt
ls -la pyspark/.venv/bin
- name: Run migrations from db_migrations directory
working-directory: ./db_migrations
run: poetry run alembic upgrade head
shell: bash
- name: Register PySpark Data Assets
uses: gabledata/cicd/github-actions/register-data-assets@python_path
with:
# Provide API key and endpoint secrets
gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE}}
gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE}}
python-path: pyspark/.venv/bin/python
# List of paths to Protobuf files that should be checked with support for glob syntax.
# Can either be specified as a space separated list ('event1.proto event2.proto'), or
# a multiline string
data-asset-options: |
--source-type pyspark \
--project-root pyspark \
--csv-schema-file pyspark/schemas.csv \
--spark-job-entrypoint 'job.py --final_output_table pnw_bookings_30_days' --trace
- name: Register Protobuf Data Assets
uses: gabledata/cicd/github-actions/register-data-assets@latest
with:
# Provide API key and endpoint secrets
gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE}}
gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE}}
# List of paths to Protobuf files that should be checked with support for glob syntax.
# Can either be specified as a space separated list ('event1.proto event2.proto'), or
# a multiline string
data-asset-options: |
--source-type protobuf \
--files ./event_schemas/*.proto
- name: Register Avro Data Assets
uses: gabledata/cicd/github-actions/register-data-assets@latest
with:
# Provide API key and endpoint secrets
gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE}}
gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE}}
# List of paths to Avro files that should be checked with support for glob syntax.
# Can either be specified as a space separated list ('event1.proto event2.proto'), or
# a multiline string
data-asset-options: |
--source-type avro \
--files ./event_schemas/*.avsc
- name: Register Postgres Data Assets
uses: gabledata/cicd/github-actions/register-data-assets@latest
with:
# Provide API key and endpoint secrets
gable-api-key: ${{secrets.GABLE_API_KEY_UNSTABLE}}
gable-api-endpoint: ${{secrets.GABLE_API_ENDPOINT_UNSTABLE}}
# List of paths to Avro files that should be checked with support for glob syntax.
# Can either be specified as a space separated list ('event1.proto event2.proto'), or
# a multiline string
data-asset-options: |
--source-type postgres \
--host prod.store.com \
--port 5432 \
--db tutorial \
--schema public \
--proxy-host 0.0.0.0 \
--proxy-port 5432 \
--proxy-user postgres \
--proxy-password postgres