test

.github/workflows/main.yaml

@@ -0,0 +1,82 @@
+name: Main
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch: { }
+
+jobs:
+  git-tag-and-push:
+    runs-on:
+      - ubuntu-latest
+    timeout-minutes: 20
+    outputs:
+      version: "1.0.0"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Configure git
+        run: |
+          git config user.name teemu61
+          git config user.email teemu.t.jappinen@gmail.com
+
+      - name: Add git tag, push the tag and set the version to outputs
+        id: versioning
+        run: |
+          git fetch --tags
+          pip3 install --user ruamel.yaml
+          VERSION=$(python3 -c "import dataeng; print(dataeng.__version__)")
+          git tag -a $(VERSION) -m v$(VERSION) && git push origin $(VERSION)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+  docker-build:
+    env:
+      DOCKERFILE_W_PATH: ./Dockerfile
+      VERSION: ${{ needs.git-tag-and-push.outputs.version }}
+    runs-on:
+      - ubuntu-latest
+    timeout-minutes: 20
+    needs:
+      - git-tag-and-push
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      - uses: 'google-github-actions/auth@v2'
+        id: gcp_auth
+        with:
+          credentials_json: ${{ secrets.GCP_REPO_JSON_KEY }}
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v2'
+
+      - name: Register gcloud as a Docker credential helper
+        run: gcloud auth configure-docker europe-north1-docker.pkg.dev
+
+      - name: Define image meta
+        id: docker_meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            europe-north1-docker.pkg.dev/vivid-env-450617-g5/teemu-images/dataeng
+          flavor: |
+            latest=false
+          tags: |
+            type=raw,value=${{ env.VERSION }}  
+
+      - name: Build release image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          no-cache: false
+          push: true
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          provenance: false
+          secret-files: gcp_secret=${{ steps.gcp_auth.outputs.credentials_file_path }}

dataeng/Makefile

@@ -1,90 +1,25 @@
-PACKAGE := cmapi
+PACKAGE := dataeng
 
-CONTAINER := vnoc/automate-$(PACKAGE)
-REGISTRY := valkama.saunalahti.fi
+CONTAINER := dataeng
+REGISTRY := europe-north1-docker.pkg.dev/vivid-env-450617-g5/teemu-images/
 TAG ?= latest
 
 # Get version from package __init__ file:
 VERSION := $(shell sed -ne "/__version__/s/^.*=.'\(.*\)'$$/\1/p" $(PACKAGE)/__init__.py)
 
 # ---- Generic rules to build containers and publish them
-.PHONY: all
-all: build docker
+# .PHONY: all
+# all: build docker
 
-.PHONY: build
-build: run-tests
+# .PHONY: build
+# build: run-tests
 
-.PHONY: docker
-docker: run-tests version.json
-	docker buildx build --secret id=gcp_secret,src=$(HOME)/.config/gcloud/application_default_credentials.json --platform linux/amd64 -t  $(CONTAINER) --pull=true .
-	rm -rf version.json
+# .PHONY: docker
+# docker: run-tests version.json
+# 	docker buildx build --secret id=gcp_secret,src=$(HOME)/.config/gcloud/application_default_credentials.json --platform linux/amd64 -t  $(CONTAINER) --pull=true .
+# 	rm -rf version.json
 
-version.json:
-	@echo "{\"container\": \"$(CONTAINER)\", \
-			\"registry\": \"$(REGISTRY)\", \
-			\"repository\": \"$(shell git config remote.origin.url)\", \
-			\"commit\": \"$(shell git rev-parse HEAD)\", \
-			\"date\": \"$(shell date +"%Y%m%dT%H%M")\", \
-			\"by\": {\
-				\"name\": \"$(shell git config user.name)\", \
-				\"email\": \"$(shell git config user.email)\"\
-			}}" | python3 -m json.tool | tee $@
+# .PHONY: git-tag-and-push
+# git-tag-and-push:
+# 	git tag -a $(VERSION) -m v$(VERSION) && git push origin $(VERSION)
 
-.PHONY: clean
-clean:
-	git clean -Xdf -e '!*.iml'
-
-.PHONY: re
-re: clean all
-
-# project specific makefile rules
-.built:
-	pipenv sync --dev
-	touch .built
-
-.PHONY: run-tests
-run-tests: .built
-	pipenv run python3 -X dev -m pytest -rs --junitxml=pytests.xml --cov-branch --cov-report=term --cov-report=xml:coverage.xml --cov-report=html --cov=$(PACKAGE) tests
-
-.PHONY: pycodestyle
-pycodestyle: .built
-	pipenv run pycodestyle --statistics --$(shell grep 'max-line-length=' .pylintrc) $(PACKAGE)
-
-.PHONY: vulture
-vulture: .built
-	pipenv run vulture --min-confidence=90 $(PACKAGE)
-
-.PHONY: pylintchecker
-pylintchecker: .built
-	pipenv run pylint --rcfile=.pylintrc --fail-under=9.7 $(PACKAGE)
-
-.PHONY: mypy
-mypy: .built
-	pipenv run mypy --ignore-missing-imports -i $(PACKAGE);
-
-.PHONY: lint
-lint: pylintchecker vulture pycodestyle mypy
-
-.PHONY: deps-check
-deps-check:
-	mkdir -p reports && dependency-check --cveValidForHours 24 --out reports --format ALL \
-		--enableExperimental --disableAssembly --disableBundleAudit --disableNodeJS --disableOssIndex \
-		--scan $(PACKAGE) --project $(PACKAGE)
-
-.PHONY: beautify
-beautify: .built
-	pipenv run yapf --in-place --parallel -vv --recursive --style='{based_on_style: google, column_limit:120, allow_split_before_dict_value: false}' $(PACKAGE) tests
-
-.PHONY: git-tag-test
-git-tag-test:
-	git tag $(VERSION)
-
-.PHONY: git-tag-and-push
-git-tag-and-push:
-	git tag -a $(VERSION) -m v$(VERSION) && git push origin $(VERSION)
-
-.PHONY: valkama
-valkama: docker
-	@echo "Tagging '$(CONTAINER)' as '$(TAG)' and pushing into $(REGISTRY)"
-	docker tag $(CONTAINER) $(REGISTRY)/$(CONTAINER):$(TAG)
-	docker push $(REGISTRY)/$(CONTAINER):$(TAG) || echo "hint: use 'docker login $(REGISTRY)' to authenticate"