chore(CI): better build plugins

Signed-off-by: zhanghongtong <rory-z@outlook.com>
lf-edge · Oct 18, 2021 · ac6fff0 · ac6fff0
1 parent 07b7cc2
commit ac6fff0
Show file tree

Hide file tree

Showing 6 changed files with 93 additions and 145 deletions.
diff --git a/.ci/Dockerfile b/.ci/Dockerfile
diff --git a/.ci/Dockerfile-plugins b/.ci/Dockerfile-plugins
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
@@ -125,7 +125,11 @@ jobs:
         docker run -d --name ekuiper docker.io/lfedge/ekuiper
         ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ekuiper)
         sleep 5
-        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
+        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then
+            echo "docker image failed"
+            docker logs ekuiper
+            exit 1
+        fi
     - uses: docker/metadata-action@v3
       id: meta
       with:
@@ -175,6 +179,14 @@ jobs:
           - functions/image
           - functions/geohash
           - functions/labelImage
+        arch:
+          - linux/amd64
+          - linux/arm64
+        golang:
+          - 1.15.11
+        exclude:
+          - arch: linux/arm64
+            plugin: functions/labelImage
 
     steps:
     - uses: actions/checkout@v2
@@ -185,21 +197,26 @@ jobs:
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
-    - name: Build single platform image
+    - name: build debian plugins
+      run: |
+        docker run -i --rm \
+            -v $(pwd):/ekuiper \
+            --workdir /ekuiper \
+            --platform ${{ matrix.arch }} \
+            ghcr.io/${{ github.repository }}/base:${{ matrix.golang }}-debian \
+            bash -euc "make ${{ matrix.plugin }}"
+    - name: Build ekuiper image
       uses: docker/build-push-action@v2
+      if: matrix.arch == 'linux/amd64'
       with:
         context: .
         platforms: linux/amd64
         push: false
         load: true
         tags: docker.io/lfedge/ekuiper
         file: deploy/docker/Dockerfile
-    - name: build debian plugins
-      env:
-        PLUGIN: ${{ matrix.plugin }}
-      run: |
-          make ${PLUGIN}
     - name: test docker and plugins
+      if: matrix.arch == 'linux/amd64'
       env:
         PLUGIN: ${{ matrix.plugin }}
       run: |

diff --git a/.gitignore b/.gitignore
@@ -27,6 +27,7 @@ node_modules/
 
 _build
 _packages
+_plugins
 jmeter_logs
 kuiper_conf_util
 corss_build.tar
@@ -35,4 +36,4 @@ corss_build_for_rpm.tar
 *.swp
 *.history
 
-*/**/*.db
+*/**/*.db
diff --git a/Makefile b/Makefile
@@ -2,7 +2,9 @@ BUILD_PATH ?= _build
 PACKAGES_PATH ?= _packages
 
 VERSION := $(shell git describe --tags --always)
-PACKAGE_NAME := kuiper-$(VERSION)-$(shell go env GOOS)-$(shell go env GOARCH)
+ARCH := $(shell go env GOARCH)
+OS := $(shell go env GOOS)
+PACKAGE_NAME := kuiper-$(VERSION)-$(OS)-$(ARCH)
 
 TARGET ?= lfedge/ekuiper
 
@@ -65,10 +67,6 @@ real_pkg:
 	@mv $(BUILD_PATH)/$(PACKAGE_NAME).zip $(BUILD_PATH)/$(PACKAGE_NAME).tar.gz $(PACKAGES_PATH)
 	@echo "Package build success"
 
-.PHONY:cross_prepare
-cross_prepare:
-	@docker run --rm --privileged tonistiigi/binfmt --install all
-
 .PHONY: docker
 docker:
 	docker buildx build --no-cache --platform=linux/amd64 -t $(TARGET):$(VERSION) -f deploy/docker/Dockerfile . --load
@@ -82,64 +80,21 @@ PLUGINS := sinks/file \
 	sinks/redis \
 	sources/random \
 	sources/zmq \
+	sinks/tdengine \
 	functions/accumulateWordCount \
 	functions/countPlusOne \
 	functions/image \
 	functions/geohash \
-	functions/echo
-
-.PHONY: plugins sinks/tdengine $(PLUGINS)
-plugins: cross_prepare sinks/tdengine functions/labelImage $(PLUGINS)
-sinks/tdengine:
-	@docker buildx build --no-cache \
-    --platform=linux/amd64,linux/arm64 \
-    -t cross_build \
-    --build-arg VERSION=$(VERSION) \
-    --build-arg PLUGIN_TYPE=sinks \
-    --build-arg PLUGIN_NAME=tdengine \
-    --output type=tar,dest=/tmp/cross_build_plugins_sinks_tdengine.tar \
-    -f .ci/Dockerfile-plugins .
-
-	@mkdir -p _plugins/debian/sinks
-	@for arch in amd64 arm64; do \
-		tar -xvf /tmp/cross_build_plugins_sinks_tdengine.tar --wildcards "linux_$${arch}/go/kuiper/plugins/sinks/tdengine/tdengine_$$(echo $${arch%%_*}).zip" \
-		&& mv $$(ls linux_$${arch}/go/kuiper/plugins/sinks/tdengine/tdengine_$$(echo $${arch%%_*}).zip) _plugins/debian/sinks; \
-	done
-	@rm -f /tmp/cross_build_plugins_sinks_tdengine.tar
-
-functions/labelImage:
-	@docker buildx build --no-cache \
-    --platform=linux/amd64 \
-    -t cross_build \
-    --build-arg VERSION=$(VERSION) \
-    --build-arg PLUGIN_TYPE=functions \
-    --build-arg PLUGIN_NAME=labelImage \
-    --output type=tar,dest=/tmp/cross_build_plugins_functions_labelImage.tar \
-    -f .ci/Dockerfile-plugins .
-
-	@mkdir -p _plugins/debian/functions
-	@tar -xvf /tmp/cross_build_plugins_functions_labelImage.tar --wildcards "go/kuiper/plugins/functions/labelImage/labelImage_amd64.zip"
-	@mv $$(ls go/kuiper/plugins/functions/labelImage/labelImage_amd64.zip) _plugins/debian/functions
-	@rm -f /tmp/cross_build_plugins_functions_labelImage.tar
+	functions/echo \
+	functions/labelImage
+
+.PHONY: plugins $(PLUGINS)
+plugins: $(PLUGINS)
 
 $(PLUGINS): PLUGIN_TYPE = $(word 1, $(subst /, , $@))
 $(PLUGINS): PLUGIN_NAME = $(word 2, $(subst /, , $@))
 $(PLUGINS):
-	@docker buildx build --no-cache \
-    --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
-    -t cross_build \
-    --build-arg VERSION=$(VERSION) \
-    --build-arg PLUGIN_TYPE=$(PLUGIN_TYPE)\
-    --build-arg PLUGIN_NAME=$(PLUGIN_NAME)\
-    --output type=tar,dest=/tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar \
-    -f .ci/Dockerfile-plugins .
-
-	@mkdir -p _plugins/debian/$(PLUGIN_TYPE)
-	@for arch in amd64 arm64 arm_v7 386; do \
-		tar -xvf /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar --wildcards "linux_$${arch}/go/kuiper/plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip" \
-		&& mv $$(ls linux_$${arch}/go/kuiper/plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip) _plugins/debian/$(PLUGIN_TYPE); \
-	done
-	@rm -f /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar
+	@$(CURDIR)/build-plugins.sh $(PLUGIN_TYPE) $(PLUGIN_NAME)
 
 .PHONY: clean
 clean:

diff --git a/build-plugins.sh b/build-plugins.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+set -euo pipefail
+
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
+
+PLUGIN_TYPE=$1
+PLUGIN_NAME=$2
+VERSION=$(git describe --tags --always)
+
+pre(){
+    mkdir -p _plugins/debian/$PLUGIN_TYPE
+    if [  $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs") != 'null' ]; then
+        for lib in $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs[]"); do
+            go get $lib;
+        done
+    fi
+}
+
+post(){
+    if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then
+        cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml extensions/$PLUGIN_TYPE/$PLUGIN_NAME;
+    fi
+    cd extensions/$PLUGIN_TYPE/$PLUGIN_NAME
+    zip -r ${PLUGIN_NAME}_$(go env GOARCH).zip .
+    cd -
+    mv $(find extensions/$PLUGIN_TYPE/$PLUGIN_NAME -name "*.zip")  _plugins/debian/$PLUGIN_TYPE
+}
+
+build(){
+    case $PLUGIN_NAME in
+        influx )
+            go build -trimpath -modfile extensions.mod --buildmode=plugin -tags plugins -o extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME.go
+            ;;
+        tdengine )
+            if [ "$(uname -m)" = "x86_64" ]; then
+                wget "https://www.taosdata.com/assets-download/TDengine-client-2.2.0.5-Linux-x64.tar.gz" -O /tmp/TDengine-client-2.2.0.5.tar.gz;
+            fi;
+            if [ "$(uname -m)" = "aarch64" ]; then
+                wget "https://www.taosdata.com/assets-download/TDengine-client-2.2.0.5-Linux-aarch64.tar.gz" -O /tmp/TDengine-client-2.2.0.5.tar.gz;
+            fi;
+            tar -zxvf /tmp/TDengine-client-2.2.0.5.tar.gz
+            cd TDengine-client-2.2.0.5 && ./install_client.sh && cd -
+            go build -trimpath -modfile extensions.mod --buildmode=plugin -tags plugins -o extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME.go
+            ;;
+        labelImage )
+            git clone -b v2.2.0-rc3 --depth 1 https://github.com/tensorflow/tensorflow.git /tmp/tensorflow;
+            CGO_CFLAGS=-I/tmp/tensorflow CGO_LDFLAGS=-L$(pwd)/extensions/functions/labelImage/lib go build -trimpath -modfile extensions.mod --buildmode=plugin -o extensions/functions/labelImage/labelImage.so extensions/functions/labelImage/*.go
+            ;;
+        * )
+            go build -trimpath -modfile extensions.mod --buildmode=plugin -o extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/*.go
+          ;;
+    esac
+}
+
+pre
+build
+post