From 90252774fc32ac46248407883ca45885aa912299 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Sat, 30 Dec 2023 20:05:19 -0800 Subject: [PATCH] first commit Signed-off-by: Tamal Saha --- .github/workflows/ci.yml | 75 ++++++++++++++++++ .gitignore | 15 ++++ README.md | 4 + dep-refresh.sh | 121 +++++++++++++++++++++++++++++ refresh-repo.sh | 161 +++++++++++++++++++++++++++++++++++++++ repos.txt | 10 +++ 6 files changed, 386 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100755 dep-refresh.sh create mode 100755 refresh-repo.sh create mode 100644 repos.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9a19c44 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,75 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + # schedule: + # - cron: "0 6 * * *" + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: '1.20' + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Install yq + run: | + curl -fsSL -o yq https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64 + chmod +x yq + sudo mv yq /usr/local/bin/yq + + - name: Install Kubernetes CLI + run: | + curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + - name: Install gomod-gen + run: | + curl -fsSL -O https://github.com/appscodelabs/gomod-gen/releases/download/v0.1.3/gomod-gen-linux-amd64 + chmod +x gomod-gen-linux-amd64 + sudo mv gomod-gen-linux-amd64 /usr/local/bin/gomod-gen + + - name: Prepare git + env: + GITHUB_USER: 1gtm + GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + run: | + set -x + git config --global user.name "1gtm" + git config --global user.email "1gtm@appscode.com" + git config --global \ + url."https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com".insteadOf \ + "https://github.com" + # git remote set-url origin https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git + + - name: Prepare Go for private repos + run: | + go env -w GOPRIVATE=go.appscode.dev/*,github.com/appscode/*,kubedb.dev/* + + - name: Install GitHub CLI + run: | + curl -fsSL https://github.com/github/hub/raw/master/script/get | bash -s 2.14.1 + sudo mv bin/hub /usr/local/bin + + - name: Refresh repositories + env: + GITHUB_TOKEN: ${{ secrets.LGTM_GITHUB_TOKEN }} + run: | + ./refresh-repo.sh repos.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..66fd13c --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..e517db5 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +![CI](https://github.com/appscodelabs/kubedb-api-refresher/workflows/CI/badge.svg) + +# kubedb-api-refresher + diff --git a/dep-refresh.sh b/dep-refresh.sh new file mode 100755 index 0000000..4899117 --- /dev/null +++ b/dep-refresh.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# set -eou pipefail + +SCRIPT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")) +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") + +GITHUB_USER=${GITHUB_USER:-1gtm} +PR_BRANCH=k129-auto # -$(date +%s) +COMMIT_MSG="Use k8s 1.29 client libs" + +REPO_ROOT=/tmp/kubedb-repo-refresher + +KUBEDB_API_REF=${KUBEDB_API_REF:-7263b50309d2e37f83f763f0448a4faeac1d5687} + +repo_uptodate() { + # gomodfiles=(go.mod go.sum vendor/modules.txt) + gomodfiles=(go.sum vendor/modules.txt) + changed=($(git diff --name-only)) + changed+=("${gomodfiles[@]}") + # https://stackoverflow.com/a/28161520 + diff=($(echo ${changed[@]} ${gomodfiles[@]} | tr ' ' '\n' | sort | uniq -u)) + return ${#diff[@]} +} + +refresh() { + echo "refreshing repository: $1" + rm -rf $REPO_ROOT + mkdir -p $REPO_ROOT + pushd $REPO_ROOT + git clone --no-tags --no-recurse-submodules --depth=1 git@github.com:$1.git + name=$(ls -b1) + cd $name + git checkout -b $PR_BRANCH + + sed -i 's/?=\ 1.20/?=\ 1.21/g' Makefile + sed -i 's|appscode/gengo:release-1.25|appscode/gengo:release-1.29|g' Makefile + sed -i 's/goconst,//g' Makefile + sed -i 's|gcr.io/distroless/static-debian11|gcr.io/distroless/static-debian12|g' Makefile + sed -i 's|debian:bullseye|debian:bookworm|g' Makefile + + pushd .github/workflows/ && { + # update GO + sed -i 's/Go\ 1.20/Go\ 1.21/g' * + sed -i 's/go-version:\ ^1.20/go-version:\ ^1.21/g' * + sed -i 's/go-version:\ 1.20/go-version:\ 1.21/g' * + popd + } + + if [ -f go.mod ]; then + go mod edit \ + -require=gomodules.xyz/logs@v0.0.7 \ + -require=kubedb.dev/apimachinery@v0.40.0 \ + -require=kubedb.dev/db-client-go@v0.0.8 \ + -require=kmodules.xyz/client-go@v0.29.4 \ + -require=kmodules.xyz/resource-metadata@v0.18.1 \ + -require=kmodules.xyz/go-containerregistry@v0.0.12 \ + -require=gomodules.xyz/password-generator@v0.2.9 \ + -require=go.bytebuilders.dev/license-verifier@v0.13.4 \ + -require=go.bytebuilders.dev/license-verifier/kubernetes@v0.13.4 \ + -require=go.bytebuilders.dev/audit@3ff33160c6f02f6151e59cdd44dd50a347c02ba0 \ + -require=github.com/elastic/go-elasticsearch/v7@v7.15.1 \ + -require=go.mongodb.org/mongo-driver@v1.10.2 \ + -replace=github.com/Masterminds/sprig/v3=github.com/gomodules/sprig/v3@v3.2.3-0.20220405051441-0a8a99bac1b8 \ + -replace=sigs.k8s.io/controller-runtime=github.com/kmodules/controller-runtime@ac-0.17.0 \ + -replace=github.com/imdario/mergo=github.com/imdario/mergo@v0.3.6 \ + -replace=k8s.io/apiserver=github.com/kmodules/apiserver@ac-1.29.0 \ + -replace=k8s.io/kubernetes=github.com/kmodules/kubernetes@ac-1.29.0 + + # sed -i 's|NewLicenseEnforcer|MustLicenseEnforcer|g' `grep 'NewLicenseEnforcer' -rl *` + go mod tidy + go mod vendor + fi + [ -z "$2" ] || ( + echo "$2" + $2 || true + # run an extra make fmt because when make gen fails, make fmt is not run + make fmt || true + ) + make fmt || true + if repo_uptodate; then + echo "Repository $1 is up-to-date." + else + git add --all + if [[ "$1" == *"stashed"* ]]; then + git commit -a -s -m "$COMMIT_MSG" -m "/cherry-pick" + else + git commit -a -s -m "$COMMIT_MSG" + fi + git push -u origin HEAD -f + hub pull-request \ + --labels automerge \ + --message "$COMMIT_MSG" \ + --message "Signed-off-by: $(git config --get user.name) <$(git config --get user.email)>" || true + # gh pr create \ + # --base master \ + # --fill \ + # --label automerge \ + # --reviewer tamalsaha + fi + popd +} + +if [ "$#" -ne 1 ]; then + echo "Illegal number of parameters" + echo "Correct usage: $SCRIPT_NAME " + exit 1 +fi + +if [ -x $GITHUB_TOKEN ]; then + echo "Missing env variable GITHUB_TOKEN" + exit 1 +fi + +# ref: https://linuxize.com/post/how-to-read-a-file-line-by-line-in-bash/#using-file-descriptor +while IFS=, read -r -u9 repo cmd; do + if [ -z "$repo" ]; then + continue + fi + refresh "$repo" "$cmd" + echo "################################################################################" +done 9<$1 diff --git a/refresh-repo.sh b/refresh-repo.sh new file mode 100755 index 0000000..595500f --- /dev/null +++ b/refresh-repo.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# set -eou pipefail + +SCRIPT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")) +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") + +GITHUB_USER=${GITHUB_USER:-1gtm} +PR_BRANCH=go19 # -$(date +%s) +COMMIT_MSG="Use k8s 1.29 client libs" + +REPO_ROOT=/tmp/kubestash-repo-refresher + +KUBEDB_API_REF=${KUBEDB_API_REF:-7263b50309d2e37f83f763f0448a4faeac1d5687} + +repo_uptodate() { + # gomodfiles=(go.mod go.sum vendor/modules.txt) + gomodfiles=(go.sum vendor/modules.txt) + changed=($(git diff --name-only)) + changed+=("${gomodfiles[@]}") + # https://stackoverflow.com/a/28161520 + diff=($(echo ${changed[@]} ${gomodfiles[@]} | tr ' ' '\n' | sort | uniq -u)) + return ${#diff[@]} +} + +refresh() { + echo "refreshing repository: $1" + rm -rf $REPO_ROOT + mkdir -p $REPO_ROOT + pushd $REPO_ROOT + git clone --no-tags --no-recurse-submodules --depth=1 git@github.com:$1.git + name=$(ls -b1) + cd $name + git checkout -b $PR_BRANCH + + sed -i 's/?=\ 1.20/?=\ 1.21/g' Makefile + sed -i 's|appscode/gengo:release-1.25|appscode/gengo:release-1.29|g' Makefile + + pushd .github/workflows/ && { + # update GO + sed -i 's/Go\ 1.20/Go\ 1.21/g' * + sed -i 's/go-version:\ ^1.20/go-version:\ ^1.21/g' * + sed -i 's/go-version:\ 1.20/go-version:\ 1.21/g' * + popd + } + sed -i 's|ioutil.ReadFile|os.ReadFile|g' `grep 'ioutil.ReadFile' -rl *` + sed -i 's|ioutil.WriteFile|os.WriteFile|g' `grep 'ioutil.WriteFile' -rl *` + sed -i 's|ioutil.ReadAll|io.ReadAll|g' `grep 'ioutil.ReadAll' -rl *` + sed -i 's|ioutil.TempDir|os.MkdirTemp|g' `grep 'ioutil.TempDir' -rl *` + sed -i 's|ioutil.TempFile|os.CreateTemp|g' `grep 'ioutil.TempFile' -rl *` + + if [ -f go.mod ]; then + cat < go.mod +module kubestash.dev/$name + +go 1.21 + +EOF + # https://stackoverflow.com/a/63918676/244009 + # sed -i '' -e 's|github.com/jetstack/cert-manager|github.com/cert-manager/cert-manager|g' `grep 'jetstack' -rl *` + # sed -i 's|github.com/jetstack/cert-manager|github.com/cert-manager/cert-manager|g' `grep 'jetstack' -rl *` + + # sed -i "s|go 1.12|go 1.17|g" go.mod + # sed -i "s|go 1.13|go 1.17|g" go.mod + # sed -i "s|go 1.14|go 1.17|g" go.mod + # sed -i "s|go 1.15|go 1.17|g" go.mod + # sed -i "s|go 1.16|go 1.17|g" go.mod + # if [ "$1" != "github.com/kubedb/apimachinery" ]; then + # go mod edit \ + # -require kubedb.dev/apimachinery@${KUBEDB_API_REF} + # go mod tidy + # fi + go mod edit \ + -require=kubedb.dev/apimachinery@${KUBEDB_API_REF} \ + -require=kubedb.dev/db-client-go@v0.0.4 \ + -require=k8s.io/kube-openapi@v0.0.0-20220803162953-67bda5d908f1 \ + -require=kmodules.xyz/resource-metadata@v0.13.0 \ + -replace=github.com/Masterminds/sprig/v3=github.com/gomodules/sprig/v3@v3.2.3-0.20220405051441-0a8a99bac1b8 \ + -require=gomodules.xyz/password-generator@v0.2.8 \ + -require=go.bytebuilders.dev/license-verifier@v0.12.0 \ + -require=go.bytebuilders.dev/license-verifier/kubernetes@v0.12.0 \ + -require=go.bytebuilders.dev/audit@v0.0.24 \ + -require=stash.appscode.dev/apimachinery@master \ + -require=github.com/elastic/go-elasticsearch/v7@v7.13.1 \ + -require=go.mongodb.org/mongo-driver@v1.10.2 \ + -replace=sigs.k8s.io/controller-runtime=github.com/kmodules/controller-runtime@ac-0.13.0 \ + -replace=github.com/imdario/mergo=github.com/imdario/mergo@v0.3.6 \ + -replace=k8s.io/apiserver=github.com/kmodules/apiserver@ac-1.25.1 \ + -replace=k8s.io/kubernetes=github.com/kmodules/kubernetes@ac-1.25.1 + +# cat <> go.mod +# replace ( +# go.opencensus.io => go.opencensus.io v0.23.0 +# go.opentelemetry.io/contrib => go.opentelemetry.io/contrib v0.20.0 +# go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful => go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.20.0 +# go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 +# go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 +# go.opentelemetry.io/contrib/propagators => go.opentelemetry.io/contrib/propagators v0.20.0 +# go.opentelemetry.io/otel => go.opentelemetry.io/otel v0.20.0 +# go.opentelemetry.io/otel/exporters/otlp => go.opentelemetry.io/otel/exporters/otlp v0.20.0 +# go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.20.0 +# go.opentelemetry.io/otel/oteltest => go.opentelemetry.io/otel/oteltest v0.20.0 +# go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v0.20.0 +# go.opentelemetry.io/otel/sdk/export/metric => go.opentelemetry.io/otel/sdk/export/metric v0.20.0 +# go.opentelemetry.io/otel/sdk/metric => go.opentelemetry.io/otel/sdk/metric v0.20.0 +# go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v0.20.0 +# go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.7.0 +# ) +# EOF + # sed -i 's|NewLicenseEnforcer|MustLicenseEnforcer|g' `grep 'NewLicenseEnforcer' -rl *` + go mod tidy + go mod vendor + fi + [ -z "$2" ] || ( + echo "$2" + $2 || true + # run an extra make fmt because when make gen fails, make fmt is not run + make fmt || true + ) + make fmt || true + if repo_uptodate; then + echo "Repository $1 is up-to-date." + else + git add --all + if [[ "$1" == *"stashed"* ]]; then + git commit -a -s -m "$COMMIT_MSG" -m "/cherry-pick" + else + git commit -a -s -m "$COMMIT_MSG" + fi + git push -u origin HEAD -f + hub pull-request \ + --labels automerge \ + --message "$COMMIT_MSG" \ + --message "Signed-off-by: $(git config --get user.name) <$(git config --get user.email)>" || true + # gh pr create \ + # --base master \ + # --fill \ + # --label automerge \ + # --reviewer tamalsaha + fi + popd +} + +if [ "$#" -ne 1 ]; then + echo "Illegal number of parameters" + echo "Correct usage: $SCRIPT_NAME " + exit 1 +fi + +if [ -x $GITHUB_TOKEN ]; then + echo "Missing env variable GITHUB_TOKEN" + exit 1 +fi + +# ref: https://linuxize.com/post/how-to-read-a-file-line-by-line-in-bash/#using-file-descriptor +while IFS=, read -r -u9 repo cmd; do + if [ -z "$repo" ]; then + continue + fi + refresh "$repo" "$cmd" + echo "################################################################################" +done 9<$1 diff --git a/repos.txt b/repos.txt new file mode 100644 index 0000000..9fee893 --- /dev/null +++ b/repos.txt @@ -0,0 +1,10 @@ +kubestash/CHANGELOG +kubestash/cli +kubestash/docs +kubestash/installer +kubestash/kubedump +kubestash/kubestash +kubestash/manifest +kubestash/pvc +kubestash/volume-snapshotter +kubestash/workload \ No newline at end of file