add local pv deployments (#465)

Makefile

@@ -7,6 +7,7 @@ DOCKER_REGISTRY ?= docker.io
 DOCKER_REPO ?= ${DOCKER_REGISTRY}/vesoft
 USERNAME ?= ng-user
 IMAGE_TAG ?= v1.7.6
+PROVISIONER_IMAGE_TAG ?= v0.1
 
 CHARTS_VERSION ?= 1.7.6
 
@@ -72,11 +73,13 @@ e2e: kind ## Run e2e test.
 	PATH="${GOBIN}:${PATH}" ./hack/e2e.sh $(E2EARGS)
 
 ##@ Build
-build: ## Build binary.
+build-operator: ## Build operator related binary.
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/controller-manager cmd/controller-manager/main.go
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/autoscaler cmd/autoscaler/main.go
 	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/scheduler cmd/scheduler/main.go
-	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/provisioner cmd/provisioner/main.go
+
+build-provisioner: ## Build provisioner binary.
+	$(GO_BUILD) -ldflags '$(LDFLAGS)' -o bin/$(TARGETDIR)/local-pv-provisioner cmd/provisioner/main.go
 
 helm-charts: ## Build helm charts.
 	helm package charts/nebula-operator --version $(CHARTS_VERSION) --app-version $(CHARTS_VERSION)
@@ -97,7 +100,7 @@ PLATFORMS = arm64 amd64
 BUILDX_PLATFORMS = linux/arm64,linux/amd64
 
 docker-multiarch: ensure-buildx ## Build and push the nebula-operator multiarchitecture docker images and manifest.
-	$(foreach PLATFORM,$(PLATFORMS), echo -n "$(PLATFORM)..."; GOARCH=$(PLATFORM) make build;)
+	$(foreach PLATFORM,$(PLATFORMS), echo -n "$(PLATFORM)..."; GOARCH=$(PLATFORM) make build-operator;)
 	echo "Building and pushing nebula-operator image... $(BUILDX_PLATFORMS)"
 	docker buildx build \
     		--no-cache \
@@ -109,7 +112,7 @@ docker-multiarch: ensure-buildx ## Build and push the nebula-operator multiarchi
     		--build-arg USERNAME=${USERNAME} \
     		-t "${DOCKER_REPO}/nebula-operator:${IMAGE_TAG}" .
 
-alpine-tools: ## Build and push the alpine-tools docker images and manifest.
+alpine-multiarch: ## Build and push the alpine-tools docker images and manifest.
 	echo "Building and pushing alpine-tools image... $(BUILDX_PLATFORMS)"
 	docker buildx rm alpine-tools || true
 	docker buildx create --driver-opt network=host --use --name=alpine-tools
@@ -122,6 +125,20 @@ alpine-tools: ## Build and push the alpine-tools docker images and manifest.
     		--file alpine.multiarch \
     		-t "${DOCKER_REPO}/nebula-alpine:latest" .
 
+provisioner-multiarch: ## Build and push the local-pv-provisioner docker images and manifest.
+	$(foreach PLATFORM,$(PLATFORMS), echo -n "$(PLATFORM)..."; GOARCH=$(PLATFORM) make build-provisioner;)
+	echo "Building and pushing local-pv-provisioner image... $(BUILDX_PLATFORMS)"
+	docker buildx rm provisioner || true
+	docker buildx create --driver-opt network=host --use --name=provisioner
+	docker buildx build \
+    		--no-cache \
+    		--pull \
+    		--push \
+    		--progress plain \
+    		--platform $(BUILDX_PLATFORMS) \
+    		--file provisioner.multiarch \
+    		-t "${DOCKER_REPO}/local-pv-provisioner:${PROVISIONER_IMAGE_TAG}" .
+
 ##@ Deployment
 
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.

config/samples/local-pv-storage.yaml

@@ -88,8 +88,8 @@ spec:
       serviceAccountName: local-pv-provisioner-sa
       containers:
         - name: local-pv-provisioner
-          image: vesoft/local-pv-provisioner
-          imagePullPolicy: IfNotPresent
+          image: vesoft/local-pv-provisioner:v0.1
+          imagePullPolicy: Always
           command:
             - local-pv-provisioner
           volumeMounts:
@@ -109,7 +109,7 @@ spec:
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: local-pv
+  name: local-nvme
 provisioner: nebula-cloud.io/local-pv
 volumeBindingMode: WaitForFirstConsumer
 reclaimPolicy: Delete

config/samples/raid-disks/eks-daemonset-raid-disks.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: eks-raid-disks
+  namespace: default
+  labels:
+    k8s-app: eks-raid-disks
+spec:
+  selector:
+    matchLabels:
+      name: eks-raid-disks
+  template:
+    metadata:
+      labels:
+        name: eks-raid-disks
+    spec:
+      nodeSelector:
+        node.kubernetes.io/instance-type: "m5ad.large"
+      hostPID: true
+      containers:
+      - name: startup-script
+        image: megabyte875/startup-script
+        imagePullPolicy: Always
+        securityContext:
+          privileged: true
+        env:
+        - name: STARTUP_SCRIPT
+          value: |
+            set -o errexit
+            set -o nounset
+            set -o pipefail
+
+            nvme_drives=$(nvme list | grep "Amazon EC2 NVMe Instance Storage" | cut -d " " -f 1 || true)
+            readarray -t nvme_drives <<< "$nvme_drives"
+            num_drives=${#nvme_drives[@]}
+            
+            if [ "${#nvme_drives[@]}" -eq 0 ]; then
+              echo "No NVMe instance storage found."
+              exit 0
+            fi
+
+            seen_arrays=(/dev/md/*)
+            device=${seen_arrays[0]}
+            echo "Setting RAID array with Local SSDs on device ${device}"
+            if [ ! -e "$device" ]; then
+              device="/dev/md/0"
+              echo "y" | mdadm --create "${device}" --level=0 --force --raid-devices=${#nvme_drives[@]} "${nvme_drives[@]}"
+            fi
+
+            if ! tune2fs -l "${device}" ; then
+              echo "Formatting '${device}'"
+              mkfs.ext4 -F "${device}"
+            fi
+
+            mountpoint=/mnt/disks/raid0
+            mkdir -p "${mountpoint}"
+            echo "Mounting '${device}' at '${mountpoint}'"
+            mount -o discard,defaults "${device}" "${mountpoint}"
+            chmod a+w "${mountpoint}"
+            rm -rf /mnt/disks/raid0/*

config/samples/raid-disks/gke-daemonset-raid-disks.yaml

@@ -0,0 +1,61 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: gke-raid-disks
+  namespace: default
+  labels:
+    k8s-app: gke-raid-disks
+spec:
+  selector:
+    matchLabels:
+      name: gke-raid-disks
+  template:
+    metadata:
+      labels:
+        name: gke-raid-disks
+    spec:
+      nodeSelector:
+        cloud.google.com/gke-local-nvme-ssd: "true"
+      hostPID: true
+      containers:
+      - name: startup-script
+        image: gcr.io/google-containers/startup-script:v1
+        securityContext:
+          privileged: true
+        env:
+        - name: STARTUP_SCRIPT
+          value: |
+            set -o errexit
+            set -o nounset
+            set -o pipefail
+
+            devices=()
+            for ssd in /dev/disk/by-id/google-local-nvme-ssd*; do
+              if [ -e "${ssd}" ]; then
+                devices+=("${ssd}")
+              fi
+            done
+            if [ "${#devices[@]}" -eq 0 ]; then
+              echo "No Local NVMe SSD disks found."
+              exit 0
+            fi
+
+            seen_arrays=(/dev/md/*)
+            device=${seen_arrays[0]}
+            echo "Setting RAID array with Local SSDs on device ${device}"
+            if [ ! -e "$device" ]; then
+              device="/dev/md/0"
+              echo "y" | mdadm --create "${device}" --level=0 --force --raid-devices=${#devices[@]} "${devices[@]}"
+            fi
+
+            if ! tune2fs -l "${device}" ; then
+              echo "Formatting '${device}'"
+              mkfs.ext4 -F "${device}"
+            fi
+
+            mountpoint=/mnt/disks/raid0
+            mkdir -p "${mountpoint}"
+            echo "Mounting '${device}' at '${mountpoint}'"
+            mount -o discard,defaults "${device}" "${mountpoint}"
+            chmod a+w "${mountpoint}"
+            rm -rf /mnt/disks/raid0/*

hack/startup-script/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.18.2
+
+RUN apk update && apk upgrade && apk add --no-cache bash nvme-cli bash util-linux
+ADD hack/manage-startup-script.sh /
+CMD /manage-startup-script.sh

hack/startup-script/Makefile

@@ -0,0 +1,10 @@
+TAG=v1
+IMAGE=vesoft/startup-script
+
+.PHONY: build push
+
+build:
+	docker build --pull --network host -t $(IMAGE):$(TAG) .
+
+push: build
+	docker push $(IMAGE):$(TAG)

hack/startup-script/manage-startup-script.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+CHECKPOINT_PATH="${CHECKPOINT_PATH:-/tmp/startup-script.kubernetes.io}"
+CHECK_INTERVAL_SECONDS="30"
+EXEC=(nsenter -t 1 -m -u -i -n -p --)
+
+do_startup_script() {
+  local err=0;
+
+  "${EXEC[@]}" bash -c "${STARTUP_SCRIPT}" && err=0 || err=$?
+  if [[ ${err} != 0 ]]; then
+    echo "!!! startup-script failed! exit code '${err}'" 1>&2
+    return 1
+  fi
+
+  "${EXEC[@]}" touch "${CHECKPOINT_PATH}"
+  echo "!!! startup-script succeeded!" 1>&2
+  return 0
+}
+
+while :; do
+  "${EXEC[@]}" stat "${CHECKPOINT_PATH}" > /dev/null 2>&1 && err=0 || err=$?
+  if [[ ${err} != 0 ]]; then
+    do_startup_script
+  fi
+
+  sleep "${CHECK_INTERVAL_SECONDS}"
+done

pkg/controller/provisioner/config.go

@@ -29,7 +29,7 @@ const (
 	DefaultMountPoint = "/mnt/disks/raid0"
 
 	// DefaultFsType is the filesystem type to mount
-	DefaultFsType = "xfs"
+	DefaultFsType = "ext4"
 
 	// LocalFilesystemEnv will contain the filesystm path when script is invoked
 	LocalFilesystemEnv = "LOCAL_PV_FILESYSTEM"

provisioner.multiarch

@@ -0,0 +1,11 @@
+FROM alpine:3.18.2
+
+ARG TARGETPLATFORM
+ARG TARGETARCH
+
+RUN apk update \
+  && apk upgrade \
+  && apk add --no-cache \
+    util-linux bash \
+  && rm -rf /var/cache/apk/*
+ADD bin/${TARGETPLATFORM}/local-pv-provisioner /usr/local/bin/local-pv-provisioner