vmcluster: improve components readiness check (#1907)

Remove vmstorage liveness probe, as vminsert already handles routing and retries, while liveness probes can inadvertently introduce delays, DNS instability, and unnecessary disruptions. Reduce the default readiness probe interval to 5s (was 15s) and the failure threshold to 10 (was 3). Add a default minReadySeconds for vmstorage, to help stabilizing service during rollout. Co-authored-by: Phuong Le <39565248+func25@users.noreply.github.com> * add update note --------- Co-authored-by: Andrii Chubatiuk <andrew.chubatiuk@gmail.com> Co-authored-by: Haley Wang <haley@victoriametrics.com>
VictoriaMetrics · Jan 9, 2025 · 8e64912 · 8e64912
1 parent 641a329
commit 8e64912
Show file tree

Hide file tree

Showing 8 changed files with 48 additions and 60 deletions.
diff --git a/charts/victoria-metrics-cluster/CHANGELOG.md b/charts/victoria-metrics-cluster/CHANGELOG.md
@@ -1,6 +1,11 @@
 ## Next release
 
-- TODO
+**Update note**: A default `minReadySeconds` has been added for the vmstorage statefulset, vmstorage pods will restart after the upgrade.
+**Update note**: The default probes of vminsert, vmselect, vmauth, and vmstorage have been changed, all pods will restart after the upgrade.
+
+- Remove vmstorage liveness probe, as vminsert already handles routing and retries, while liveness probes can inadvertently introduce delays, DNS instability, and unnecessary disruptions.
+- Reduce the default readiness probe interval to 5s (was 15s) and the failure threshold to 10 (was 3).
+- Add a default minReadySeconds for vmstorage, to help stabilizing service during rollout.
 
 ## 0.16.2
 

diff --git a/charts/victoria-metrics-cluster/README.md b/charts/victoria-metrics-cluster/README.md
@@ -1591,10 +1591,10 @@ labels: {}
     tcpSocket: {}
     timeoutSeconds: 5
 readiness:
-    failureThreshold: 3
+    failureThreshold: 10
     httpGet: {}
     initialDelaySeconds: 5
-    periodSeconds: 15
+    periodSeconds: 5
     timeoutSeconds: 5
 startup: {}
 </code>
@@ -2531,10 +2531,10 @@ labels: {}
     tcpSocket: {}
     timeoutSeconds: 5
 readiness:
-    failureThreshold: 3
+    failureThreshold: 10
     httpGet: {}
     initialDelaySeconds: 5
-    periodSeconds: 15
+    periodSeconds: 5
     timeoutSeconds: 5
 startup: {}
 </code>
@@ -3435,10 +3435,10 @@ labels: {}
     tcpSocket: {}
     timeoutSeconds: 5
 readiness:
-    failureThreshold: 3
+    failureThreshold: 10
     httpGet: {}
     initialDelaySeconds: 5
-    periodSeconds: 15
+    periodSeconds: 5
     timeoutSeconds: 5
 startup: {}
 </code>
@@ -3952,11 +3952,11 @@ loggerFormat: json
         port: manager-http
     timeoutSeconds: 5
 readiness:
-    failureThreshold: 3
+    failureThreshold: 10
     httpGet:
         port: manager-http
     initialDelaySeconds: 5
-    periodSeconds: 15
+    periodSeconds: 5
     timeoutSeconds: 5
 startup: {}
 </code>

diff --git a/charts/victoria-metrics-cluster/templates/vmstorage-statefulset.yaml b/charts/victoria-metrics-cluster/templates/vmstorage-statefulset.yaml
@@ -250,6 +250,7 @@ spec:
         {{- end }}
         {{- include "vm.license.volume" . | nindent 8 }}
       {{- end }}
+  minReadySeconds: {{ $app.minReadySeconds }}
   {{- if and $app.persistentVolume.enabled (not $app.persistentVolume.existingClaim) }}
   volumeClaimTemplates:
     - apiVersion: v1

diff --git a/charts/victoria-metrics-cluster/tests/__snapshot__/vmauth_test.yaml.snap b/charts/victoria-metrics-cluster/tests/__snapshot__/vmauth_test.yaml.snap
@@ -49,13 +49,13 @@ deployment should match snapshot:
                 - containerPort: 8427
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               volumeMounts:
                 - mountPath: /config
@@ -118,13 +118,13 @@ deployment should match snapshot with fullnameOverride, extraLabels and podLabel
                 - containerPort: 8427
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               volumeMounts:
                 - mountPath: /config

diff --git a/charts/victoria-metrics-cluster/tests/__snapshot__/vminsert_test.yaml.snap b/charts/victoria-metrics-cluster/tests/__snapshot__/vminsert_test.yaml.snap
@@ -50,13 +50,13 @@ deployment should match snapshot:
                 - containerPort: 8480
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
           serviceAccountName: RELEASE-NAME-victoria-metrics-cluster
 deployment should match snapshot with fullnameOverride, extraLabels and podLabels:
@@ -113,12 +113,12 @@ deployment should match snapshot with fullnameOverride, extraLabels and podLabel
                 - containerPort: 8480
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
           serviceAccountName: RELEASE-NAME-victoria-metrics-cluster
diff --git a/charts/victoria-metrics-cluster/tests/__snapshot__/vmselect_test.yaml.snap b/charts/victoria-metrics-cluster/tests/__snapshot__/vmselect_test.yaml.snap
@@ -51,13 +51,13 @@ deployment should match snapshot:
                 - containerPort: 8481
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               securityContext: {}
               volumeMounts:
@@ -123,13 +123,13 @@ deployment should match snapshot with fullnameOverride, extraLabels and podLabel
                 - containerPort: 8481
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               securityContext: {}
               volumeMounts:
@@ -197,13 +197,13 @@ statefulset should match snapshot:
                 - containerPort: 8481
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               securityContext: {}
               volumeMounts:
@@ -273,13 +273,13 @@ statefulset should match snapshot with fullnameOverride, extraLabels and podLabe
                 - containerPort: 8481
                   name: http
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               securityContext: {}
               volumeMounts:

diff --git a/charts/victoria-metrics-cluster/tests/__snapshot__/vmstorage_test.yaml.snap b/charts/victoria-metrics-cluster/tests/__snapshot__/vmstorage_test.yaml.snap
@@ -13,6 +13,7 @@ statefulset should match snapshot:
       name: RELEASE-NAME-victoria-metrics-cluster-vmstorage
       namespace: NAMESPACE
     spec:
+      minReadySeconds: 5
       podManagementPolicy: OrderedReady
       replicas: 2
       selector:
@@ -40,13 +41,6 @@ statefulset should match snapshot:
                 - --storageDataPath=/storage
               image: victoriametrics/vmstorage:0.1.0-cluster
               imagePullPolicy: IfNotPresent
-              livenessProbe:
-                failureThreshold: 10
-                initialDelaySeconds: 30
-                periodSeconds: 30
-                tcpSocket:
-                  port: http
-                timeoutSeconds: 5
               name: vmstorage
               ports:
                 - containerPort: 8482
@@ -56,13 +50,13 @@ statefulset should match snapshot:
                 - containerPort: 8401
                   name: vmselect
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               volumeMounts:
                 - mountPath: /storage
@@ -96,6 +90,7 @@ statefulset should match snapshot with fullnameOverride, extraLabels and podLabe
       name: vmstorage-node
       namespace: NAMESPACE
     spec:
+      minReadySeconds: 5
       podManagementPolicy: OrderedReady
       replicas: 2
       selector:
@@ -124,13 +119,6 @@ statefulset should match snapshot with fullnameOverride, extraLabels and podLabe
                 - --storageDataPath=/storage
               image: victoriametrics/vmstorage:0.1.0-cluster
               imagePullPolicy: IfNotPresent
-              livenessProbe:
-                failureThreshold: 10
-                initialDelaySeconds: 30
-                periodSeconds: 30
-                tcpSocket:
-                  port: http
-                timeoutSeconds: 5
               name: vmstorage
               ports:
                 - containerPort: 8482
@@ -140,13 +128,13 @@ statefulset should match snapshot with fullnameOverride, extraLabels and podLabe
                 - containerPort: 8401
                   name: vmselect
               readinessProbe:
-                failureThreshold: 3
+                failureThreshold: 10
                 httpGet:
                   path: /health
                   port: http
                   scheme: HTTP
                 initialDelaySeconds: 5
-                periodSeconds: 15
+                periodSeconds: 5
                 timeoutSeconds: 5
               volumeMounts:
                 - mountPath: /storage

diff --git a/charts/victoria-metrics-cluster/values.yaml b/charts/victoria-metrics-cluster/values.yaml
@@ -109,9 +109,9 @@ vmselect:
     readiness:
       httpGet: {}
       initialDelaySeconds: 5
-      periodSeconds: 15
+      periodSeconds: 5
       timeoutSeconds: 5
-      failureThreshold: 3
+      failureThreshold: 10
     # -- VMSelect liveness probe
     liveness:
       tcpSocket: {}
@@ -389,9 +389,9 @@ vminsert:
     readiness:
       httpGet: {}
       initialDelaySeconds: 5
-      periodSeconds: 15
+      periodSeconds: 5
       timeoutSeconds: 5
-      failureThreshold: 3
+      failureThreshold: 10
     # -- VMInsert liveness probe
     liveness:
       tcpSocket: {}
@@ -639,9 +639,9 @@ vmauth:
     readiness:
       httpGet: {}
       initialDelaySeconds: 5
-      periodSeconds: 15
+      periodSeconds: 5
       timeoutSeconds: 5
-      failureThreshold: 3
+      failureThreshold: 10
     # -- VMAuth liveness probe
     liveness:
       tcpSocket: {}
@@ -1021,20 +1021,14 @@ vmstorage:
     ipFamilies: []
   # -- Pod's termination grace period in seconds
   terminationGracePeriodSeconds: 60
-  # -- Readiness & Liveness probes
+  minReadySeconds: 5
+  # -- Readiness probes
   probe:
     # -- VMStorage readiness probe
     readiness:
       httpGet: {}
       initialDelaySeconds: 5
-      periodSeconds: 15
-      timeoutSeconds: 5
-      failureThreshold: 3
-    # -- VMStorage liveness probe
-    liveness:
-      tcpSocket: {}
-      initialDelaySeconds: 30
-      periodSeconds: 30
+      periodSeconds: 5
       timeoutSeconds: 5
       failureThreshold: 10
     # -- VMStorage startup probe
@@ -1108,9 +1102,9 @@ vmstorage:
         httpGet:
           port: manager-http
         initialDelaySeconds: 5
-        periodSeconds: 15
+        periodSeconds: 5
         timeoutSeconds: 5
-        failureThreshold: 3
+        failureThreshold: 10
       # -- VMBackupManager liveness probe
       liveness:
         tcpSocket: