Do not send QoS messages is we're still busy processing the previous ones

.github/workflows/build-container.yml

@@ -575,7 +575,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster"]
+        test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster", "set_qos", "set_qos_2ms"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 1024  # 4 spdk instances

ceph-nvmeof.conf

@@ -98,6 +98,8 @@ transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr
 # Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168}
 # iobuf_options =
 
+qos_timeslice_in_usecs = 0
+
 [monitor]
 #timeout = 1.0
 #log_file_dir =

control/grpc.py

@@ -653,6 +653,8 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler,
         self.up_and_running = True
         self.rebalance = Rebalance(self)
         self.spdk_version = None
+        self.spdk_qos_timeslice = self.config.getint_with_default("spdk",
+                                                                  "qos_timeslice_in_usecs", None)
 
     def get_directories_for_key_file(self, key_type: str,
                                      subsysnqn: str, create_dir: bool = False) -> []:
@@ -2703,6 +2705,8 @@ def namespace_set_qos_limits_safe(self, request, context):
             set_qos_limits_args["r_mbytes_per_sec"] = request.r_mbytes_per_second
         if request.HasField("w_mbytes_per_second"):
             set_qos_limits_args["w_mbytes_per_sec"] = request.w_mbytes_per_second
+        if self.spdk_qos_timeslice:
+            set_qos_limits_args["timeslice_in_usecs"] = self.spdk_qos_timeslice
 
         ns_qos_entry = None
         if context:

tests/ceph-nvmeof.cluster_pool.conf

@@ -92,6 +92,8 @@ transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr
 # Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168}
 # iobuf_options =
 
+# qos_timeslice_in_usecs = 0
+
 [monitor]
 #timeout = 1.0
 #log_file_dir =

tests/ceph-nvmeof.flat_bdevs_per_cluster.conf

@@ -92,6 +92,8 @@ transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr
 # Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168}
 # iobuf_options =
 
+# qos_timeslice_in_usecs = 0
+
 [monitor]
 #timeout = 1.0
 #log_file_dir =

tests/ceph-nvmeof.no-huge.conf

@@ -90,6 +90,11 @@ timeout = 60.0
 # Example value: {"max_queue_depth" : 16, "max_io_size" : 4194304, "io_unit_size" : 1048576, "zcopy" : false}
 transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr" : 7}
 
+# Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168}
+# iobuf_options =
+
+# qos_timeslice_in_usecs = 0
+
 [monitor]
 #timeout = 1.0
 #log_file_dir =

tests/ceph-nvmeof.tls.conf

@@ -88,6 +88,11 @@ timeout = 60.0
 # Example value: {"max_queue_depth" : 16, "max_io_size" : 4194304, "io_unit_size" : 1048576, "zcopy" : false}
 transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr" : 7}
 
+# Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168}
+# iobuf_options =
+
+# qos_timeslice_in_usecs = 0
+
 [monitor]
 #timeout = 1.0
 #log_file_dir =

tests/ha/set_qos.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -xe
+
+NS_COUNT=400
+NQN="nqn.2016-06.io.spdk:cnode1QOS"
+GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}')
+GW1_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW1_NAME")"
+cephnvmf="docker compose run --rm nvmeof-cli --server-address $GW1_IP --server-port 5500"
+
+$cephnvmf subsystem add --subsystem $NQN --max-namespaces 1024 --no-group-append
+
+for i in `seq 1 $NS_COUNT`
+do
+    $cephnvmf namespace add -n $NQN --rbd-pool rbd --rbd-image image${i} --rbd-create-image --size 10MB
+    $cephnvmf namespace set_qos -n $NQN --nsid $i --rw-ios-per-second 150 --rw-megabytes-per-second 19 --r-megabytes-per-second 19 --w-megabytes-per-second 19
+done
+
+$cephnvmf subsystem del --subsystem $NQN --force
+sleep 10

tests/ha/set_qos_2ms.sh

@@ -0,0 +1 @@
+set_qos.sh

tests/ha/setup_set_qos.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exit 0

tests/ha/setup_set_qos_2ms.sh

@@ -0,0 +1 @@
+setup_set_qos.sh

tests/ha/start_up_set_qos.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+# Check if GITHUB_WORKSPACE is defined
+if [ -n "$GITHUB_WORKSPACE" ]; then
+    test_dir="$GITHUB_WORKSPACE/tests/ha"
+else
+    test_dir=$(dirname $0)
+fi
+
+$test_dir/start_up.sh 1

tests/ha/start_up_set_qos_2ms.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+# Check if GITHUB_WORKSPACE is defined
+if [ -n "$GITHUB_WORKSPACE" ]; then
+    test_dir="$GITHUB_WORKSPACE/tests/ha"
+else
+    test_dir=$(dirname $0)
+fi
+
+sed 's/^ *qos_timeslice_in_usecs.*$/qos_timeslice_in_usecs = 2000/' ceph-nvmeof.conf > /tmp/ceph-nvmeof.2ms.conf
+export NVMEOF_CONFIG=/tmp/ceph-nvmeof.2ms.conf
+$test_dir/start_up.sh 1
+rm -f /tmp/ceph-nvmeof.2ms.conf

tests/ha/wait_gateways_set_qos.sh

@@ -0,0 +1,8 @@
+# Check if GITHUB_WORKSPACE is defined
+if [ -n "$GITHUB_WORKSPACE" ]; then
+    test_dir="$GITHUB_WORKSPACE/tests/ha"
+else
+    test_dir=$(dirname $0)
+fi
+
+$test_dir/wait_gateways.sh 1

tests/ha/wait_gateways_set_qos_2ms.sh

@@ -0,0 +1 @@
+wait_gateways_set_qos.sh