diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index ec737684..4127623b 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -575,7 +575,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster"] + test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster", "set_qos", "set_qos_2ms"] runs-on: ubuntu-latest env: HUGEPAGES: 1024 # 4 spdk instances diff --git a/ceph-nvmeof.conf b/ceph-nvmeof.conf index f24777f8..8e3e29bc 100644 --- a/ceph-nvmeof.conf +++ b/ceph-nvmeof.conf @@ -98,6 +98,8 @@ transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr # Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168} # iobuf_options = +qos_timeslice_in_usecs = 0 + [monitor] #timeout = 1.0 #log_file_dir = diff --git a/control/grpc.py b/control/grpc.py index 195c52aa..dfdb5418 100644 --- a/control/grpc.py +++ b/control/grpc.py @@ -653,6 +653,8 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, self.up_and_running = True self.rebalance = Rebalance(self) self.spdk_version = None + self.spdk_qos_timeslice = self.config.getint_with_default("spdk", + "qos_timeslice_in_usecs", None) def get_directories_for_key_file(self, key_type: str, subsysnqn: str, create_dir: bool = False) -> []: @@ -2703,6 +2705,8 @@ def namespace_set_qos_limits_safe(self, request, context): set_qos_limits_args["r_mbytes_per_sec"] = request.r_mbytes_per_second if request.HasField("w_mbytes_per_second"): set_qos_limits_args["w_mbytes_per_sec"] = request.w_mbytes_per_second + if self.spdk_qos_timeslice: + set_qos_limits_args["timeslice_in_usecs"] = self.spdk_qos_timeslice ns_qos_entry = None if context: diff --git a/spdk b/spdk index 4ee571e3..dd170cf7 160000 --- a/spdk +++ b/spdk @@ -1 +1 @@ -Subproject commit 4ee571e3e500d19eca66e5a72f04bf11c391f615 +Subproject commit dd170cf751f3f9ed50e7dabe45a421cdfbe5be33 diff --git a/tests/ceph-nvmeof.cluster_pool.conf b/tests/ceph-nvmeof.cluster_pool.conf index e1640717..ee1029f7 100644 --- a/tests/ceph-nvmeof.cluster_pool.conf +++ b/tests/ceph-nvmeof.cluster_pool.conf @@ -92,6 +92,8 @@ transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr # Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168} # iobuf_options = +# qos_timeslice_in_usecs = 0 + [monitor] #timeout = 1.0 #log_file_dir = diff --git a/tests/ceph-nvmeof.flat_bdevs_per_cluster.conf b/tests/ceph-nvmeof.flat_bdevs_per_cluster.conf index 8bf3353d..43b9b4a3 100644 --- a/tests/ceph-nvmeof.flat_bdevs_per_cluster.conf +++ b/tests/ceph-nvmeof.flat_bdevs_per_cluster.conf @@ -92,6 +92,8 @@ transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr # Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168} # iobuf_options = +# qos_timeslice_in_usecs = 0 + [monitor] #timeout = 1.0 #log_file_dir = diff --git a/tests/ceph-nvmeof.no-huge.conf b/tests/ceph-nvmeof.no-huge.conf index 5d36ba74..4996d85f 100644 --- a/tests/ceph-nvmeof.no-huge.conf +++ b/tests/ceph-nvmeof.no-huge.conf @@ -90,6 +90,11 @@ timeout = 60.0 # Example value: {"max_queue_depth" : 16, "max_io_size" : 4194304, "io_unit_size" : 1048576, "zcopy" : false} transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr" : 7} +# Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168} +# iobuf_options = + +# qos_timeslice_in_usecs = 0 + [monitor] #timeout = 1.0 #log_file_dir = diff --git a/tests/ceph-nvmeof.tls.conf b/tests/ceph-nvmeof.tls.conf index c8ad09c2..840ad497 100644 --- a/tests/ceph-nvmeof.tls.conf +++ b/tests/ceph-nvmeof.tls.conf @@ -88,6 +88,11 @@ timeout = 60.0 # Example value: {"max_queue_depth" : 16, "max_io_size" : 4194304, "io_unit_size" : 1048576, "zcopy" : false} transport_tcp_options = {"in_capsule_data_size" : 8192, "max_io_qpairs_per_ctrlr" : 7} +# Example value: {"small_pool_count" : 8192, "large_pool_count" : 1024, "small_bufsize" : 8192, "large_bufsize" : 135168} +# iobuf_options = + +# qos_timeslice_in_usecs = 0 + [monitor] #timeout = 1.0 #log_file_dir = diff --git a/tests/ha/set_qos.sh b/tests/ha/set_qos.sh new file mode 100755 index 00000000..d91f5f3c --- /dev/null +++ b/tests/ha/set_qos.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -xe + +NS_COUNT=400 +NQN="nqn.2016-06.io.spdk:cnode1QOS" +GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}') +GW1_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$GW1_NAME")" +cephnvmf="docker compose run --rm nvmeof-cli --server-address $GW1_IP --server-port 5500" + +$cephnvmf subsystem add --subsystem $NQN --max-namespaces 1024 --no-group-append + +for i in `seq 1 $NS_COUNT` +do + $cephnvmf namespace add -n $NQN --rbd-pool rbd --rbd-image image${i} --rbd-create-image --size 10MB + $cephnvmf namespace set_qos -n $NQN --nsid $i --rw-ios-per-second 150 --rw-megabytes-per-second 19 --r-megabytes-per-second 19 --w-megabytes-per-second 19 +done + +$cephnvmf subsystem del --subsystem $NQN --force +sleep 10 diff --git a/tests/ha/set_qos_2ms.sh b/tests/ha/set_qos_2ms.sh new file mode 120000 index 00000000..62b2042a --- /dev/null +++ b/tests/ha/set_qos_2ms.sh @@ -0,0 +1 @@ +set_qos.sh \ No newline at end of file diff --git a/tests/ha/setup_set_qos.sh b/tests/ha/setup_set_qos.sh new file mode 100755 index 00000000..8c3cbfc3 --- /dev/null +++ b/tests/ha/setup_set_qos.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exit 0 diff --git a/tests/ha/setup_set_qos_2ms.sh b/tests/ha/setup_set_qos_2ms.sh new file mode 120000 index 00000000..f5a79935 --- /dev/null +++ b/tests/ha/setup_set_qos_2ms.sh @@ -0,0 +1 @@ +setup_set_qos.sh \ No newline at end of file diff --git a/tests/ha/start_up_set_qos_2ms.sh b/tests/ha/start_up_set_qos_2ms.sh new file mode 100755 index 00000000..cbd05642 --- /dev/null +++ b/tests/ha/start_up_set_qos_2ms.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -ex + +# Check if GITHUB_WORKSPACE is defined +if [ -n "$GITHUB_WORKSPACE" ]; then + test_dir="$GITHUB_WORKSPACE/tests/ha" +else + test_dir=$(dirname $0) +fi + +sed 's/^ *qos_timeslice_in_usecs.*$/qos_timeslice_in_usecs = 2000/' ceph-nvmeof.conf > /tmp/ceph-nvmeof.2ms.conf +export NVMEOF_CONFIG=/tmp/ceph-nvmeof.2ms.conf +$test_dir/start_up.sh +rm -f /tmp/ceph-nvmeof.2ms.conf