From f8f4b931b46f12a1081d960d6e05b01a9658dbd8 Mon Sep 17 00:00:00 2001 From: perdelt Date: Mon, 13 Jan 2025 06:45:31 +0100 Subject: [PATCH] YCSB: if we want to loop over pooling pods, we only allow 1 configuration of loading pods (number and target) --- ycsb.py | 121 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 65 insertions(+), 56 deletions(-) diff --git a/ycsb.py b/ycsb.py index de6af6bf..fbd16ed1 100644 --- a/ycsb.py +++ b/ycsb.py @@ -169,9 +169,13 @@ num_benchmarking_threads = experiment.get_parameter_as_list('num_benchmarking_threads') num_benchmarking_target_factors = experiment.get_parameter_as_list('num_benchmarking_target_factors') num_pooling_pods = experiment.get_parameter_as_list('num_pooling_pods') - num_pooling_pods = int(num_pooling_pods[0]) num_pooling_in = experiment.get_parameter_as_list('num_pooling_in') num_pooling_out = experiment.get_parameter_as_list('num_pooling_out') + # if we want to loop over pooling pods, we only allow 1 configuration of loading pods (number and target) + # this is because naming length is limited in k8s + if len(num_pooling_pods) > 0: + num_loading_pods = [num_loading_pods[0]] + num_loading_threads = [num_loading_threads[0]] # set node labes for components if aws: # set node labes for components @@ -259,62 +263,67 @@ num_pooling_in = [int(loading_threads)] if len(num_pooling_out) == 0: num_pooling_out = [int(loading_threads)] - for num_c_in in num_pooling_in: - for num_c_out in num_pooling_out: - name_format = 'pgb-{threads}-{pods}-{c_in}-{c_out}' - #name_format = 'PGBouncer-{threads}-{pods}-{target}-{c_in}-{c_out}' - config = configurations.ycsb(experiment=experiment, docker='PGBouncer', configuration=name_format.format(threads=loading_threads, pods=loading_pods, target=loading_target, c_in=num_c_in, c_out=num_c_out), alias='DBMS A') - config.path_experiment_docker = 'PostgreSQL' # take init scripts of PostgreSQL - config.sut_has_pool = True # in particular monitor pool component - config.sut_envs = { - 'DEFAULT_POOL_SIZE': int(num_c_out/num_pooling_pods), # max connections to PostgreSQL - 'MIN_POOL_SIZE': int(num_c_out/num_pooling_pods), # min connections to PostgreSQL - 'MAX_CLIENT_CONN': int(num_c_in/num_pooling_pods), # max connections to PGBouncer - } - config.set_resources( - replicas_pooling = num_pooling_pods, - ) - config.set_storage( - storageConfiguration = 'postgresql' + for num_c_pods in num_pooling_pods: + for num_c_in in num_pooling_in: + for num_c_out in num_pooling_out: + if len(num_pooling_pods) > 0: + name_format = 'pgb-{threads}-{pooling_pods}-{c_in}-{c_out}' + else: + name_format = 'pgb-{threads}-{loading_pods}-{c_in}-{c_out}' + # this is too long + #name_format = 'PGBouncer-{threads}-{pods}-{target}-{c_in}-{c_out}' + config = configurations.ycsb(experiment=experiment, docker='PGBouncer', configuration=name_format.format(threads=loading_threads, loading_pods=loading_pods, pooling_pods=num_c_pods, target=loading_target, c_in=num_c_in, c_out=num_c_out), alias='DBMS A') + config.path_experiment_docker = 'PostgreSQL' # take init scripts of PostgreSQL + config.sut_has_pool = True # in particular monitor pool component + config.sut_envs = { + 'DEFAULT_POOL_SIZE': int(num_c_out/num_pooling_pods), # max connections to PostgreSQL + 'MIN_POOL_SIZE': int(num_c_out/num_pooling_pods), # min connections to PostgreSQL + 'MAX_CLIENT_CONN': int(num_c_in/num_pooling_pods), # max connections to PGBouncer + } + config.set_resources( + replicas_pooling = num_pooling_pods, ) - config.set_loading_parameters( - PARALLEL = str(loading_pods), - SF = SF, - BEXHOMA_SYNCH_LOAD = 1, - YCSB_THREADCOUNT = loading_threads_per_pod, - YCSB_TARGET = loading_target_per_pod, - YCSB_STATUS = 1, - YCSB_WORKLOAD = args.workload, - YCSB_ROWS = ycsb_rows, - YCSB_OPERATIONS = ycsb_operations_per_pod, - YCSB_BATCHSIZE = batchsize, - ) - config.set_loading(parallel=loading_pods, num_pods=loading_pods) - executor_list = [] - for factor_benchmarking in num_benchmarking_target_factors:#range(1, 9):#range(1, 2):#range(1, 15): - benchmarking_target = target_base*factor_benchmarking#4*4096*t - for benchmarking_threads in num_benchmarking_threads: - for benchmarking_pods in num_benchmarking_pods:#[1,2]:#[1,8]:#range(2,5): - for num_executor in list_clients: - benchmarking_pods_scaled = num_executor*benchmarking_pods - benchmarking_threads_per_pod = int(benchmarking_threads/benchmarking_pods) - ycsb_operations_per_pod = int(ycsb_operations/benchmarking_pods_scaled) - benchmarking_target_per_pod = int(benchmarking_target/benchmarking_pods) - executor_list.append(benchmarking_pods_scaled) - config.add_benchmarking_parameters( - PARALLEL = str(benchmarking_pods_scaled), - SF = SF, - BEXHOMA_SYNCH_LOAD = 1, - YCSB_THREADCOUNT = benchmarking_threads_per_pod, - YCSB_TARGET = benchmarking_target_per_pod, - YCSB_STATUS = 1, - YCSB_WORKLOAD = args.workload, - YCSB_ROWS = ycsb_rows, - YCSB_OPERATIONS = ycsb_operations_per_pod, - YCSB_BATCHSIZE = batchsize, - ) - #print(executor_list) - config.add_benchmark_list(executor_list) + config.set_storage( + storageConfiguration = 'postgresql' + ) + config.set_loading_parameters( + PARALLEL = str(loading_pods), + SF = SF, + BEXHOMA_SYNCH_LOAD = 1, + YCSB_THREADCOUNT = loading_threads_per_pod, + YCSB_TARGET = loading_target_per_pod, + YCSB_STATUS = 1, + YCSB_WORKLOAD = args.workload, + YCSB_ROWS = ycsb_rows, + YCSB_OPERATIONS = ycsb_operations_per_pod, + YCSB_BATCHSIZE = batchsize, + ) + config.set_loading(parallel=loading_pods, num_pods=loading_pods) + executor_list = [] + for factor_benchmarking in num_benchmarking_target_factors:#range(1, 9):#range(1, 2):#range(1, 15): + benchmarking_target = target_base*factor_benchmarking#4*4096*t + for benchmarking_threads in num_benchmarking_threads: + for benchmarking_pods in num_benchmarking_pods:#[1,2]:#[1,8]:#range(2,5): + for num_executor in list_clients: + benchmarking_pods_scaled = num_executor*benchmarking_pods + benchmarking_threads_per_pod = int(benchmarking_threads/benchmarking_pods) + ycsb_operations_per_pod = int(ycsb_operations/benchmarking_pods_scaled) + benchmarking_target_per_pod = int(benchmarking_target/benchmarking_pods) + executor_list.append(benchmarking_pods_scaled) + config.add_benchmarking_parameters( + PARALLEL = str(benchmarking_pods_scaled), + SF = SF, + BEXHOMA_SYNCH_LOAD = 1, + YCSB_THREADCOUNT = benchmarking_threads_per_pod, + YCSB_TARGET = benchmarking_target_per_pod, + YCSB_STATUS = 1, + YCSB_WORKLOAD = args.workload, + YCSB_ROWS = ycsb_rows, + YCSB_OPERATIONS = ycsb_operations_per_pod, + YCSB_BATCHSIZE = batchsize, + ) + #print(executor_list) + config.add_benchmark_list(executor_list) if ("MySQL" in args.dbms or len(args.dbms) == 0): # MySQL name_format = 'MySQL-{threads}-{pods}-{target}'