From a2f0b0244f940400a959e37f2283b44d2bfc9ec3 Mon Sep 17 00:00:00 2001 From: Seth Zegelstein Date: Wed, 13 Dec 2023 00:30:31 +0000 Subject: [PATCH] Collect timing data for EFA Provider 1. Gather timing data from the following code paths a. fi_senddata -> rdma-core, rdma-core, rdma-core -> caller b. fi_writedata -> rdma-core, rdma-core, rdma-core -> caller c. Posting a recv buffer d. An empty progress e. A fruitful progress d. How many messages were progressed in each progress Modify fi_rdm_pingpong, and fi_rma_pingpong to print these results to a file. Create a python script to parse the data, and generate statistics. Signed-off-by: Seth Zegelstein --- fabtests/benchmarks/benchmark_shared.c | 83 +- fabtests/common/shared.c | 4 +- generate_statistics.py | 243 ++ include/rdma/fi_endpoint.h | 53 +- include/rdma/fi_eq.h | 40 +- include/rdma/fi_rma.h | 18 +- prov/efa/src/rdm/efa_rdm_cq.c | 10 +- prov/efa/src/rdm/efa_rdm_ep_fiops.c | 9 + prov/efa/src/rdm/efa_rdm_pke.c | 41 + seth_test/.gitignore | 1 + seth_test/compile.sh | 1 + seth_test/result_stat.py | 26 + seth_test/results.txt | 5000 ++++++++++++++++++++++++ seth_test/test.c | 46 + 14 files changed, 5558 insertions(+), 17 deletions(-) create mode 100644 generate_statistics.py create mode 100644 seth_test/.gitignore create mode 100755 seth_test/compile.sh create mode 100644 seth_test/result_stat.py create mode 100644 seth_test/results.txt create mode 100644 seth_test/test.c diff --git a/fabtests/benchmarks/benchmark_shared.c b/fabtests/benchmarks/benchmark_shared.c index 9a2d0c14eaf..c0f0ac78bd7 100644 --- a/fabtests/benchmarks/benchmark_shared.c +++ b/fabtests/benchmarks/benchmark_shared.c @@ -134,11 +134,44 @@ int pingpong(void) } ft_stop(); - if (opts.machr) - show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2, - opts.argc, opts.argv); - else - show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 2); + // if (opts.machr) + // show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2, + // opts.argc, opts.argv); + // else + // show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 2); + + FILE* fptr; + + if (opts.dst_addr) { + fptr = fopen("/home/ec2-user/libfabric/client_fi_senddata_output.txt", "w"); + } else { + fptr = fopen("/home/ec2-user/libfabric/server_fi_senddata_output.txt", "w"); + } + + for (int i = 0; i < ep->iterations; i++) { + fprintf(fptr, " libfabric to rdma_core (ns): %ld, rdma_core (ns): %ld, rdma-core to user (ns): %ld\n", + ep->libfabric_start_to_rdma_time[i], + ep->rdma_core_time[i], + ep->libfabric_from_rdma_to_end_time[i]); + } + + for (int i = 0; i < ep->iterations; i++) { + fprintf(fptr, " post recv buff (ns): %ld\n", ep->post_recv_buf_time[i]); + } + + for (int i = 0; i < rxcq->iterations; i++) { + fprintf(fptr, " empty cq progress (ns): %ld\n", rxcq->empty_progress[i]); + } + + for (int i = 0; i < rxcq->iterations; i++) { + fprintf(fptr, " fruitful cq progress (ns): %ld\n", rxcq->fruitful_progress[i]); + } + + for (int i = 0; i < rxcq->iterations; i++) { + fprintf(fptr, " num completion events in fruitful progress: %d\n", rxcq->fruitful_progress_num_events[i]); + } + + fclose(fptr); return 0; } @@ -219,11 +252,41 @@ int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote) } ft_stop(); - if (opts.machr) - show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2, - opts.argc, opts.argv); - else - show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 2); + // if (opts.machr) + // show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2, + // opts.argc, opts.argv); + // else + // show_perf(NULL, opts.transfer_size, opts.iterations, &start, &end, 2); + + + FILE* fptr; + + if (opts.dst_addr) { + fptr = fopen("/home/ec2-user/libfabric/client_fi_writedata_output.txt", "w"); + } else { + fptr = fopen("/home/ec2-user/libfabric/server_fi_writedata_output.txt", "w"); + } + + for (int i = 0; i < ep->iterations; i++) { + fprintf(fptr, " libfabric to rdma_core (ns): %ld, rdma_core (ns): %ld, rdma-core to user (ns): %ld\n", + ep->libfabric_start_to_rdma_time[i], + ep->rdma_core_time[i], + ep->libfabric_from_rdma_to_end_time[i]); + } + + for (int i = 0; i < rxcq->iterations; i++) { + fprintf(fptr, " empty cq progress (ns): %ld\n", rxcq->empty_progress[i]); + } + + for (int i = 0; i < rxcq->iterations; i++) { + fprintf(fptr, " fruitful cq progress (ns): %ld\n", rxcq->fruitful_progress[i]); + } + + for (int i = 0; i < rxcq->iterations; i++) { + fprintf(fptr, " num completion events in fruitful progress: %d\n", rxcq->fruitful_progress_num_events[i]); + } + + fclose(fptr); return 0; } diff --git a/fabtests/common/shared.c b/fabtests/common/shared.c index 0b34d1f64be..624c1a8c40d 100644 --- a/fabtests/common/shared.c +++ b/fabtests/common/shared.c @@ -2137,9 +2137,9 @@ ssize_t ft_post_tx_buf(struct fid_ep *ep, fi_addr_t fi_addr, size_t size, op_mr_desc, data, fi_addr, ctx); } else { - FT_POST(fi_send, ft_progress, txcq, tx_seq, + FT_POST(fi_senddata, ft_progress, txcq, tx_seq, &tx_cq_cntr, "transmit", ep, op_buf, size, - op_mr_desc, fi_addr, ctx); + op_mr_desc, data, fi_addr, ctx); } } return 0; diff --git a/generate_statistics.py b/generate_statistics.py new file mode 100644 index 00000000000..e53ea16c576 --- /dev/null +++ b/generate_statistics.py @@ -0,0 +1,243 @@ +#!/bin/python3 +# source ~/env/bin/activate to get PF's env +# This script needs to be kicked off on the server IP's host +from matplotlib import pyplot as plt +from PortaFiducia.tests.utils import run_command +import subprocess +from time import sleep + + +# Configurable server IP (in case we change instances) +sever_ip = "172.31.35.142" +client_ip = "172.31.35.207" +iterations = 2000000 +warmup_iterations = 100 + +# Step 1: Compile Libfabric ~20s +print("Compile libfabric and fabtests:") +autogen = "./autogen.sh" +configure = "./configure --prefix=$HOME/libfabric/install --disable-verbs --disable-psm3 --disable-opx --disable-usnic --disable-rstream --enable-efa --with-cuda=/usr/local/cuda --enable-cuda-dlopen --with-gdrcopy --enable-gdrcopy-dlopen" +clean = "rm -rf $HOME/libfabric/install && make clean" +install = "make -j install" +run_command(f"cd $HOME/libfabric; {autogen} && {configure} && {clean} && {install}") + +# Step 2: Compile Fabtests ~10s +configure = "./configure --prefix=$HOME/libfabric/fabtests/install --with-libfabric=$HOME/libfabric/install --with-cuda=/usr/local/cuda" +clean = "rm -rf $HOME/libfabric/fabtests/install && make clean" +run_command(f"cd $HOME/libfabric/fabtests; {autogen} && {configure} && {clean} && {install}") + + +# Step 3: Run fi_rdm_pingpong +print("\n\n Running Eager Send Data Tests \n\n") + +executable = "/home/ec2-user/libfabric/fabtests/install/bin/fi_rdm_pingpong" +server_args = f" --pin-core 1 -i 0 -p efa -D cuda -w {warmup_iterations} -I {iterations} -S 1024 -E " +server_command = executable + server_args +client_command = "ssh " + client_ip + " " + executable + server_args + sever_ip + +print(f"Running Server Command: {server_command}") +print(f"Running Client Command: {client_command}") +server_process = subprocess.Popen(server_command, stdout=subprocess.PIPE, stderr=None, + shell=True, universal_newlines=True) +sleep(5) + +client_process = subprocess.Popen(client_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True, universal_newlines=True) + +sleep(5) + +try: + _, _ = server_process.communicate(timeout=720) + sleep(5) + _, _ = client_process.communicate(timeout=720) +except subprocess.TimeoutExpired: + server_process.terminate() + client_process.terminate() + raise RuntimeError("Client/Server Timed Out") + + +def parse_rdm_pingpong_output(filename): + libfabric_to_rdma_core = [] + rdma_core = [] + rdma_core_to_libfabric = [] + post_recv_buff = [] + empty_cq_progress = [] + fruitful_cq_progress = [] + fruitful_cq_progress_num_completions = [] + + with open(filename, "r") as f: + lines = f.readlines() + + for line in lines: + if line.startswith(" libfabric to rdma_core"): + data = line.strip().split(" ") + libfabric_to_rdma_core.append(int(data[5].strip(","))) + rdma_core.append(int(data[8].strip(","))) + rdma_core_to_libfabric.append(int(data[-1])) + + elif line.startswith(" post recv buff"): + post_recv_buff.append(int(line.strip().split(" ")[-1])) + + elif line.startswith(" empty cq progress"): + empty_cq_progress.append(int(line.strip().split(" ")[-1])) + + elif line.startswith(" fruitful cq progress"): + fruitful_cq_progress.append(int(line.strip().split(" ")[-1])) + + elif line.startswith(" num completion events in fruitful progress"): + fruitful_cq_progress_num_completions.append(int(line.strip().split(" ")[-1])) + + return libfabric_to_rdma_core, rdma_core, rdma_core_to_libfabric, post_recv_buff, empty_cq_progress, fruitful_cq_progress, fruitful_cq_progress_num_completions + +(server_libfabric_to_rdma_core, server_rdma_core, server_rdma_core_to_libfabric, + server_post_recv_buff, server_empty_cq_progress, server_fruitful_cq_progress, + server_fruitful_cq_progress_num_completions) = parse_rdm_pingpong_output("/home/ec2-user/libfabric/server_fi_senddata_output.txt") +(client_libfabric_to_rdma_core, client_rdma_core, client_rdma_core_to_libfabric, + client_post_recv_buff, client_empty_cq_progress, client_fruitful_cq_progress, + client_fruitful_cq_progress_num_completions) = parse_rdm_pingpong_output("/home/ec2-user/libfabric/client_fi_senddata_output.txt") + +print("\n") + +# Step 3: Get/print statistics + +def get_and_print_statistics(vector, name): + import statistics + avg = statistics.mean(vector) + num_outliers = sum(i > avg * 5 for i in vector) + # TODO Remove this logic + # Remove outliers from vector + vector = [i for i in vector if i <= avg * 5] + + # get a new average + avg = statistics.mean(vector) + pstd = statistics.pstdev(vector) + minimum = min(vector) + maximum = max(vector) + median = sorted(vector)[len(vector) // 2] + num_samples = len(vector) + unique = len(set(vector)) + + unit = "ns" + if "fruitful cq progress num completion" in name: + unit = None + + print(f"{name} average {unit}: {avg}, pstd: {pstd}, min: {minimum}, max: {maximum}, median: {median}, num_samples: {num_samples}, unique: {unique}, num_outliers removed (5x mean): {num_outliers}") + + if "fruitful cq progress num completion" in name: + # num completions is usually always 1, no hist + return + + plt.figure() + plt.hist(vector, round(unique/5), range=[minimum-10, avg*2], align='mid') + plt.title(name) + plt.xlabel("Time (ns)") + plt.ylabel("Frequency") + plt.savefig(f"/home/ec2-user/libfabric/plots/{name}.png") + plt.close() + + +print("Server's fi_senddata stats:") +get_and_print_statistics(server_libfabric_to_rdma_core, "Server fi_senddata libfabric to rdma_core") +get_and_print_statistics(server_rdma_core, "Server fi_senddata rdma_core") +get_and_print_statistics(server_rdma_core_to_libfabric, "Server fi_senddata rdma_core to libfabric") +get_and_print_statistics(server_post_recv_buff, "Server fi_senddata post recv buff") +get_and_print_statistics(server_empty_cq_progress, "Server fi_senddata empty cq progress") +get_and_print_statistics(server_fruitful_cq_progress, "Server fi_senddata fruitful cq progress") +get_and_print_statistics(server_fruitful_cq_progress_num_completions, "Server fi_senddata fruitful cq progress num completion") + +print("\n") + +print("Client's fi_senddata stats:") +get_and_print_statistics(client_libfabric_to_rdma_core, "Client fi_senddata libfabric to rdma_core") +get_and_print_statistics(client_rdma_core, "Client fi_senddata rdma_core") +get_and_print_statistics(client_rdma_core_to_libfabric, "Client fi_senddata rdma_core to libfabric") +get_and_print_statistics(client_post_recv_buff, "Client fi_senddata post recv buff") +get_and_print_statistics(client_empty_cq_progress, "Client fi_senddata empty cq progress") +get_and_print_statistics(client_fruitful_cq_progress, "Client fi_senddata fruitful cq progress") +get_and_print_statistics(client_fruitful_cq_progress_num_completions, "Client fi_senddata fruitful cq progress num completion") + + +# Step 4: Run fi_rdm_rma_pingpong +print("\n\n Running RMA Write Data Tests \n\n") +executable = "/home/ec2-user/libfabric/fabtests/install/bin/fi_rma_pingpong" +server_args = f" --pin-core 1 -i 0 -o writedata -p efa -D cuda -w {warmup_iterations} -I {iterations} -S 131072 -E " +server_command = executable + server_args +client_command = "ssh " + client_ip + " " + executable + server_args + sever_ip + +print(f"Running Server Command: {server_command}") +print(f"Running Client Command: {client_command}") +server_process = subprocess.Popen(server_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True, universal_newlines=True) + +sleep(5) + +client_process = subprocess.Popen(client_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True, universal_newlines=True) + +sleep(5) + +try: + server_output, _ = server_process.communicate(timeout=720) + sleep(5) + client_output, _ = client_process.communicate(timeout=720) +except subprocess.TimeoutExpired: + server_process.terminate() + client_process.terminate() + raise RuntimeError("Client/Server Timed Out") + +def parse_rma_pingpong_output(filename): + libfabric_to_rdma_core = [] + rdma_core = [] + rdma_core_to_libfabric = [] + empty_cq_progress = [] + fruitful_cq_progress = [] + fruitful_cq_progress_num_completions = [] + + with open(filename, "r") as f: + lines = f.readlines() + + for line in lines: + if line.startswith(" libfabric to rdma_core"): + data = line.strip().split(" ") + libfabric_to_rdma_core.append(int(data[5].strip(","))) + rdma_core.append(int(data[8].strip(","))) + rdma_core_to_libfabric.append(int(data[-1])) + + elif line.startswith(" empty cq progress"): + empty_cq_progress.append(int(line.strip().split(" ")[-1])) + + elif line.startswith(" fruitful cq progress"): + fruitful_cq_progress.append(int(line.strip().split(" ")[-1])) + + elif line.startswith(" num completion events in fruitful progress"): + fruitful_cq_progress_num_completions.append(int(line.strip().split(" ")[-1])) + + return libfabric_to_rdma_core, rdma_core, rdma_core_to_libfabric, empty_cq_progress, fruitful_cq_progress, fruitful_cq_progress_num_completions + +(server_libfabric_to_rdma_core, server_rdma_core, server_rdma_core_to_libfabric, + server_empty_cq_progress, server_fruitful_cq_progress, + server_fruitful_cq_progress_num_completions) = parse_rma_pingpong_output("/home/ec2-user/libfabric/server_fi_writedata_output.txt") +(client_libfabric_to_rdma_core, client_rdma_core, client_rdma_core_to_libfabric, + client_empty_cq_progress, client_fruitful_cq_progress, + client_fruitful_cq_progress_num_completions) = parse_rma_pingpong_output("/home/ec2-user/libfabric/client_fi_writedata_output.txt") + + +print("\n") + +# Setp 5: Get/print statistics +print("Server's fi_writedata stats:") +get_and_print_statistics(server_libfabric_to_rdma_core, "Server fi_writedata libfabric to rdma_core") +get_and_print_statistics(server_rdma_core, "Server fi_writedata rdma_core") +get_and_print_statistics(server_rdma_core_to_libfabric, "Server fi_writedata rdma_core to libfabric") +get_and_print_statistics(server_empty_cq_progress, "Server fi_writedata empty cq progress") +get_and_print_statistics(server_fruitful_cq_progress, "Server fi_writedata fruitful cq progress") +get_and_print_statistics(server_fruitful_cq_progress_num_completions, "Server fi_writedata fruitful cq progress num completion") + +print("\n") +print("Client's fi_writedata stats:") +get_and_print_statistics(client_libfabric_to_rdma_core, "Client fi_writedata libfabric to rdma_core") +get_and_print_statistics(client_rdma_core, "Client fi_writedata rdma_core") +get_and_print_statistics(client_rdma_core_to_libfabric, "Client fi_writedata rdma_core to libfabric") +get_and_print_statistics(client_empty_cq_progress, "Client fi_writedata empty cq progress") +get_and_print_statistics(client_fruitful_cq_progress, "Client fi_writedata fruitful cq progress") +get_and_print_statistics(client_fruitful_cq_progress_num_completions, "Client fi_writedata fruitful cq progress num completion") diff --git a/include/rdma/fi_endpoint.h b/include/rdma/fi_endpoint.h index cf0611b1bf2..d6ef9ca9171 100644 --- a/include/rdma/fi_endpoint.h +++ b/include/rdma/fi_endpoint.h @@ -41,6 +41,14 @@ extern "C" { #endif +static inline void timespec_diff(struct timespec *start, struct timespec *end, struct timespec *result) { + result->tv_sec = end->tv_sec - start->tv_sec; + result->tv_nsec = end->tv_nsec - start->tv_nsec; + if (result->tv_nsec < 0) { + --result->tv_sec; + result->tv_nsec += 1000000000L; + } +} struct fi_msg { const struct iovec *msg_iov; @@ -145,6 +153,16 @@ struct fid_ep { struct fi_ops_tagged *tagged; struct fi_ops_atomic *atomic; struct fi_ops_collective *collective; + int warmup_iterations; + int iterations; + int msg_count; + int rma_count; + int recv_count; + long *post_recv_buf_time; + long *rdma_core_time; + long *libfabric_start_to_rdma_time; + long *libfabric_from_rdma_to_end_time; + struct timespec libfabric_start; }; struct fid_pep { @@ -294,7 +312,21 @@ static inline ssize_t fi_recv(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context) { - return ep->msg->recv(ep, buf, len, desc, src_addr, context); + ssize_t rv; + struct timespec start, end, result; + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + + rv = ep->msg->recv(ep, buf, len, desc, src_addr, context); + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff(&start, &end, &result); + if (ep->recv_count < ep->iterations + ep->warmup_iterations) + if (ep->recv_count >= ep->warmup_iterations) + ep->post_recv_buf_time[ep->recv_count - ep->warmup_iterations] = result.tv_nsec; + + ep->recv_count++; + + return rv; } static inline ssize_t @@ -340,7 +372,24 @@ static inline ssize_t fi_senddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, uint64_t data, fi_addr_t dest_addr, void *context) { - return ep->msg->senddata(ep, buf, len, desc, data, dest_addr, context); + ssize_t rv; + struct timespec end, result; + + clock_gettime(CLOCK_MONOTONIC_RAW, &ep->libfabric_start); + + rv = ep->msg->senddata(ep, buf, len, desc, data, dest_addr, context); + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + // ep->libfabric_start gets updated in efa_rdm_pke.c to time rdma-core -> user section + timespec_diff(&ep->libfabric_start, &end, &result); + if (rv != -FI_EAGAIN) { + if (ep->msg_count < ep->iterations + ep->warmup_iterations) + if (ep->msg_count >= ep->warmup_iterations) + ep->libfabric_from_rdma_to_end_time[ep->msg_count - ep->warmup_iterations] = result.tv_nsec; + + ep->msg_count++; + } + return rv; } static inline ssize_t diff --git a/include/rdma/fi_eq.h b/include/rdma/fi_eq.h index 6664ed48dcf..171cf7b5f84 100644 --- a/include/rdma/fi_eq.h +++ b/include/rdma/fi_eq.h @@ -276,6 +276,13 @@ struct fi_ops_cq { struct fid_cq { struct fid fid; struct fi_ops_cq *ops; + int iterations; + int warmup_iterations; + int count_empty_progress; + int count_fruitful_progress; + int *fruitful_progress_num_events; + long *fruitful_progress; + long *empty_progress; }; @@ -389,10 +396,41 @@ fi_eq_strerror(struct fid_eq *eq, int prov_errno, const void *err_data, return eq->ops->strerror(eq, prov_errno, err_data, buf, len); } +static inline void timespec_diff_cq(struct timespec *start, struct timespec *end, struct timespec *result) { + result->tv_sec = end->tv_sec - start->tv_sec; + result->tv_nsec = end->tv_nsec - start->tv_nsec; + if (result->tv_nsec < 0) { + --result->tv_sec; + result->tv_nsec += 1000000000L; + } +} static inline ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count) { - return cq->ops->read(cq, buf, count); + ssize_t rv; + struct timespec start, end, result; + + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + + rv = cq->ops->read(cq, buf, count); + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff_cq(&start, &end, &result); + + if (rv == -FI_EAGAIN && cq->count_empty_progress < cq->iterations + cq->warmup_iterations) { + if (cq->count_empty_progress >= cq->warmup_iterations) + cq->empty_progress[cq->count_empty_progress - cq->warmup_iterations] = result.tv_nsec; + cq->count_empty_progress++; + + } else if (rv > 0 && cq->count_fruitful_progress < cq->iterations + cq->warmup_iterations) { + if (cq->count_fruitful_progress >= cq->warmup_iterations) { + cq->fruitful_progress[cq->count_fruitful_progress - cq->warmup_iterations] = result.tv_nsec; + cq->fruitful_progress_num_events[cq->count_fruitful_progress - cq->warmup_iterations] = rv; + } + cq->count_fruitful_progress++; + } + + return rv; } static inline ssize_t diff --git a/include/rdma/fi_rma.h b/include/rdma/fi_rma.h index 003c10fd9cf..2375041914a 100644 --- a/include/rdma/fi_rma.h +++ b/include/rdma/fi_rma.h @@ -148,8 +148,24 @@ fi_writedata(struct fid_ep *ep, const void *buf, size_t len, void *desc, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { - return ep->rma->writedata(ep, buf, len, desc,data, dest_addr, + ssize_t rv; + struct timespec end, result; + ep->msg_count = ep->iterations + 1; // Make sure we don't get any senddata overriding our results + clock_gettime(CLOCK_MONOTONIC_RAW, &ep->libfabric_start); + + rv = ep->rma->writedata(ep, buf, len, desc,data, dest_addr, addr, key, context); + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + // ep->libfabric_start gets updated in efa_rdm_pke.c to time rdma-core -> user section + timespec_diff(&ep->libfabric_start, &end, &result); + if (ep->rma_count < ep->iterations + ep->warmup_iterations) + if (ep->rma_count >= ep->warmup_iterations) + ep->libfabric_from_rdma_to_end_time[ep->rma_count - ep->warmup_iterations] = result.tv_nsec; + + ep->rma_count++; + + return rv; } static inline ssize_t diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c index 5578d9d2ab8..893a967aaf5 100644 --- a/prov/efa/src/rdm/efa_rdm_cq.c +++ b/prov/efa/src/rdm/efa_rdm_cq.c @@ -97,7 +97,7 @@ static ssize_t efa_rdm_cq_readfrom(struct fid_cq *cq_fid, void *buf, size_t coun if (cq->shm_cq) { fi_cq_read(cq->shm_cq, NULL, 0); - /* + /* * fi_cq_read(cq->shm_cq, NULL, 0) will progress shm ep and write * completion to efa. Use ofi_cq_read_entries to get the number of * shm completions without progressing efa ep again. @@ -171,6 +171,14 @@ int efa_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, (*cq_fid)->fid.ops = &efa_rdm_cq_fi_ops; (*cq_fid)->ops = &efa_rdm_cq_ops; + (*cq_fid)->iterations = 2000000; + (*cq_fid)->warmup_iterations = 100; + (*cq_fid)->count_empty_progress = 0; + (*cq_fid)->count_fruitful_progress = 0; + (*cq_fid)->fruitful_progress = malloc(sizeof(long) * (*cq_fid)->iterations); + (*cq_fid)->fruitful_progress_num_events = malloc(sizeof(int) * (*cq_fid)->iterations); + (*cq_fid)->empty_progress = malloc(sizeof(long) * (*cq_fid)->iterations); + /* open shm cq as peer cq */ if (efa_domain->shm_domain) { memcpy(&shm_cq_attr, attr, sizeof(*attr)); diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c index 0ae97d82913..b6746267077 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_fiops.c +++ b/prov/efa/src/rdm/efa_rdm_ep_fiops.c @@ -556,6 +556,15 @@ int efa_rdm_ep_open(struct fid_domain *domain, struct fi_info *info, (*ep)->fid.ops = &efa_rdm_ep_base_ops; (*ep)->ops = &efa_rdm_ep_ep_ops; (*ep)->cm = &efa_rdm_ep_cm_ops; + (*ep)->msg_count = 0; + (*ep)->rma_count = 0; + (*ep)->recv_count = 0; + (*ep)->warmup_iterations = 100; + (*ep)->iterations = 2000000; + (*ep)->post_recv_buf_time = malloc(sizeof(long) * (*ep)->iterations); + (*ep)->rdma_core_time = malloc(sizeof(long) * (*ep)->iterations); + (*ep)->libfabric_start_to_rdma_time = malloc(sizeof(long) * (*ep)->iterations); + (*ep)->libfabric_from_rdma_to_end_time = malloc(sizeof(long) * (*ep)->iterations); return 0; err_close_core_cq: diff --git a/prov/efa/src/rdm/efa_rdm_pke.c b/prov/efa/src/rdm/efa_rdm_pke.c index aac99ef11de..915546788c3 100644 --- a/prov/efa/src/rdm/efa_rdm_pke.c +++ b/prov/efa/src/rdm/efa_rdm_pke.c @@ -394,9 +394,12 @@ ssize_t efa_rdm_pke_sendv(struct efa_rdm_pke **pkt_entry_vec, struct ibv_sge sg_list[2]; /* efa device support up to 2 iov */ struct ibv_data_buf inline_data_list[2]; int ret, pkt_idx, iov_cnt; + struct fid_ep *fid_ep; + struct timespec start, end, result; assert(pkt_entry_cnt); ep = pkt_entry_vec[0]->ep; + fid_ep = &ep->base_ep.util_ep.ep_fid; assert(ep); assert(pkt_entry_vec[0]->ope); @@ -409,6 +412,15 @@ ssize_t efa_rdm_pke_sendv(struct efa_rdm_pke **pkt_entry_vec, assert(conn && conn->ep_addr); qp = ep->base_ep.qp; + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff(&fid_ep->libfabric_start, &end, &result); + if (fid_ep->msg_count < fid_ep->iterations + fid_ep->warmup_iterations) + if (fid_ep->msg_count >= fid_ep->warmup_iterations) + fid_ep->libfabric_start_to_rdma_time[fid_ep->msg_count - fid_ep->warmup_iterations] = result.tv_nsec; + + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + ibv_wr_start(qp->ibv_qp_ex); for (pkt_idx = 0; pkt_idx < pkt_entry_cnt; ++pkt_idx) { pkt_entry = pkt_entry_vec[pkt_idx]; @@ -460,6 +472,15 @@ ssize_t efa_rdm_pke_sendv(struct efa_rdm_pke **pkt_entry_vec, } ret = ibv_wr_complete(qp->ibv_qp_ex); + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff(&start, &end, &result); + if (fid_ep->msg_count < fid_ep->iterations + fid_ep->warmup_iterations) + if (fid_ep->msg_count >= fid_ep->warmup_iterations) + fid_ep->rdma_core_time[fid_ep->msg_count - fid_ep->warmup_iterations] = result.tv_nsec; + + clock_gettime(CLOCK_MONOTONIC_RAW, &fid_ep->libfabric_start); + if (OFI_UNLIKELY(ret)) { return ret; } @@ -560,8 +581,11 @@ int efa_rdm_pke_write(struct efa_rdm_pke *pkt_entry) uint64_t remote_buf; size_t remote_key; int err = 0; + struct fid_ep *fid_ep; + struct timespec start, end, result; ep = pkt_entry->ep; + fid_ep = &ep->base_ep.util_ep.ep_fid; assert(ep); txe = pkt_entry->ope; @@ -579,6 +603,15 @@ int efa_rdm_pke_write(struct efa_rdm_pke *pkt_entry) pkt_entry->flags |= EFA_RDM_PKE_LOCAL_WRITE; qp = ep->base_ep.qp; + + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff(&fid_ep->libfabric_start, &end, &result); + if (fid_ep->rma_count < fid_ep->iterations + fid_ep->warmup_iterations) + if (fid_ep->rma_count >= fid_ep->warmup_iterations) + fid_ep->libfabric_start_to_rdma_time[fid_ep->rma_count - fid_ep->warmup_iterations] = result.tv_nsec; + + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + ibv_wr_start(qp->ibv_qp_ex); qp->ibv_qp_ex->wr_id = (uintptr_t)pkt_entry; @@ -612,6 +645,14 @@ int efa_rdm_pke_write(struct efa_rdm_pke *pkt_entry) err = ibv_wr_complete(qp->ibv_qp_ex); + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff(&start, &end, &result); + if (fid_ep->rma_count < fid_ep->iterations + fid_ep->warmup_iterations) + if (fid_ep->rma_count >= fid_ep->warmup_iterations) + fid_ep->rdma_core_time[fid_ep->rma_count - fid_ep->warmup_iterations] = result.tv_nsec; + + clock_gettime(CLOCK_MONOTONIC_RAW, &fid_ep->libfabric_start); + if (OFI_UNLIKELY(err)) return err; diff --git a/seth_test/.gitignore b/seth_test/.gitignore new file mode 100644 index 00000000000..9daeafb9864 --- /dev/null +++ b/seth_test/.gitignore @@ -0,0 +1 @@ +test diff --git a/seth_test/compile.sh b/seth_test/compile.sh new file mode 100755 index 00000000000..294e6e7620b --- /dev/null +++ b/seth_test/compile.sh @@ -0,0 +1 @@ +gcc -Wall -Werror -Wextra -pedantic test.c -o test -O3 \ No newline at end of file diff --git a/seth_test/result_stat.py b/seth_test/result_stat.py new file mode 100644 index 00000000000..02585ce24dd --- /dev/null +++ b/seth_test/result_stat.py @@ -0,0 +1,26 @@ +#!/bin/python3 +import statistics + +with open("results.txt", "r") as f: + lines = f.readlines() + +vector = [int(line.strip()) for line in lines] + +avg = statistics.mean(vector) +pstd = statistics.pstdev(vector) +minimum = min(vector) +maximum = max(vector) +median = sorted(vector)[len(vector) // 2] +num_samples = len(vector) +unique = len(set(vector)) +num_outliers = sum(i > avg * 2 for i in vector) + + +# remove outliers from vector which are more than 5x the avg +vector = [i for i in vector if i <= avg * 5] + + +print(f"average: {avg}, pstd: {pstd}, min: {minimum}, max: {maximum}, median: {median}, num_samples: {num_samples}, unique: {unique}, num_outliers: {num_outliers}") +print("value: number of occurrences") +for i in sorted(set(vector)): + print(f"{i}: {vector.count(i)}") diff --git a/seth_test/results.txt b/seth_test/results.txt new file mode 100644 index 00000000000..b1bbf9c54f0 --- /dev/null +++ b/seth_test/results.txt @@ -0,0 +1,5000 @@ +370 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +20 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +30 +20 +20 +20 +30 +30 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +20 +20 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 +30 +30 +20 +20 +20 +20 +30 diff --git a/seth_test/test.c b/seth_test/test.c new file mode 100644 index 00000000000..d46b839a233 --- /dev/null +++ b/seth_test/test.c @@ -0,0 +1,46 @@ +#include +#include +#include + + +static inline void timespec_diff(struct timespec *start, struct timespec *end, struct timespec *result) { + result->tv_sec = end->tv_sec - start->tv_sec; + result->tv_nsec = end->tv_nsec - start->tv_nsec; + if (result->tv_nsec < 0) { + --result->tv_sec; + result->tv_nsec += 1000000000L; + } +} + + +int main() { + struct timespec start, end, result; + // int iterations = 2000000; + int iterations = 5000; + int *result_vec = malloc(sizeof(int) * iterations); + + // // Warmup the cache + // for (int i = 0; i < 1000; i++) { + // clock_gettime(CLOCK_MONOTONIC_RAW, &start); + // clock_gettime(CLOCK_MONOTONIC_RAW, &end); + // timespec_diff(&start, &end, &result); + // } + + // Get results + for (int i = 0; i < iterations; i++) { + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + clock_gettime(CLOCK_MONOTONIC_RAW, &end); + timespec_diff(&start, &end, &result); + result_vec[i] = result.tv_nsec; + } + + + FILE* fptr; + fptr = fopen("results.txt", "w"); + for (int i = 0; i < iterations; i++) { + fprintf(fptr, "%d\n", result_vec[i]); + } + fclose(fptr); + + return 0; +} \ No newline at end of file