diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..468ceea --- /dev/null +++ b/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: LLVM \ No newline at end of file diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml new file mode 100644 index 0000000..c371429 --- /dev/null +++ b/.github/workflows/ccpp.yml @@ -0,0 +1,15 @@ +name: C/C++ CI + +on: [push] + +jobs: + build-ubuntu: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: configure + run: mkdir build && cd build && cmake -DCMAKE_CXX_FLAGS="-Werror" .. + - name: build + run: cmake --build build \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50c5f79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +build +cmake-build-* +*~ +.idea/ +.vscode/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..4bf35f4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.2) +project(hiccups CXX) + +set(CMAKE_CXX_STANDARD 17) + +find_package(Threads) + +add_executable(hiccups src/hiccups.cpp) +target_compile_options(hiccups PRIVATE -Wall -Wextra -Wpedantic) +target_link_libraries(hiccups Threads::Threads) + +install(TARGETS hiccups DESTINATION bin/) \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0c1b0c9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2020 Erik Rigtorp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..ccb61e8 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +# hiccups + +[![License](https://img.shields.io/github/license/rigtorp/hiccups)](https://github.com/rigtorp/hiccups/blob/master/LICENSE) + +*hiccups* measures the system induced jitter ("hiccups") a CPU bound thread +experiences. + +It runs a thread on each processor core that loops for a fixed interval (by +default 5 seconds). Each loop iteration acquires a timestamp and if the +difference between the last two successive timestamps exceeds a threshold, the +thread is assumed to have been interrupted and the difference is recorded. At +the end of the run the number of interruptions, 99th percentile, 99.9th +percentile and maximum interruption per processor core is output. + +By default the threshold is calculated as 8 times the smallest difference +between two consecutive timestamps out of 10000 runs. Difference between two +successive timestamps exceeding the threshold is indicative of jitter introduced +by thread context switching, interrupt processing, TLB shootdowns etc. + +*hiccups* was inspired by [David Riddoch's](mailto:david@riddoch.org.uk) +[sysjitter](https://www.openonload.org/download/sysjitter/sysjitter-1.4.tgz). + +## Example + +Measure jitter on CPUs 0 through 3: +``` +$ taskset -c 0-3 hiccups | column -t -R 1,2,3,4,5,6 +cpu threshold_ns hiccups pct99_ns pct999_ns max_ns + 0 168 17110 83697 6590444 17010845 + 1 168 9929 169555 5787333 9517076 + 2 168 20728 73359 6008866 16008460 + 3 168 28336 1354 4870 17869 +``` + +In this example CPU 3 is a quiet CPU with low jitter as indicated by the 99.9th +percentile being much lower then the other CPUs. + +## Building & Installing + +*hiccups* requires [CMake](https://cmake.org/) 3.2 or higher and a C++17 +compiler. + +Building on Debian/Ubuntu: + +``` +sudo apt install cmake g++ +cd hiccups +mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release +make +``` + +Building on RHEL/CentOS: + +``` +sudo yum install cmake3 g++ +cd hiccups +mkdir build && cd build +cmake3 .. -DCMAKE_BUILD_TYPE=Release +make +``` + +Installing: + +``` +$ sudo make install +``` + +## About + +This project was created by [Erik Rigtorp](https://rigtorp.se) +<[erik@rigtorp.se](mailto:erik@rigtorp.se)>. diff --git a/src/hiccups.cpp b/src/hiccups.cpp new file mode 100644 index 0000000..ebcf1a4 --- /dev/null +++ b/src/hiccups.cpp @@ -0,0 +1,153 @@ +// © 2020 Erik Rigtorp +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + + auto runtime = std::chrono::seconds(5); + auto threshold = std::chrono::nanoseconds::max(); + size_t nsamples = runtime.count() * 1 << 16; + + int opt; + while ((opt = getopt(argc, argv, "r:s:t:")) != -1) { + switch (opt) { + case 'r': + runtime = std::chrono::seconds(std::stoul(optarg)); + break; + case 't': + threshold = std::chrono::nanoseconds(std::stoul(optarg)); + break; + case 's': + nsamples = std::stoul(optarg); + break; + default: + goto usage; + } + } + + if (optind != argc) { + usage: + std::cerr + << "hiccups 1.0.0 © 2020 Erik Rigtorp " + "https://github.com/rigtorp/hiccups\n" + "usage: hiccups [-r runtime_seconds] [-t threshold_nanoseconds] " + "[-s number_of_samples]\n"; + exit(1); + } + + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof(set), &set) == -1) { + perror("sched_getaffinity"); + exit(1); + } + + // enumerate available CPUs + std::vector cpus; + for (int i = 0; i < CPU_SETSIZE; ++i) { + if (CPU_ISSET(i, &set)) { + cpus.push_back(i); + } + } + + // calculate threshold as minimum timestamp delta * 8 + if (threshold == std::chrono::nanoseconds::max()) { + auto ts1 = std::chrono::steady_clock::now(); + for (int i = 0; i < 10000; ++i) { + auto ts2 = std::chrono::steady_clock::now(); + if (ts2 - ts1 < threshold) { + threshold = ts2 - ts1; + } + ts1 = ts2; + } + threshold *= 8; + } + + // reserve memory for samples + std::map> samples; + for (int cpu : cpus) { + samples[cpu].reserve(nsamples); + } + + // avoid page faults and TLB shootdowns when saving samples + if (mlockall(MCL_CURRENT | MCL_FUTURE) == -1) { + perror("mlockall"); + std::cerr << "WARNING failed to lock memory, increase RLIMIT_MEMLOCK " + "or run with CAP_IPC_LOC capability.\n"; + } + + const auto deadline = std::chrono::steady_clock::now() + runtime; + std::atomic active_threads = {0}; + + auto func = [&](int cpu) { + // pin current thread to assigned CPU + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + if (sched_setaffinity(0, sizeof(set), &set) == -1) { + perror("sched_setaffinity"); + exit(1); + } + + auto &s = samples[cpu]; + active_threads.fetch_add(1, std::memory_order_release); + + // wait for all threads to be ready + while (active_threads.load(std::memory_order_relaxed) != cpus.size()) + ; + + // run jitter measurement loop + auto ts1 = std::chrono::steady_clock::now(); + while (ts1 < deadline) { + auto ts2 = std::chrono::steady_clock::now(); + if (ts2 - ts1 < threshold) { + ts1 = ts2; + continue; + } + if (s.size() == s.capacity()) { + std::cerr << "WARNING preallocated sample space exceeded, increase " + "threshold or number of samples.\n"; + } + s.push_back(ts2 - ts1); + // ts1 = ts2; + ts1 = std::chrono::steady_clock::now(); + } + }; + + // start measurements threads + std::vector threads; + for (auto it = ++cpus.begin(); it != cpus.end(); ++it) { + threads.emplace_back([&, cpu = *it] { func(cpu); }); + } + func(cpus.front()); + + // wait for all threads to finish + for (auto &t : threads) { + t.join(); + } + + // print statistics + std::cout << "cpu threshold_ns hiccups pct99_ns pct999_ns max_ns\n"; + for (auto &[cpu, s] : samples) { + std::sort(s.begin(), s.end()); + std::cout << cpu << " " << threshold.count() << " " << s.size() << " " + << s[s.size() * 0.99].count() << " " + << s[s.size() * 0.999].count() << " " << s.back().count() + << std::endl; + } + + return 0; +}