From ccaf034496bdac3c5f138186ecd350b612bfb13f Mon Sep 17 00:00:00 2001 From: wander Date: Fri, 24 May 2024 15:34:03 +0300 Subject: [PATCH 01/10] Added initial implementation --- flux-foundation/CMakeLists.txt | 10 +- flux-foundation/flux/foundation/dummy.cpp | 0 flux-foundation/flux/foundation/memory.hpp | 4 + .../flux/foundation/memory/align-test.cpp | 106 +++++ .../flux/foundation/memory/align.hpp | 67 ++++ .../flux/foundation/memory/debugging.cpp | 76 ++++ .../flux/foundation/memory/debugging.hpp | 55 +++ .../memory/detail/debug_helpers-test.cpp | 79 ++++ .../memory/detail/debug_helpers.cpp | 76 ++++ .../memory/detail/debug_helpers.hpp | 179 +++++++++ .../memory/detail/fixed_stack-test.cpp | 73 ++++ .../foundation/memory/detail/fixed_stack.hpp | 79 ++++ .../memory/detail/free_list-test.cpp | 288 ++++++++++++++ .../foundation/memory/detail/free_list.hpp | 361 ++++++++++++++++++ .../memory/detail/free_list_array-test.cpp | 44 +++ .../memory/detail/free_list_array.hpp | 99 +++++ .../memory/detail/free_list_helpers.hpp | 259 +++++++++++++ .../detail/low_level_allocator_adapter.hpp | 69 ++++ .../memory/detail/malloc_allocator.hpp | 64 ++++ .../memory/detail/test_allocator.hpp | 70 ++++ .../memory/detail/win32_heap_allocator.hpp | 80 ++++ .../foundation/memory/memory_block-test.cpp | 2 + .../flux/foundation/memory/memory_block.hpp | 28 ++ .../memory/static_allocator-test.cpp | 84 ++++ .../foundation/memory/static_allocator.hpp | 110 ++++++ 25 files changed, 2361 insertions(+), 1 deletion(-) delete mode 100644 flux-foundation/flux/foundation/dummy.cpp create mode 100644 flux-foundation/flux/foundation/memory/align-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/align.hpp create mode 100644 flux-foundation/flux/foundation/memory/debugging.cpp create mode 100644 flux-foundation/flux/foundation/memory/debugging.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/debug_helpers-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/detail/debug_helpers.cpp create mode 100644 flux-foundation/flux/foundation/memory/detail/debug_helpers.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/fixed_stack-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/detail/fixed_stack.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/free_list-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/detail/free_list.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/free_list_array-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/detail/free_list_array.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/free_list_helpers.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/low_level_allocator_adapter.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/malloc_allocator.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/test_allocator.hpp create mode 100644 flux-foundation/flux/foundation/memory/detail/win32_heap_allocator.hpp create mode 100644 flux-foundation/flux/foundation/memory/memory_block-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/memory_block.hpp create mode 100644 flux-foundation/flux/foundation/memory/static_allocator-test.cpp create mode 100644 flux-foundation/flux/foundation/memory/static_allocator.hpp diff --git a/flux-foundation/CMakeLists.txt b/flux-foundation/CMakeLists.txt index 18a7254..0bdc67f 100644 --- a/flux-foundation/CMakeLists.txt +++ b/flux-foundation/CMakeLists.txt @@ -2,12 +2,20 @@ flux_static_library(foundation COMMON TEST "flux/foundation/memory/detail/constexpr_memcpy-test.cpp" + "flux/foundation/memory/detail/debug_helpers-test.cpp" + "flux/foundation/memory/detail/fixed_stack-test.cpp" + "flux/foundation/memory/detail/free_list_array-test.cpp" + "flux/foundation/memory/detail/free_list-test.cpp" + "flux/foundation/memory/align-test.cpp" "flux/foundation/memory/construct-test.cpp" + "flux/foundation/memory/memory_block-test.cpp" "flux/foundation/memory/relocate-test.cpp" + "flux/foundation/memory/static_allocator-test.cpp" "flux/foundation/memory/uninitialized_algorithms-test.cpp" "flux/foundation/memory/uninitialized_storage-test.cpp" SOURCE - "flux/foundation/dummy.cpp" + "flux/foundation/memory/detail/debug_helpers.cpp" + "flux/foundation/memory/debugging.cpp" LINK flux::io flux::platform) diff --git a/flux-foundation/flux/foundation/dummy.cpp b/flux-foundation/flux/foundation/dummy.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/flux-foundation/flux/foundation/memory.hpp b/flux-foundation/flux/foundation/memory.hpp index 4d9f401..94a916f 100644 --- a/flux-foundation/flux/foundation/memory.hpp +++ b/flux-foundation/flux/foundation/memory.hpp @@ -1,4 +1,8 @@ #pragma once +#include +#include +#include +#include #include #include \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/align-test.cpp b/flux-foundation/flux/foundation/memory/align-test.cpp new file mode 100644 index 0000000..7cfa64b --- /dev/null +++ b/flux-foundation/flux/foundation/memory/align-test.cpp @@ -0,0 +1,106 @@ +#include + +#include + +using namespace flux::fou; + +TEST_CASE("fou::align_offset", "[flux-memory/align.hpp]") { + auto ptr = reinterpret_cast(0); + REQUIRE(align_offset(ptr, 1) == 0u); + REQUIRE(align_offset(ptr, 16) == 0u); + ptr = reinterpret_cast(1); + REQUIRE(align_offset(ptr, 1) == 0u); + REQUIRE(align_offset(ptr, 16) == 15u); + ptr = reinterpret_cast(8); + REQUIRE(align_offset(ptr, 4) == 0u); + REQUIRE(align_offset(ptr, 8) == 0u); + REQUIRE(align_offset(ptr, 16) == 8u); + ptr = reinterpret_cast(16); + REQUIRE(align_offset(ptr, 16) == 0u); + ptr = reinterpret_cast(1025); + REQUIRE(align_offset(ptr, 16) == 15u); +} + +TEST_CASE("fou::is_aligned", "[flux-memory/align.hpp]") { + auto ptr = reinterpret_cast(0); + REQUIRE(is_aligned(ptr, 1)); + REQUIRE(is_aligned(ptr, 8)); + REQUIRE(is_aligned(ptr, 16)); + ptr = reinterpret_cast(1); + REQUIRE(is_aligned(ptr, 1)); + REQUIRE(!is_aligned(ptr, 16)); + ptr = reinterpret_cast(8); + REQUIRE(is_aligned(ptr, 1)); + REQUIRE(is_aligned(ptr, 4)); + REQUIRE(is_aligned(ptr, 8)); + REQUIRE(!is_aligned(ptr, 16)); + ptr = reinterpret_cast(16); + REQUIRE(is_aligned(ptr, 1)); + REQUIRE(is_aligned(ptr, 8)); + REQUIRE(is_aligned(ptr, 16)); + ptr = reinterpret_cast(1025); + REQUIRE(is_aligned(ptr, 1)); + REQUIRE(!is_aligned(ptr, 16)); +} + +TEST_CASE("fou::alignment_for", "[flux-memory/align.hpp]") { + static_assert(detail::max_alignment >= 8, "test case not working"); + REQUIRE(alignment_for(1) == 1); + REQUIRE(alignment_for(2) == 2); + REQUIRE(alignment_for(3) == 2); + REQUIRE(alignment_for(4) == 4); + REQUIRE(alignment_for(5) == 4); + REQUIRE(alignment_for(6) == 4); + REQUIRE(alignment_for(7) == 4); + REQUIRE(alignment_for(8) == 8); + REQUIRE(alignment_for(9) == 8); + REQUIRE(alignment_for(100) == detail::max_alignment); +} + +TEST_CASE("fou::ilog2", "[flux-memory/align.hpp]") { + SECTION("Check everything up to 2^16") { + for (::std::size_t i = 0; i != 16; ++i) { + auto power = 1u << i; + auto next_power = 2 * power; + for (auto x = power; x != next_power; ++x) + CHECK(ilog2(x) == i); + } + } + + CHECK(ilog2(::std::size_t(1) << 32) == 32); + CHECK(ilog2((::std::size_t(1) << 32) + 44) == 32); + CHECK(ilog2((::std::size_t(1) << 32) + 2048) == 32); + + CHECK(ilog2(::std::size_t(1) << 48) == 48); + CHECK(ilog2((::std::size_t(1) << 48) + 44) == 48); + CHECK(ilog2((::std::size_t(1) << 48) + 2048) == 48); + + CHECK(ilog2(::std::size_t(1) << 63) == 63); + CHECK(ilog2((::std::size_t(1) << 63) + 44) == 63); + CHECK(ilog2((::std::size_t(1) << 63) + 2063) == 63); +} + +TEST_CASE("fou::ilog2_ceil", "[flux-memory/align.hpp]") { + SECTION("Check everything up to 2^16") { + for (::std::size_t i = 0; i != 16; ++i) { + auto power = 1u << i; + CHECK(ilog2_ceil(power) == i); + + auto next_power = 2 * power; + for (auto x = power + 1; x != next_power; ++x) + CHECK(ilog2_ceil(x) == i + 1); + } + } + + CHECK(ilog2_ceil(::std::size_t(1) << 32) == 32); + CHECK(ilog2_ceil((::std::size_t(1) << 32) + 44) == 33); + CHECK(ilog2_ceil((::std::size_t(1) << 32) + 2048) == 33); + + CHECK(ilog2_ceil(::std::size_t(1) << 48) == 48); + CHECK(ilog2_ceil((::std::size_t(1) << 48) + 44) == 49); + CHECK(ilog2_ceil((::std::size_t(1) << 48) + 2048) == 49); + + CHECK(ilog2_ceil(::std::size_t(1) << 63) == 63); + CHECK(ilog2_ceil((::std::size_t(1) << 63) + 44) == 64); + CHECK(ilog2_ceil((::std::size_t(1) << 63) + 2063) == 64); +} \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/align.hpp b/flux-foundation/flux/foundation/memory/align.hpp new file mode 100644 index 0000000..6157e59 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/align.hpp @@ -0,0 +1,67 @@ +#pragma once +#include + +namespace flux::fou { + +namespace detail { +// __STDCPP_DEFAULT_NEW_ALIGNMENT__ +inline constexpr auto max_alignment = alignof(::std::max_align_t); +} // namespace detail + +// clang-format off +template +constexpr bool is_pow2(I value) noexcept { + return value && !((value) & (value - 1)); +} +// clang-format on + +namespace detail { + +constexpr auto ilog2_base(unsigned long long value) noexcept { +#if __has_builtin(__builtin_clzll) + return sizeof(value) * CHAR_BIT - static_cast(__builtin_clzll(value)); +#else + auto clz = 64ull; + auto c = 32ull; + do { + auto tmp = value >> c; + if (tmp != 0ull) { + clz -= c; + value = tmp; + } + c = c >> 1ull; + } while (c != 0ull); + clz -= value ? 1ull : 0ull; + + return 64ull - clz; +#endif +} + +} // namespace detail + +constexpr ::std::size_t ilog2(::std::size_t value) noexcept { + return detail::ilog2_base(value) - 1ull; +} + +constexpr ::std::size_t ilog2_ceil(::std::size_t value) noexcept { + return detail::ilog2_base(value) - static_cast<::std::size_t>(is_pow2(value)); +} + +constexpr ::std::size_t alignment_for(::std::size_t size) noexcept { + return size >= detail::max_alignment ? detail::max_alignment : (::std::size_t{1} << ilog2(size)); +} + +constexpr ::std::size_t align_offset(::std::uintptr_t address, ::std::size_t alignment) noexcept { + auto const offset = address & (alignment - 1); + return 0ull != offset ? (alignment - offset) : 0ull; +} + +inline ::std::size_t align_offset(void* ptr, ::std::size_t alignment) noexcept { + return align_offset(reinterpret_cast<::std::uintptr_t>(ptr), alignment); +} + +inline bool is_aligned(void* ptr, ::std::size_t alignment) noexcept { + return reinterpret_cast<::std::uintptr_t>(ptr) % alignment == 0ull; +} + +} // namespace salt::ext \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/debugging.cpp b/flux-foundation/flux/foundation/memory/debugging.cpp new file mode 100644 index 0000000..c507061 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/debugging.cpp @@ -0,0 +1,76 @@ +#include +#include + +#include + +namespace flux::fou { + +constexpr void print_define(io::reserve_type_t, io::output_stream auto stream, + allocator_info const& info) noexcept { + io::print(stream, "Allocator ", info.name, " (at ", io::address(info.allocator), ") "); +} + +namespace { + +void default_leak_handler(allocator_info const& info, ::std::ptrdiff_t amount) noexcept { + if (amount > 0) { + io::panic(info, "leaked ", amount, " bytes"); + } else { + io::panic(info, "has deallocated ", amount, " bytes more than ever allocated"); + } +} + +::std::atomic internal_leak_handler(default_leak_handler); + +} // namespace + +leak_handler set_leak_handler(leak_handler handler) noexcept { + return internal_leak_handler.exchange(handler ? handler : default_leak_handler); +} + +leak_handler get_leak_handler() noexcept { + return internal_leak_handler; +} + +namespace { + +void default_invalid_ptr_handler(allocator_info const& info, const void* ptr) noexcept { + io::panic("Deallocation function of ", info, "received invalid pointer ", io::address(ptr)); +} + +::std::atomic internal_invalid_ptr_handler(default_invalid_ptr_handler); + +} // namespace + +invalid_pointer_handler set_invalid_pointer_handler(invalid_pointer_handler handler) noexcept { + return internal_invalid_ptr_handler.exchange(handler ? handler : default_invalid_ptr_handler); +} + +invalid_pointer_handler get_invalid_pointer_handler() noexcept { + return internal_invalid_ptr_handler; +} + +namespace { + +void default_buffer_overflow_handler(const void* memory, ::std::size_t node_size, + const void* ptr) noexcept { + io::panic("Buffer overflow at address ", io::address(ptr), + " detected, corresponding memory block", io::address(memory), " has only size ", + node_size); +} + +::std::atomic + internal_buffer_overflow_handler(default_buffer_overflow_handler); + +} // namespace + +buffer_overflow_handler set_buffer_overflow_handler(buffer_overflow_handler handler) noexcept { + return internal_buffer_overflow_handler.exchange(handler ? handler + : default_buffer_overflow_handler); +} + +buffer_overflow_handler get_buffer_overflow_handler() noexcept { + return internal_buffer_overflow_handler; +} + +} // namespace flux::fou \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/debugging.hpp b/flux-foundation/flux/foundation/memory/debugging.hpp new file mode 100644 index 0000000..1be9569 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/debugging.hpp @@ -0,0 +1,55 @@ +#pragma once +#include + +namespace flux::fou { + +// The magic values that are used for debug filling. If SALT_MEMORY_DEBUG_FILL is true, memory will +// be filled to help detect use-after-free or missing initialization errors. These are the +// constants for the different types. +enum class debug_magic : ::std::uint8_t { + // Marks internal memory used by the allocator - "allocated block". + internal_memory = 0xAB, + // Marks internal memory currently not used by the allocator - "freed block". + internal_freed_memory = 0xFB, + // Marks allocated, but not yet used memory - "clean memory". + new_memory = 0xCD, + // Marks freed memory - "dead memory". + freed_memory = 0xDD, + // Marks buffer memory used to ensure proper alignment. + alignment_memory = 0xED, + // Marks buffer memory used to protect against overflow - "fence memory". + fence_memory = 0xFD +}; + +// Contains information about an allocator. It can be used for logging in the various handler +// functions. +struct [[nodiscard]] allocator_info final { + ::std::string_view name; + void const* allocator; + + friend constexpr bool operator==(allocator_info const& lhs, + allocator_info const& rhs) noexcept = default; +}; + +// The type of the handler called when a memory leak is detected. +using leak_handler = void (*)(allocator_info const& info, ::std::ptrdiff_t amount); + +leak_handler set_leak_handler(leak_handler handler) noexcept; + +leak_handler get_leak_handler() noexcept; + +// The type of the handler called when an invalid pointer is passed to a deallocation function. +using invalid_pointer_handler = void (*)(allocator_info const& info, void const* ptr); + +invalid_pointer_handler set_invalid_pointer_handler(invalid_pointer_handler handler) noexcept; + +invalid_pointer_handler get_invalid_pointer_handler() noexcept; + +// The type of the handler called when a buffer under/overflow is detected. +using buffer_overflow_handler = void (*)(void const* memory, ::std::size_t size, void const* ptr); + +buffer_overflow_handler set_buffer_overflow_handler(buffer_overflow_handler handler) noexcept; + +buffer_overflow_handler get_buffer_overflow_handler() noexcept; + +} // namespace flux::fou \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/debug_helpers-test.cpp b/flux-foundation/flux/foundation/memory/detail/debug_helpers-test.cpp new file mode 100644 index 0000000..bc6b91d --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/debug_helpers-test.cpp @@ -0,0 +1,79 @@ +#include +#include + +#include + +using namespace flux::fou; +using namespace flux::fou::detail; + +TEST_CASE("fou::detail::debug_fill", "[flux-memory/debug_helpers.hpp]") { + debug_magic magic_values[10]; + for (auto& magic_value : magic_values) { + magic_value = debug_magic::freed_memory; + } + + debug_fill(magic_values, sizeof(magic_values), debug_magic::new_memory); +#if FLUX_MEMORY_DEBUG_FILL + for (auto magic_value : magic_values) { + CHECK(magic_value == debug_magic::new_memory); + } +#else + for (auto magic_value : magic_values) { + CHECK(magic_value == debug_magic::freed_memory); + } +#endif +} + +TEST_CASE("fou::detail::debug_is_filled", "[flux-memory/debug_helpers.hpp]") { + debug_magic magic_values[10]; + for (auto& magic_value : magic_values) { + magic_value = debug_magic::freed_memory; + } + + CHECK(debug_is_filled(magic_values, sizeof(magic_values), debug_magic::freed_memory) == + nullptr); + + magic_values[5] = debug_magic::new_memory; + auto ptr = static_cast( + debug_is_filled(magic_values, sizeof(magic_values), debug_magic::freed_memory)); +#if FLUX_MEMORY_DEBUG_FILL + CHECK(ptr == magic_values + 5); +#else + CHECK(ptr == nullptr); +#endif +} + +TEST_CASE("fou::detail::debug_fill_new/free", "[flux-memory/debug_helpers.hpp]") { + debug_magic magic_values[10]; + + auto result = debug_fill_new(magic_values, 8 * sizeof(debug_magic), sizeof(debug_magic)); + auto offset = static_cast(result) - magic_values; + auto expected_offset = static_cast<::std::int32_t>(debug_fence_size ? sizeof(debug_magic) : 0); + CHECK(offset == expected_offset); + +#if FLUX_MEMORY_DEBUG_FILL +# if FLUX_MEMORY_DEBUG_FENCE + CHECK(magic_values[0] == debug_magic::fence_memory); + CHECK(magic_values[9] == debug_magic::fence_memory); + const auto start = 1; +# else + const auto start = 0; +# endif + for (auto i = start; i < start + 8; ++i) { + CHECK(magic_values[i] == debug_magic::new_memory); + } +#endif + + result = debug_fill_free(result, 8 * sizeof(debug_magic), sizeof(debug_magic)); + CHECK(static_cast(result) == magic_values); + +#if FLUX_MEMORY_DEBUG_FILL +# if FLUX_MEMORY_DEBUG_FENCE + CHECK(magic_values[0] == debug_magic::fence_memory); + CHECK(magic_values[9] == debug_magic::fence_memory); +# endif + for (auto i = start; i < start + 8; ++i) { + CHECK(magic_values[i] == debug_magic::freed_memory); + } +#endif +} \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/debug_helpers.cpp b/flux-foundation/flux/foundation/memory/detail/debug_helpers.cpp new file mode 100644 index 0000000..fc94b5c --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/debug_helpers.cpp @@ -0,0 +1,76 @@ +#include + +#include + +namespace flux::fou::detail { + +#if FLUX_MEMORY_DEBUG_FILL +void debug_fill(void* memory, ::std::size_t size, debug_magic magic_value) noexcept { +# if __has_builtin(__builtin_memset) + __builtin_memset(memory, static_cast(magic_value), size); +# else + ::std::memset(memory, static_cast(magic_value), size); +# endif +} + +void* debug_is_filled(void* memory, ::std::size_t size, debug_magic magic_value) noexcept { + auto byte = static_cast<::std::byte*>(memory); + for (auto const last = byte + size; byte != last; ++byte) { + if (static_cast<::std::byte>(magic_value) != *byte) { + return byte; + } + } + return nullptr; +} + +void* debug_fill_new(void* memory, ::std::size_t node_size, ::std::size_t fence_size) noexcept { + if (!debug_fence_size) { + fence_size = 0u; + } + + auto mem = static_cast<::std::byte*>(memory); + debug_fill(mem, fence_size, debug_magic::fence_memory); + + mem += fence_size; + debug_fill(mem, node_size, debug_magic::new_memory); + + debug_fill(mem + node_size, fence_size, debug_magic::fence_memory); + + return mem; +} + +void* debug_fill_free(void* memory, ::std::size_t node_size, ::std::size_t fence_size) noexcept { + if (!debug_fence_size) { + fence_size = 0u; + } + + debug_fill(memory, node_size, debug_magic::freed_memory); + + auto pre_fence = static_cast<::std::byte*>(memory) - fence_size; + if (auto pre_dirty = debug_is_filled(pre_fence, fence_size, debug_magic::fence_memory)) { + get_buffer_overflow_handler()(memory, node_size, pre_dirty); + } + + auto post_mem = static_cast<::std::byte*>(memory) + node_size; + if (auto post_dirty = debug_is_filled(post_mem, fence_size, debug_magic::fence_memory)) { + get_buffer_overflow_handler()(memory, node_size, post_dirty); + } + + return pre_fence; +} + +void debug_fill_internal(void* memory, ::std::size_t size, bool free) noexcept { + debug_fill(memory, size, + free ? debug_magic::internal_freed_memory : debug_magic::internal_memory); +} +#endif // FLUX_MEMORY_DEBUG_FILL + +void debug_handle_invalid_ptr(allocator_info const& info, void* ptr) noexcept { + get_invalid_pointer_handler()(info, ptr); +} + +void debug_handle_memory_leak(allocator_info const& info, ::std::ptrdiff_t amount) noexcept { + get_leak_handler()(info, amount); +} + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/debug_helpers.hpp b/flux-foundation/flux/foundation/memory/detail/debug_helpers.hpp new file mode 100644 index 0000000..bd31676 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/debug_helpers.hpp @@ -0,0 +1,179 @@ +#pragma once +#include +#include + +#include + +namespace flux::fou { + +struct [[nodiscard]] allocator_info; + +enum class debug_magic : ::std::uint8_t; + +namespace detail { + +constexpr ::std::size_t debug_fence_size = FLUX_MEMORY_DEBUG_FILL ? FLUX_MEMORY_DEBUG_FENCE : 0u; + +#if FLUX_MEMORY_DEBUG_FILL +void debug_fill(void* memory, ::std::size_t size, debug_magic magic_value) noexcept; + +void* debug_is_filled(void* memory, ::std::size_t size, debug_magic magic_value) noexcept; + +void* debug_fill_new(void* memory, ::std::size_t node_size, + ::std::size_t fence_size = debug_fence_size) noexcept; + +void* debug_fill_free(void* memory, ::std::size_t node_size, + ::std::size_t fence_size = debug_fence_size) noexcept; + +void debug_fill_internal(void* memory, ::std::size_t size, bool free) noexcept; +#else +constexpr void debug_fill(void*, ::std::size_t, debug_magic) noexcept {} + +constexpr void* debug_is_filled(void*, ::std::size_t, debug_magic) noexcept { + return nullptr; +} + +constexpr void* debug_fill_new(void* memory, ::std::size_t, ::std::size_t) noexcept { + return memory; +} + +constexpr void* debug_fill_free(void* memory, ::std::size_t, ::std::size_t) noexcept { + return memory; +} + +constexpr void debug_fill_internal(void*, ::std::size_t, bool) noexcept {} +#endif // FLUX_MEMORY_DEBUG_FILL + +void debug_handle_invalid_ptr(allocator_info const& info, void* ptr) noexcept; +void debug_handle_memory_leak(allocator_info const& info, ::std::ptrdiff_t amount) noexcept; + +// clang-format off +template +constexpr void debug_check_pointer([[maybe_unused]] Predicate pred, + [[maybe_unused]] allocator_info const& info, + [[maybe_unused]] void* const ptr ) noexcept { +#if FLUX_MEMORY_DEBUG_POINTER + if (!pred()) + debug_handle_invalid_ptr(info, ptr); +#endif +} + +template +constexpr void debug_check_double_free([[maybe_unused]] Predicate pred, + [[maybe_unused]] allocator_info const& info, + [[maybe_unused]] void* const ptr ) noexcept { +#if FLUX_MEMORY_DEBUG_DOUBLE_FREE + debug_check_pointer(pred, info, ptr); +#endif +} + +template +concept leak_handler = + meta::is_class_v and + meta::default_initializable and + requires(Handler handler) { + { handler(meta::declval<::std::ptrdiff_t>()) } -> meta::same_as; + }; + +template +struct [[maybe_unused]] dummy_leak_detector { + constexpr dummy_leak_detector() noexcept = default; + constexpr ~dummy_leak_detector() = default; + + constexpr dummy_leak_detector(dummy_leak_detector&&) noexcept = default; + constexpr dummy_leak_detector& operator=(dummy_leak_detector&&) noexcept = default; + + constexpr void on_allocate(::std::size_t) noexcept {} + constexpr void on_deallocate(::std::size_t) noexcept {} +}; + +template +struct [[maybe_unused]] object_leak_detector : Handler { + constexpr object_leak_detector() noexcept : allocated_{0} {} + + constexpr object_leak_detector(object_leak_detector&& other) noexcept + : allocated_{::std::exchange(other.allocated_, 0)} {} + + constexpr ~object_leak_detector() { + if (allocated_ != 0) { + Handler::operator()(allocated_); + } + } + + constexpr object_leak_detector& operator=(object_leak_detector&& other) noexcept { + allocated_ = ::std::exchange(other.allocated_, 0); + return *this; + } + + constexpr void on_allocate(::std::size_t size) noexcept { + allocated_ += static_cast<::std::ptrdiff_t>(size); + } + + constexpr void on_deallocate(::std::size_t size) noexcept { + allocated_ -= static_cast<::std::ptrdiff_t>(size); + } + +private: + ::std::ptrdiff_t allocated_; +}; + +template +struct [[maybe_unused]] global_leak_detector { + + struct [[maybe_unused]] object_counter final : Handler { + constexpr object_counter() noexcept { + ++objects_; + } + + constexpr ~object_counter() { + --objects_; + if (0u == objects_ && 0u != allocated_) { + Handler::operator()(allocated_); + } + } + }; + + constexpr global_leak_detector() noexcept = default; + constexpr ~global_leak_detector() = default; + + constexpr global_leak_detector(global_leak_detector&&) noexcept = default; + constexpr global_leak_detector& operator=(global_leak_detector&&) noexcept = default; + + constexpr void on_allocate(::std::size_t size) noexcept { + allocated_ += static_cast<::std::ptrdiff_t>(size); + } + + constexpr void on_deallocate(::std::size_t size) noexcept { + allocated_ -= static_cast<::std::ptrdiff_t>(size); + } + +private: + static ::std::atomic_size_t objects_; + static ::std::atomic_ptrdiff_t allocated_; +}; + +template +::std::atomic_size_t global_leak_detector::objects_ = 0u; + +template +::std::atomic_ptrdiff_t global_leak_detector::allocated_ = 0; +// clang-format on + +} // namespace detail + +#if FLUX_MEMORY_DEBUG_LEAK +template using global_leak_detector = detail::global_leak_detector; +template using default_leak_detector = detail::object_leak_detector; +#else +template using global_leak_detector = detail::dummy_leak_detector; +template using default_leak_detector = detail::dummy_leak_detector; +#endif // FLUX_MEMORY_DEBUG_LEAK + +#if FLUX_MEMORY_DEBUG_LEAK +# define FLUX_MEMORY_GLOBAL_LEAK_DETECTOR(handler, name) \ + static global_leak_detector::object_counter name; +#else +# define FLUX_MEMORY_GLOBAL_LEAK_DETECTOR(handler, name) +#endif // FLUX_MEMORY_DEBUG_LEAK + +} // namespace flux::fou \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/fixed_stack-test.cpp b/flux-foundation/flux/foundation/memory/detail/fixed_stack-test.cpp new file mode 100644 index 0000000..154cb07 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/fixed_stack-test.cpp @@ -0,0 +1,73 @@ +#include + +#include + +using namespace flux::fou; + +TEST_CASE("fou::detail::fixed_stack", "[flux-memory/fixed_stack.hpp]") { + detail::fixed_stack stack; + REQUIRE(stack.top() == nullptr); + + SECTION("allocate") { + static_allocator_storage<1024> memory; + stack = detail::fixed_stack{&memory}; + auto end = stack.top() + 1024; + + CHECK(stack.top() == reinterpret_cast<::std::byte*>(&memory)); + + SECTION("alignment for allocate") { + auto* ptr = stack.allocate(end, 13, 1u); + CHECK(ptr); + CHECK(is_aligned(ptr, 1u)); + + ptr = stack.allocate(end, 10, 2u); + CHECK(ptr); + CHECK(is_aligned(ptr, 2u)); + + ptr = stack.allocate(end, 10, detail::max_alignment); + CHECK(ptr); + CHECK(is_aligned(ptr, detail::max_alignment)); + + ptr = stack.allocate(end, 10, 2 * detail::max_alignment); + CHECK(ptr); + CHECK(is_aligned(ptr, 2 * detail::max_alignment)); + } + + SECTION("unwind") { + CHECK(stack.allocate(end, 10u, 1u)); + auto diff = ::std::size_t(stack.top() - reinterpret_cast<::std::byte*>(&memory)); + CHECK(diff == 2 * detail::debug_fence_size + 10u); + + CHECK(stack.allocate(end, 16u, 1u)); + auto diff2 = ::std::size_t(stack.top() - reinterpret_cast<::std::byte*>(&memory)); + CHECK(diff2 == 2 * detail::debug_fence_size + 16u + diff); + + stack.unwind(reinterpret_cast<::std::byte*>(&memory) + diff); + CHECK(stack.top() == reinterpret_cast<::std::byte*>(&memory) + diff); + + auto* top = stack.top(); + CHECK(!stack.allocate(end, 1024, 1)); + CHECK(stack.top() == top); + } + } + + SECTION("move") { + static_allocator_storage<1024> memory; + auto end = reinterpret_cast<::std::byte*>(&memory) + 1024; + + detail::fixed_stack other(reinterpret_cast<::std::byte*>(&memory)); + CHECK(other.top() == reinterpret_cast<::std::byte*>(&memory)); + + stack = ::std::move(other); + CHECK(stack.top() == reinterpret_cast<::std::byte*>(&memory)); + + CHECK(!other.allocate(end, 10, 1)); + CHECK(stack.allocate(end, 10, 1)); + auto* top = stack.top(); + + other = ::std::move(stack); + CHECK(other.top() == top); + CHECK(!stack.allocate(end, 10, 1)); + CHECK(other.allocate(end, 10, 1)); + } +} \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/fixed_stack.hpp b/flux-foundation/flux/foundation/memory/detail/fixed_stack.hpp new file mode 100644 index 0000000..82d0767 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/fixed_stack.hpp @@ -0,0 +1,79 @@ +#pragma once +#include +#include +#include + +namespace flux::fou::detail { + +// Simple memory stack implementation that does not support growing. +struct [[nodiscard]] fixed_stack final { + constexpr fixed_stack() noexcept : current_{nullptr} {} + + constexpr explicit fixed_stack(void* const memory) noexcept + : current_{static_cast<::std::byte*>(memory)} {} + + constexpr fixed_stack(fixed_stack&& other) noexcept + : current_{::std::exchange(other.current_, nullptr)} {} + + constexpr ~fixed_stack() = default; + + constexpr fixed_stack& operator=(fixed_stack&& other) noexcept { + current_ = ::std::exchange(other.current_, nullptr); + return *this; + } + + constexpr void advance(::std::size_t offset) noexcept { + current_ += offset; + } + + constexpr void advance(::std::size_t offset, debug_magic magic_value) noexcept { + debug_fill(current_, offset, magic_value); + advance(offset); + } + + constexpr void* advance_return(::std::size_t offset, + debug_magic magic_value = debug_magic::new_memory) noexcept { + auto memory = current_; + debug_fill(memory, offset, magic_value); + advance(offset); + return memory; + } + + constexpr void* allocate(::std::byte const* end, ::std::size_t size, ::std::size_t alignment, + ::std::size_t fence_size = debug_fence_size) noexcept { + if (current_ == nullptr) { + return current_; + } + + auto const remaining = static_cast<::std::size_t>(end - current_); + auto const offset = align_offset(current_ + fence_size, alignment); + if (fence_size + offset + size + fence_size > remaining) { + return nullptr; + } + + return allocate_unchecked(size, offset, fence_size); + } + + constexpr void* allocate_unchecked(::std::size_t size, ::std::size_t align_offset, + ::std::size_t fence_size = debug_fence_size) noexcept { + advance(fence_size, debug_magic::fence_memory); + advance(align_offset, debug_magic::alignment_memory); + auto* memory = advance_return(size); + advance(fence_size, debug_magic::fence_memory); + return memory; + } + + constexpr void unwind(::std::byte* top) noexcept { + debug_fill(top, static_cast<::std::size_t>(current_ - top), debug_magic::freed_memory); + current_ = top; + } + + constexpr ::std::byte* top() const noexcept { + return current_; + } + +private: + ::std::byte* current_; +}; + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/free_list-test.cpp b/flux-foundation/flux/foundation/memory/detail/free_list-test.cpp new file mode 100644 index 0000000..03303e1 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/free_list-test.cpp @@ -0,0 +1,288 @@ +#include +#include + +#include + +#include +#include +#include + +using namespace flux::fou; + +// clang-format off +template +void use_list_node(MemoryList& list) { + ::std::vector ptrs; + auto capacity = list.capacity(); + + // allocate and deallocate in reverse order + { + for (::std::size_t i = 0u; i != capacity; ++i) { + auto* ptr = list.allocate(); + CHECK(ptr); + CHECK(is_aligned(ptr, list.alignment())); + ptrs.push_back(ptr); + } + CHECK(list.capacity() == 0u); + CHECK(list.empty()); + + ::std::reverse(ptrs.begin(), ptrs.end()); + + for (auto p : ptrs) + list.deallocate(p); + CHECK(list.capacity() == capacity); + CHECK(!list.empty()); + ptrs.clear(); + } + + // allocate and deallocate in same order + { + for (::std::size_t i = 0u; i != capacity; ++i) { + auto* ptr = list.allocate(); + CHECK(ptr); + CHECK(is_aligned(ptr, list.alignment())); + ptrs.push_back(ptr); + } + CHECK(list.capacity() == 0u); + CHECK(list.empty()); + + for (auto p : ptrs) + list.deallocate(p); + CHECK(list.capacity() == capacity); + CHECK(!list.empty()); + ptrs.clear(); + } + + // allocate and deallocate in random order + { + for (::std::size_t i = 0u; i != capacity; ++i) { + auto* ptr = list.allocate(); + CHECK(ptr); + CHECK(is_aligned(ptr, list.alignment())); + ptrs.push_back(ptr); + } + CHECK(list.capacity() == 0u); + CHECK(list.empty()); + + ::std::shuffle(ptrs.begin(), ptrs.end(), ::std::mt19937{}); + + for (auto p : ptrs) { + list.deallocate(p); + } + CHECK(list.capacity() == capacity); + CHECK(!list.empty()); + ptrs.clear(); + } +} + +template +void check_list(MemoryList& list, void* memory, ::std::size_t size) { + auto old_capacity = list.capacity(); + + list.insert(memory, size); + CHECK(!list.empty()); + CHECK(list.capacity() <= old_capacity + size / list.node_size()); + + old_capacity = list.capacity(); + + auto* node = list.allocate(); + CHECK(node); + CHECK(is_aligned(node, list.alignment())); + CHECK(list.capacity() == old_capacity - 1); + + list.deallocate(node); + CHECK(list.capacity() == old_capacity); + + use_list_node(list); +} + +template +void check_move(MemoryList& list) { + static_allocator_storage<1024> memory; + list.insert(&memory, 1024); + + auto* ptr = list.allocate(); + CHECK(ptr); + CHECK(is_aligned(ptr, list.alignment())); + auto capacity = list.capacity(); + + MemoryList list2(::std::move(list)); + CHECK(list.empty()); + CHECK(list.capacity() == 0u); + CHECK(!list2.empty()); + CHECK(list2.capacity() == capacity); + + MemoryList list3(4); + list3 = ::std::move(list2); + CHECK(list2.empty()); + CHECK(list2.capacity() == 0u); + CHECK(!list3.empty()); + CHECK(list3.capacity() == capacity); + + list3.deallocate(ptr); +} + +void use_list_array(detail::free_list& list) { + ::std::vector ptrs; + auto capacity = list.capacity(); + // We would need capacity / 3 nodes, but the memory might not be contiguous. + auto number_of_allocations = capacity / 6; + + // allocate and deallocate in reverse order + { + for (::std::size_t i = 0u; i != number_of_allocations; ++i) { + auto ptr = list.allocate(3 * list.node_size()); + REQUIRE(ptr); + REQUIRE(is_aligned(ptr, list.alignment())); + ptrs.push_back(ptr); + } + + ::std::reverse(ptrs.begin(), ptrs.end()); + + for (auto p : ptrs) { + list.deallocate(p, 3 * list.node_size()); + } + REQUIRE(list.capacity() == capacity); + ptrs.clear(); + } + + // allocate and deallocate in same order + { + for (::std::size_t i = 0u; i != number_of_allocations; ++i) { + auto ptr = list.allocate(3 * list.node_size()); + REQUIRE(ptr); + REQUIRE(is_aligned(ptr, list.alignment())); + ptrs.push_back(ptr); + } + + for (auto p : ptrs) { + list.deallocate(p, 3 * list.node_size()); + } + REQUIRE(list.capacity() == capacity); + REQUIRE(!list.empty()); + ptrs.clear(); + } + + // allocate and deallocate in random order + { + for (::std::size_t i = 0u; i != number_of_allocations; ++i) { + auto ptr = list.allocate(3 * list.node_size()); + REQUIRE(ptr); + REQUIRE(is_aligned(ptr, list.alignment())); + ptrs.push_back(ptr); + } + + ::std::shuffle(ptrs.begin(), ptrs.end(), ::std::mt19937{}); + + for (auto p : ptrs) { + list.deallocate(p, 3 * list.node_size()); + } + REQUIRE(list.capacity() == capacity); + REQUIRE(!list.empty()); + ptrs.clear(); + } +} +// clang-format on + +TEST_CASE("fou::detail::unordered_free_list", "[flux-memory/unordered_free_list.hpp]") { + SECTION("construct") { + detail::unordered_free_list list(4); + REQUIRE(list.empty()); + REQUIRE(list.node_size() >= 4); + REQUIRE(list.capacity() == 0u); + } + + SECTION("normal insert") { + static_allocator_storage<1024> memory; + detail::unordered_free_list list(4); + check_list(list, &memory, 1024); + + check_move(list); + } + + SECTION("uneven insert") { + static_allocator_storage<1023> memory; // not dividable + detail::unordered_free_list list(4); + check_list(list, &memory, 1023); + + check_move(list); + } + + SECTION("multiple insert") { + static_allocator_storage<1024> a; + static_allocator_storage<100> b; + static_allocator_storage<1337> c; + detail::unordered_free_list list(4); + + check_list(list, &a, 1024); + check_list(list, &b, 100); + check_list(list, &c, 1337); + + check_move(list); + } + + SECTION("move") { + static_allocator_storage<1024> memory; + detail::unordered_free_list list(4); + check_list(list, &memory, 1024); + + static_allocator_storage<1024> new_memory; + detail::unordered_free_list new_list(::std::move(list)); + check_list(new_list, &new_memory, 1024); + } +} + +TEST_CASE("fou::detail::free_list", "[flux-memory/unordered_free_list.hpp]") { + SECTION("construct") { + detail::free_list list(4); + REQUIRE(list.empty()); + REQUIRE(list.node_size() >= 4); + REQUIRE(list.capacity() == 0u); + } + + SECTION("normal insert") { + static_allocator_storage<1024> memory; + detail::free_list list(4); + check_list(list, &memory, 1024); + use_list_array(list); + + check_move(list); + } + + SECTION("uneven insert") { + static_allocator_storage<1023> memory; // not dividable + detail::free_list list(4); + check_list(list, &memory, 1023); + use_list_array(list); + + check_move(list); + } + + SECTION("multiple insert") { + static_allocator_storage<1024> a; + static_allocator_storage<100> b; + static_allocator_storage<1337> c; + detail::free_list list(4); + + check_list(list, &a, 1024); + use_list_array(list); + check_list(list, &b, 100); + use_list_array(list); + check_list(list, &c, 1337); + use_list_array(list); + + check_move(list); + } + + SECTION("move") { + static_allocator_storage<1024> memory; + detail::free_list list(4); + check_list(list, &memory, 1023); + use_list_array(list); + + static_allocator_storage<1024> new_memory; + detail::free_list new_list(::std::move(list)); + check_list(new_list, &new_memory, 1024); + use_list_array(new_list); + } +} \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/free_list.hpp b/flux-foundation/flux/foundation/memory/detail/free_list.hpp new file mode 100644 index 0000000..e61a539 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/free_list.hpp @@ -0,0 +1,361 @@ +#pragma once +#include +#include + +#include +#include + +namespace flux::fou::detail { + +// clang-format off +// Stores free blocks for a memory pool, memory blocks are fragmented and stored in a list. +class [[nodiscard]] unordered_free_list final { + constexpr auto info() noexcept { + return allocator_info{"flux::fou::detail::unordered_free_list", this}; + } + +public: + using byte_type = ::std::byte; + using size_type = ::std::size_t; + using iterator = byte_type*; + using const_iterator = byte_type const*; + + static constexpr auto min_node_size = sizeof (iterator); + static constexpr auto min_alignment = alignof(iterator); + + constexpr explicit unordered_free_list(size_type node_size) noexcept + : first_{nullptr}, node_size_{min(node_size)}, capacity_{0u} {} + + constexpr unordered_free_list(size_type node_size, void* memory, size_type size) noexcept + : unordered_free_list{node_size} { + insert(memory, size); + } + + constexpr ~unordered_free_list() = default; + + constexpr unordered_free_list(unordered_free_list&& other) noexcept + : first_ {::std::exchange(other.first_, nullptr)}, + node_size_{other.node_size_ }, + capacity_ {::std::exchange(other.capacity_, 0u) } {} + + constexpr unordered_free_list& operator=(unordered_free_list&& other) noexcept { + unordered_free_list tmp{::std::move(other)}; + first_ = tmp.first_; + node_size_ = tmp.node_size_; + capacity_ = tmp.capacity_; + return *this; + } + + constexpr void insert(void* memory, size_type size) noexcept { + FLUX_ASSERT(memory); + FLUX_ASSERT(is_aligned(memory, alignment())); + debug_fill_internal(memory, size, false); + insert_impl(memory, size); + } + + constexpr void* allocate() noexcept { + FLUX_ASSERT(!empty()); + --capacity_; + + auto memory = first_; + first_ = get_next(first_); + return debug_fill_new(memory, node_size_, 0); + } + + constexpr void* allocate(size_type n) noexcept { + FLUX_ASSERT(!empty()); + if (n <= node_size_) { + return allocate(); + } + + auto range = find(first_, n, node_size_); + if (!range.first) [[unlikely]] { + return nullptr; + } + + if (range.prev) { + set_next(range.prev, range.next); + } else { + first_ = range.next; + } + capacity_ -= node_count(range, node_size_); + + return debug_fill_new(range.first, n, 0); + } + + constexpr void deallocate(void* ptr) noexcept { + ++capacity_; + + auto range = static_cast(debug_fill_free(ptr, node_size_, 0)); + set_next(range, first_); + first_ = range; + } + + constexpr void deallocate(void* ptr, size_type n) noexcept { + if (n <= node_size_) { + deallocate(ptr); + } else { + insert_impl(debug_fill_free(ptr, n, 0), n); + } + } + + constexpr size_type alignment() const noexcept { + return alignment_for(node_size_); + } + + constexpr size_type node_size() const noexcept { + return node_size_; + } + + constexpr size_type usable_size(size_type size) const noexcept { + return (size / node_size_) * node_size_; + } + + constexpr size_type capacity() const noexcept { + return capacity_; + } + + constexpr bool empty() const noexcept { + return nullptr == first_; + } + + static constexpr size_type min_block_size(size_type node_size, size_type node_count) noexcept { + return (node_size < min_node_size ? min_node_size : node_size) * node_count; + } + +private: + static constexpr size_type min(size_type node_size) noexcept { + return (node_size < min_node_size ? min_node_size : node_size); + } + + constexpr void insert_impl(void* memory, size_type size) noexcept { + auto node_count = size / node_size_; + FLUX_ASSERT(node_count > 0); + + auto range = static_cast(memory); + for (size_type i = 0u; i < node_count - 1; ++i) { + set_next(range, range + node_size_); + range += node_size_; + } + set_next(range, first_); + first_ = static_cast(memory); + capacity_ += node_count; + } + + iterator first_; + size_type node_size_; + size_type capacity_; +}; + +// Stores free blocks for a memory pool, memory blocks are fragmented and stored in a list. Keeps +// the nodes ordered this allows array allocations, that is, consecutive nodes. +class [[nodiscard]] free_list final { + constexpr auto info() noexcept { + return allocator_info{"flux::fou::detail::free_list", this}; + } + +public: + using byte_type = ::std::byte; + using size_type = ::std::size_t; + using iterator = byte_type*; + using const_iterator = byte_type const*; + + static constexpr auto min_node_size = sizeof (iterator); + static constexpr auto min_alignment = alignof(iterator); + + constexpr explicit free_list(size_type node_size) noexcept + : node_size_ {min(node_size)}, capacity_ {0u }, + last_dealloc_{end_node() }, last_dealloc_prev_{begin_node()} { + xor_set_next(begin_node(), nullptr, end_node()); + xor_set_next(end_node(), begin_node(), nullptr); + } + + constexpr free_list(size_type node_size, void* memory, size_type size) noexcept + : free_list{node_size} { + insert(memory, size); + } + + constexpr ~free_list() = default; + + constexpr free_list(free_list&& other) noexcept + : node_size_{other.node_size_}, capacity_{other.capacity_} { + if (!other.empty()) { + auto* begin = xor_get_next(other.begin_node(), nullptr); + auto* end = xor_get_next(other.end_node(), nullptr); + + xor_set_next(begin_node(), nullptr, begin); + xor_exchange(begin, other.begin_node(), begin_node()); + xor_exchange(end, other.end_node(), end_node()); + xor_set_next(end_node(), end, nullptr); + + other.capacity_ = 0u; + xor_set_next(other.begin_node(), nullptr, other.end_node()); + xor_set_next(other.end_node(), other.begin_node(), nullptr); + } else { + xor_set_next(begin_node(), nullptr, end_node()); + xor_set_next(end_node(), begin_node(), nullptr); + } + + last_dealloc_prev_ = begin_node(); + last_dealloc_ = xor_get_next(last_dealloc_prev_, nullptr); + } + + constexpr free_list& operator=(free_list&& other) noexcept { + free_list tmp{::std::move(other)}; + node_size_ = tmp.node_size_; + capacity_ = tmp.capacity_; + last_dealloc_ = tmp.last_dealloc_; + last_dealloc_prev_ = tmp.last_dealloc_prev_; + return *this; + } + + constexpr void insert(void* memory, size_type size) noexcept { + FLUX_ASSERT(memory); + FLUX_ASSERT(is_aligned(memory, alignment())); + debug_fill_internal(memory, size, false); + insert_impl(memory, size); + } + + constexpr void* allocate() noexcept { + FLUX_ASSERT(!empty()); + auto prev = begin_node(); + auto node = xor_get_next(prev, nullptr); + auto next = xor_get_next(node, prev); + + xor_set_next(prev, nullptr, next); + xor_exchange(next, node, prev); + --capacity_; + + if (node == last_dealloc_) { + last_dealloc_ = next; + FLUX_ASSERT(last_dealloc_prev_ == prev); + } else if (node == last_dealloc_prev_) { + last_dealloc_prev_ = prev; + FLUX_ASSERT(last_dealloc_ == next); + } + + return debug_fill_new(node, node_size_, 0); + } + + constexpr void* allocate(size_type size) noexcept { + FLUX_ASSERT(!empty()); + if (size <= node_size_) { + return allocate(); + } + + auto range = xor_find({begin_node(), end_node()}, size, node_size_); + if (!range.first) [[unlikely]] { + return nullptr; + } + + xor_exchange(range.prev, range.first, range.next); + xor_exchange(range.next, range.last, range.prev); + capacity_ -= node_count(range, node_size_); + + if (less_equal(range.first, last_dealloc_) && less_equal(last_dealloc_, range.last)) { + last_dealloc_ = range.next; + last_dealloc_prev_ = range.prev; + } else if (last_dealloc_prev_ == range.last) { + FLUX_ASSERT(last_dealloc_ == range.next); + last_dealloc_prev_ = range.prev; + } + + return debug_fill_new(range.first, size, 0); + } + + constexpr void deallocate(void* memory) noexcept { + auto node_next = static_cast(debug_fill_free(memory, node_size_, 0)); + + auto node = find_node(info(), node_next, {begin_node(), end_node()}, + last_dealloc_, last_dealloc_prev_); + // Links new node between prev and next. + xor_set_next(node_next, node.prev, node.next); + xor_exchange(node.prev, node.next, node_next); + xor_exchange(node.next, node.prev, node_next); + ++capacity_; + + last_dealloc_prev_ = node.prev; + last_dealloc_ = node_next; + } + + constexpr void deallocate(void* memory, size_type size) noexcept { + if (size <= node_size_) { + deallocate(memory); + } else { + auto ptr = debug_fill_free(memory, size, 0); + auto prev = insert_impl(ptr, size); + + last_dealloc_prev_ = prev; + last_dealloc_ = static_cast(ptr); + } + } + + constexpr size_type alignment() const noexcept { + return alignment_for(node_size_); + } + + constexpr size_type node_size() const noexcept { + return node_size_; + } + + constexpr size_type usable_size(size_type size) const noexcept { + return (size / node_size_) * node_size_; + } + + constexpr size_type capacity() const noexcept { + return capacity_; + } + + constexpr bool empty() const noexcept { + return 0u == capacity_; + } + + static constexpr size_type min_block_size(size_type node_size, size_type node_count) noexcept { + return (node_size < min_node_size ? min_node_size : node_size) * node_count; + } + +private: + static constexpr size_type min(size_type node_size) noexcept { + return (node_size < min_node_size ? min_node_size : node_size); + } + + constexpr iterator insert_impl(void* memory, size_type size) noexcept { + auto node_count = size / node_size_; + FLUX_ASSERT(node_count > 0); + + auto node = find_node(info(), static_cast(memory), {begin_node(), end_node()}, + last_dealloc_, last_dealloc_prev_); + xor_link_block(memory, node_size_, node_count, node); + capacity_ += node_count; + + if (node.prev == last_dealloc_prev_) { + last_dealloc_ = static_cast(memory); + } + return node.prev; + } + + constexpr iterator begin_node() noexcept { + void* begin = &begin_proxy_; + return static_cast(begin); + } + + constexpr iterator end_node() noexcept { + void* end = &end_proxy_; + return static_cast(end); + } + + size_type node_size_, capacity_; + iterator last_dealloc_, last_dealloc_prev_; + ::std::uintptr_t begin_proxy_, end_proxy_; +}; +// clang-format on + +#if FLUX_MEMORY_DEBUG_DOUBLE_FREE +using node_free_list = free_list; +using array_free_list = free_list; +#else +using node_free_list = unordered_free_list; +using array_free_list = free_list; +#endif + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/free_list_array-test.cpp b/flux-foundation/flux/foundation/memory/detail/free_list_array-test.cpp new file mode 100644 index 0000000..422db48 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/free_list_array-test.cpp @@ -0,0 +1,44 @@ +#include +#include +#include + +#include + +using namespace flux::fou; + +using log2_policy = detail::log2_access_policy; + +TEST_CASE("fou::detail::log2_access_policy", "[flux-memory/free_list_array.hpp]") { + REQUIRE(log2_policy::index_from_size(1) == 0u); + REQUIRE(log2_policy::index_from_size(2) == 1u); + REQUIRE(log2_policy::index_from_size(3) == 2u); + REQUIRE(log2_policy::index_from_size(4) == 2u); + REQUIRE(log2_policy::index_from_size(5) == 3u); + REQUIRE(log2_policy::index_from_size(6) == 3u); + REQUIRE(log2_policy::index_from_size(8) == 3u); + REQUIRE(log2_policy::index_from_size(9) == 4u); + + REQUIRE(log2_policy::size_from_index(0) == 1u); + REQUIRE(log2_policy::size_from_index(1) == 2u); + REQUIRE(log2_policy::size_from_index(2) == 4u); + REQUIRE(log2_policy::size_from_index(3) == 8u); +} + +TEST_CASE("fou::detail::free_list_array", "[flux-memory/free_list_array.hpp]") { + using fixed_stack = detail::fixed_stack; + using free_list = detail::unordered_free_list; + static_allocator_storage<1024> memory; + fixed_stack stack(&memory); + + SECTION("non power of two max size, normal list") { + using log2_free_list_array = detail::free_list_array; + log2_free_list_array array(stack, stack.top() + 1024, 15); + CHECK(array.max_node_size() == 16u); + CHECK(array.size() <= 5u); + + CHECK(array[1].node_size() == free_list::min_node_size); + CHECK(array[2].node_size() == free_list::min_node_size); + CHECK(array[9].node_size() == 16u); + CHECK(array[15].node_size() == 16u); + } +} \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/free_list_array.hpp b/flux-foundation/flux/foundation/memory/detail/free_list_array.hpp new file mode 100644 index 0000000..d859b93 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/free_list_array.hpp @@ -0,0 +1,99 @@ +#pragma once +#include + +namespace flux::fou::detail { + +// clang-format off +template +concept access_policy = + requires { + { AccessPolicy::index_from_size(1) } -> meta::same_as<::std::size_t>; + { AccessPolicy::size_from_index(1) } -> meta::same_as<::std::size_t>; + }; +// clang-format on + +// An array of `free_list` types indexed via size, AccessPolicy does necessary conversions. +template +struct [[nodiscard]] free_list_array final { + using access_policy = AccessPolicy; + using free_list_type = FreeList; + using size_type = typename free_list_type::size_type; + using iterator = typename free_list_type::iterator; + using const_iterator = typename free_list_type::const_iterator; + + // clang-format off + constexpr free_list_array(fixed_stack& stack, + const_iterator begin, + size_type max_node_size) noexcept + : size_ {access_policy::index_from_size(max_node_size) - min_node_size + 1}, + array_{static_cast(stack.allocate( + begin, size_ * sizeof(free_list_type), alignof(free_list_type)))} + { + FLUX_ASSERT(array_, "Insufficient memory for free lists"); + for (size_type i = 0u; i < size_; ++i) { + auto node_size = access_policy::size_from_index(i + min_node_size); + construct_at(array_ + i, node_size); + } + } + + constexpr ~free_list_array() = default; + + constexpr free_list_array(free_list_array&& other) noexcept + : size_ {::std::exchange(other.size_ , 0u )}, + array_{::std::exchange(other.array_, nullptr)} {} + + constexpr free_list_array& operator=(free_list_array&& other) noexcept { + size_ = ::std::exchange(other.size_ , 0u); + array_ = ::std::exchange(other.array_, nullptr); + return *this; + } + // clang-format on + + constexpr free_list_type& operator[](::std::size_t node_size) const noexcept { + auto index = access_policy::index_from_size(node_size); + if (index < min_node_size) { + index = min_node_size; + } + return array_[index - min_node_size]; + } + + constexpr size_type size() const noexcept { + return size_; + } + + constexpr size_type max_node_size() const noexcept { + return access_policy::size_from_index(size_ + min_node_size - 1); + } + +private: + static constexpr size_type min_node_size = + access_policy::index_from_size(free_list_type::min_node_size); + + size_type size_; + free_list_type* array_; +}; + +// AccessPolicy that maps size to indices 1:1. +struct [[nodiscard]] identity_access_policy final { + static constexpr auto index_from_size(::std::size_t size) noexcept { + return size; + } + + static constexpr auto size_from_index(::std::size_t index) noexcept { + return index; + } +}; + +// AccessPolicy that maps sizes to the integral log2. This creates more nodes and never wastes more +// than half the size. +struct [[nodiscard]] log2_access_policy final { + static constexpr auto index_from_size(::std::size_t size) noexcept { + return ilog2_ceil(size); + } + + static constexpr auto size_from_index(::std::size_t index) noexcept { + return ::std::size_t{1} << index; + } +}; + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/free_list_helpers.hpp b/flux-foundation/flux/foundation/memory/detail/free_list_helpers.hpp new file mode 100644 index 0000000..5d173e7 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/free_list_helpers.hpp @@ -0,0 +1,259 @@ +#pragma once +#include + +namespace flux::fou::detail { + +// A load operation copies data from a pointer to an integer. +inline ::std::uintptr_t load_int(void* address) noexcept { + FLUX_ASSERT(address); + ::std::uintptr_t value; +#if __has_builtin(__builtin_memcpy) + __builtin_memcpy(&value, address, sizeof(::std::uintptr_t)); +#else + ::std::memcpy(&value, address, sizeof(::std::uintptr_t)); +#endif + return value; +} + +// A store operation copies data from an integer to a pointer. +inline void store_int(void* address, ::std::uintptr_t value) noexcept { + FLUX_ASSERT(address); +#if __has_builtin(__builtin_memcpy) + __builtin_memcpy(address, &value, sizeof(::std::uintptr_t)); +#else + ::std::memcpy(address, &value, sizeof(::std::uintptr_t)); +#endif +} + +// clang-format off +FLUX_ALWAYS_INLINE +inline ::std::uintptr_t to_int(::std::byte* ptr) noexcept { + return reinterpret_cast<::std::uintptr_t>(ptr); +} + +FLUX_ALWAYS_INLINE +inline ::std::byte* from_int(::std::uintptr_t ptr) noexcept { + return reinterpret_cast<::std::byte*>(ptr); +} +// clang-format oт + +inline ::std::byte* get_next(void* address) noexcept { + return from_int(load_int(address)); +} + +inline void set_next(void* address, ::std::byte* next) noexcept { + store_int(address, to_int(next)); +} + +inline ::std::byte* xor_get_next(void* address, ::std::byte* prev_or_next) noexcept { + return from_int(load_int(address) ^ to_int(prev_or_next)); +} + +inline void xor_set_next(void* address, ::std::byte* prev, ::std::byte* next) noexcept { + store_int(address, to_int(prev) ^ to_int(next)); +} + +inline void xor_advance(::std::byte*& current, ::std::byte*& prev) noexcept { + auto next = xor_get_next(current, prev); + prev = current; + current = next; +} + +inline void xor_exchange(void* address, ::std::byte* old_ptr, ::std::byte* new_ptr) noexcept { + auto next = xor_get_next(address, old_ptr); + xor_set_next(address, next, new_ptr); +} + +constexpr bool less(void* a, void* b) noexcept { + return ::std::less()(a, b); +} + +constexpr bool less_equal(void* a, void* b) noexcept { + return a == b || less(a, b); +} + +constexpr bool greater(void* a, void* b) noexcept { + return ::std::greater()(a, b); +} + +constexpr bool greater_equal(void* a, void* b) noexcept { + return a == b || greater(a, b); +} + +// clang-format off +template +struct [[nodiscard]] memory_range final { + Iterator prev; // last before + Iterator first; // first in + Iterator last; // last in + Iterator next; // first after +}; + +template +struct [[nodiscard]] proxy_range final { + T begin; + T end; +}; + +// Number of nodes in the contiguous memory range. +template +constexpr auto node_count(memory_range const& range, ::std::size_t node_size) noexcept { + auto* end = range.last + node_size; // last is inclusive, so add actual size to it + FLUX_ASSERT(0u == static_cast<::std::size_t>(end - range.first) % node_size); + return static_cast<::std::size_t>(end - range.first) / node_size; +} + +// Searches for a range in memory that fits the size required for the allocation. +template +constexpr memory_range find(Iterator begin, + ::std::size_t bytes_needed, + ::std::size_t node_size) noexcept { + memory_range range; + range.prev = nullptr; + range.first = begin; + range.last = begin; // << used as iterator for the end of the range + range.next = get_next(begin); // << used as iterator for the end of the range + + for (auto bytes_so_far = node_size; range.next;) { + if (range.last + node_size != range.next) { // not continous + // restart at next + range.prev = range.last; + range.first = range.next; + range.last = range.next; + range.next = get_next(range.last); + + bytes_so_far = node_size; + } else { + // extend range + auto new_next = get_next(range.next); + range.last = range.next; + range.next = new_next; + + bytes_so_far += node_size; + if (bytes_so_far >= bytes_needed) { + return range; + } + } + } + // not enough continuous space + return {nullptr, nullptr, nullptr, nullptr}; +} + +// Searches for a range in memory that fits the size required for the allocation. +template +constexpr memory_range xor_find(proxy_range proxy, + ::std::size_t bytes_needed, + ::std::size_t node_size) noexcept { + memory_range range; + range.prev = proxy.begin; + range.first = xor_get_next(proxy.begin, nullptr); + range.last = range.first; // << used as iterator for the end of the range + range.next = xor_get_next(range.last, range.prev); // << used as iterator for the end of the range + + for (auto bytes_so_far = node_size; range.next != proxy.end;) { + if (range.last + node_size != range.next) { // not continous + // restart at next + range.prev = range.last; + range.first = range.next; + range.last = range.next; + range.next = xor_get_next(range.first, range.prev); + + bytes_so_far = node_size; + } else { + // extend range + auto new_next = xor_get_next(range.next, range.last); + range.last = range.next; + range.next = new_next; + + bytes_so_far += node_size; + if (bytes_so_far >= bytes_needed) { + return range; + } + } + } + // not enough continuous space + return {nullptr, nullptr, nullptr, nullptr}; +} + +struct [[nodiscard]] node_to_insert final { + ::std::byte* prev; + ::std::byte* next; +}; + +// Converts a memory block into a list of linked nodes. +constexpr void xor_link_block(void* const memory, + ::std::size_t node_size, + ::std::size_t node_count, + node_to_insert node) noexcept { + auto begin_node = static_cast<::std::byte*>(memory); + xor_exchange(node.prev, node.next, begin_node); + + auto last_node = node.prev; + for (::std::size_t i = 0u; i < node_count - 1u; ++i) { + xor_set_next(begin_node, last_node, begin_node + node_size); + last_node = begin_node; + begin_node += node_size; + } + xor_set_next(begin_node, last_node, node.next); + xor_exchange(node.next, node.prev, begin_node); +} + +// Finds node to insert to keep list ordered. +template +constexpr node_to_insert find_node(allocator_info const& info, + Iterator begin, + memory_range range) noexcept { + FLUX_ASSERT(less(range.first, begin) && less(begin, range.last)); + auto curr_forward = range.first; + auto prev_forward = range.prev; + + auto curr_backward = range.last; + auto prev_backward = range.next; + do { + if (greater(curr_forward, begin)) + return {prev_forward, curr_forward}; + else if (less(curr_backward, begin)) + return {curr_backward, prev_backward}; + + debug_check_double_free( + [&] { return curr_forward != begin && curr_backward != begin; }, info, begin); + xor_advance(curr_forward, prev_forward); + xor_advance(curr_backward, prev_backward); + } while (less(prev_forward, prev_backward)); + + debug_check_double_free([] { return false; }, info, begin); + return {nullptr, nullptr}; +} + +// Finds the node to insert in the entire list. +template +constexpr node_to_insert find_node(allocator_info const& info, + Iterator memory, + proxy_range node, + Iterator last_dealloc, + Iterator last_dealloc_prev) noexcept { + auto first = xor_get_next(node.begin, nullptr); + auto last = xor_get_next(node.end , nullptr); + + if (greater(first, memory)) + // insert at front + return {node.begin, first}; + else if (less(last, memory)) + // insert at the end + return {last, node.end}; + else if (less(last_dealloc_prev, memory) && less(memory, last_dealloc)) + // insert before last_dealloc + return {last_dealloc_prev, last_dealloc}; + else if (less(memory, last_dealloc)) + // insert into [first, last_dealloc_prev] + return find_node(info, memory, {node.begin, first, last_dealloc_prev, last_dealloc}); + else if (greater(memory, last_dealloc)) + // insert into (last_dealloc, last] + return find_node(info, memory, {last_dealloc_prev, last_dealloc, last, node.end}); + + __builtin_unreachable(); + return {nullptr, nullptr}; +} +// clang-format on + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/low_level_allocator_adapter.hpp b/flux-foundation/flux/foundation/memory/detail/low_level_allocator_adapter.hpp new file mode 100644 index 0000000..2d884cd --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/low_level_allocator_adapter.hpp @@ -0,0 +1,69 @@ +#pragma once +#include +#include + +namespace flux::fou::detail { + +// clang-format off +template +struct [[nodiscard]] low_level_allocator_leak_handler { + constexpr void operator()(::std::ptrdiff_t amount) noexcept { + debug_handle_memory_leak(Allocator::info(), amount); + } +}; + +template +concept low_level_allocator = + requires(::std::size_t size, ::std::size_t align) { + // A low-level allocator should have static allocate/deallocate functions... + { Allocator::allocate ( size, align) } noexcept -> meta::same_as; + { Allocator::deallocate(nullptr, size, align) } noexcept -> meta::same_as; + // ...as well as auxiliary utilities. + { Allocator::max_size() } noexcept -> meta::same_as<::std::size_t>; + { Allocator::info() } noexcept -> meta::same_as; + }; +// clang-format on + +template +struct [[nodiscard]] low_level_allocator_adapter + : global_leak_detector> { + using allocator_type = Allocator; + using size_type = typename allocator_type::size_type; + using difference_type = typename allocator_type::difference_type; + using leak_detector = global_leak_detector>; + using stateful = meta::false_type; + + constexpr low_level_allocator_adapter() noexcept = default; + constexpr ~low_level_allocator_adapter() = default; + + constexpr low_level_allocator_adapter(low_level_allocator_adapter&&) noexcept = default; + constexpr low_level_allocator_adapter& + operator=(low_level_allocator_adapter&&) noexcept = default; + + constexpr void* allocate_node(size_type size, size_type alignment) noexcept { + auto actual_size = size + (debug_fence_size ? 2u * max_alignment : 0u); + auto memory = allocator_type::allocate(actual_size, alignment); + + leak_detector::on_allocate(actual_size); + + return debug_fill_new(memory, size, max_alignment); + } + + constexpr void deallocate_node(void* node, size_type size, size_type alignment) noexcept { + auto actual_size = size + (debug_fence_size ? 2u * max_alignment : 0u); + auto memory = debug_fill_free(node, size, max_alignment); + + allocator_type::deallocate(memory, actual_size, alignment); + + leak_detector::on_deallocate(actual_size); + } + + constexpr size_type max_node_size() const noexcept { + return allocator_type::max_size(); + } +}; + +#define FLUX_MEMORY_LL_ALLOCATOR_LEAK_HANDLER(allocator, name) \ + FLUX_MEMORY_GLOBAL_LEAK_DETECTOR(low_level_allocator_leak_handler, name) + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/malloc_allocator.hpp b/flux-foundation/flux/foundation/memory/detail/malloc_allocator.hpp new file mode 100644 index 0000000..44aba3c --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/malloc_allocator.hpp @@ -0,0 +1,64 @@ +#pragma once +#include +#include + +namespace flux::fou::detail { + +// clang-format off +// Low-level allocator. +struct [[nodiscard]] malloc_allocator final { + using size_type = ::std::size_t; + using difference_type = ::std::ptrdiff_t; + + static inline auto info() noexcept { + return allocator_info{"flux::fou::detail::malloc_allocator", nullptr}; + } + +#if __has_cpp_attribute(__gnu__::__returns_nonnull__) + [[__gnu__::__returns_nonnull__]] +#endif + static inline void* allocate(size_type size, size_type) noexcept { + void* memory = +# if __has_builtin(__builtin_malloc) + __builtin_malloc(size); +# else + ::std::malloc(size); +# endif + if (!memory) [[unlikely]] + fast_terminate(); + + return memory; + } + + static inline void* reallocate(void* memory, size_type size) noexcept { + memory = +# if __has_builtin(__builtin_realloc) + __builtin_realloc(memory, size); +# else + ::std::realloc(memory, size); +# endif + if (!memory) [[unlikely]] + terminate(); + + return memory; + } + + static inline void deallocate(void* memory, size_type, size_type) noexcept { + if (!memory) + return; + +# if __has_builtin(__builtin_free) + __builtin_free(memory); +# else + ::std::free(memory); +# endif + } + + static inline size_type max_size() noexcept { + // The maximum size of a user request for memory that can be granted. + return size_type(-1) / sizeof(::std::byte); + } +}; +// clang-format on + +} // namespace flux::fou::detail \ No newline at end of file diff --git a/flux-foundation/flux/foundation/memory/detail/test_allocator.hpp b/flux-foundation/flux/foundation/memory/detail/test_allocator.hpp new file mode 100644 index 0000000..f0bd2c1 --- /dev/null +++ b/flux-foundation/flux/foundation/memory/detail/test_allocator.hpp @@ -0,0 +1,70 @@ +#pragma once +#include + +namespace flux::fou { + +struct memory_info { + void* memory; + ::std::size_t size, align; +}; + +template