From 4895ee4b527b71a0892c6bf8172dc1f3be2e2339 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Tue, 23 Apr 2024 16:46:47 +0800 Subject: [PATCH 1/9] reduce include --- .../secondary_index/common_query_filter.cpp | 14 ++++++++++ .../secondary_index/common_query_filter.cppm | 27 +++++-------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/storage/secondary_index/common_query_filter.cpp b/src/storage/secondary_index/common_query_filter.cpp index 6bf2587a2a..5fe272fd70 100644 --- a/src/storage/secondary_index/common_query_filter.cpp +++ b/src/storage/secondary_index/common_query_filter.cpp @@ -116,6 +116,20 @@ void MergeIntoBitmask(const VectorBuffer *input_bool_column_buffer, } } +CommonQueryFilter::CommonQueryFilter(SharedPtr original_filter, SharedPtr base_table_ref, TxnTimeStamp begin_ts) + : begin_ts_(begin_ts), original_filter_(std::move(original_filter)), base_table_ref_(std::move(base_table_ref)) { + const HashMap &segment_index = base_table_ref_->block_index_->segment_index_; + if (segment_index.empty()) { + finish_build_.test_and_set(std::memory_order_release); + } else { + tasks_.reserve(segment_index.size()); + for (const auto &[segment_id, _] : segment_index) { + tasks_.push_back(segment_id); + } + total_task_num_ = tasks_.size(); + } +} + void CommonQueryFilter::BuildFilter(u32 task_id, TxnTimeStamp begin_ts, BufferManager *buffer_mgr) { const HashMap &segment_index = base_table_ref_->block_index_->segment_index_; const SegmentID segment_id = tasks_[task_id]; diff --git a/src/storage/secondary_index/common_query_filter.cppm b/src/storage/secondary_index/common_query_filter.cppm index 2a13dc046b..654aa3011c 100644 --- a/src/storage/secondary_index/common_query_filter.cppm +++ b/src/storage/secondary_index/common_query_filter.cppm @@ -16,16 +16,15 @@ module; export module common_query_filter; import stl; import bitmask; -import base_expression; -import base_table_ref; -import fast_rough_filter; -import table_index_entry; -import segment_entry; import secondary_index_scan_execute_expression; -import query_context; -import buffer_manager; namespace infinity { +class FastRoughFilterEvaluator; +class BaseTableRef; +class BaseExpression; +class QueryContext; +class BufferManager; +struct TableIndexEntry; export struct CommonQueryFilter { TxnTimeStamp begin_ts_; @@ -56,19 +55,7 @@ export struct CommonQueryFilter { u32 begin_task_num_ = 0; atomic_u32 end_task_num_ = 0; - CommonQueryFilter(SharedPtr original_filter, SharedPtr base_table_ref, TxnTimeStamp begin_ts) - : begin_ts_(begin_ts), original_filter_(std::move(original_filter)), base_table_ref_(std::move(base_table_ref)) { - const HashMap &segment_index = base_table_ref_->block_index_->segment_index_; - if (segment_index.empty()) { - finish_build_.test_and_set(std::memory_order_release); - } else { - tasks_.reserve(segment_index.size()); - for (const auto &[segment_id, _] : segment_index) { - tasks_.push_back(segment_id); - } - total_task_num_ = tasks_.size(); - } - } + CommonQueryFilter(SharedPtr original_filter, SharedPtr base_table_ref, TxnTimeStamp begin_ts); // 1. try to finish building the filter // 2. return true if the filter is available for query From 5824b3494a3d653b9874cbecf5a92ad7d7a2c919 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Mon, 22 Apr 2024 11:30:07 +0800 Subject: [PATCH 2/9] update third party PGM-index version --- third_party/pgm/include/pgm/morton_nd.hpp | 3 +- third_party/pgm/include/pgm/pgm_index.hpp | 129 ++--- .../pgm/include/pgm/pgm_index_dynamic.hpp | 525 ++++++++++-------- .../pgm/include/pgm/pgm_index_variants.hpp | 333 +++++++---- .../include/pgm/piecewise_linear_model.hpp | 194 +++---- third_party/pgm/include/pgm/sdsl.hpp | 104 ++-- third_party/versions | 3 +- 7 files changed, 725 insertions(+), 566 deletions(-) diff --git a/third_party/pgm/include/pgm/morton_nd.hpp b/third_party/pgm/include/pgm/morton_nd.hpp index 58b149dd2e..b6eb9e9d42 100644 --- a/third_party/pgm/include/pgm/morton_nd.hpp +++ b/third_party/pgm/include/pgm/morton_nd.hpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -571,4 +572,4 @@ using MortonNDLutEncoder_3D_32 = MortonNDLutEncoder<3, 10, 10>; using MortonNDLutEncoder_3D_64 = MortonNDLutEncoder<3, 21, 11>; } -#endif \ No newline at end of file +#endif diff --git a/third_party/pgm/include/pgm/pgm_index.hpp b/third_party/pgm/include/pgm/pgm_index.hpp index 53be0b3e36..a2d0163af3 100644 --- a/third_party/pgm/include/pgm/pgm_index.hpp +++ b/third_party/pgm/include/pgm/pgm_index.hpp @@ -15,11 +15,17 @@ #pragma once +#include "piecewise_linear_model.hpp" +#include +#include +#include +#include #include -#include +#include +#include +#include #include -#include -#include "piecewise_linear_model.hpp" +#include namespace pgm { @@ -37,12 +43,12 @@ struct ApproxPos { }; /** - * A space-efficient index that enables fast search operations on a sorted sequence of @c n numbers. + * A space-efficient index that enables fast search operations on a sorted sequence of numbers. * * A search returns a struct @ref ApproxPos containing an approximate position of the sought key in the sequence and - * the bounds of a range of size 2*Epsilon+1 where the sought key is guaranteed to be found if present. - * If the key is not present, the range is guaranteed to contain a key that is not less than (i.e. greater or equal to) - * the sought key, or @c n if no such key is found. + * the bounds of a range where the sought key is guaranteed to be found if present. + * If the key is not present, a @ref std::lower_bound search on the range finds a key that is greater or equal to the + * sought key, if any. * In the case of repeated keys, the index finds the position of the first occurrence of a key. * * The @p Epsilon template parameter should be set according to the desired space-time trade-off. A smaller value @@ -60,7 +66,7 @@ struct ApproxPos { template class PGMIndex { protected: - template + template friend class BucketingPGMIndex; template @@ -72,60 +78,52 @@ class PGMIndex { size_t n; ///< The number of elements this index was built on. K first_key; ///< The smallest element. std::vector segments; ///< The segments composing the index. - std::vector levels_sizes; ///< The number of segment in each level, in reverse order. std::vector levels_offsets; ///< The starting position of each level in segments[], in reverse order. + /// Sentinel value to avoid bounds checking. + static constexpr K sentinel = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() + : std::numeric_limits::max(); + template static void build(RandomIt first, RandomIt last, size_t epsilon, size_t epsilon_recursive, std::vector &segments, - std::vector &levels_sizes, std::vector &levels_offsets) { - auto n = std::distance(first, last); + auto n = (size_t) std::distance(first, last); if (n == 0) return; levels_offsets.push_back(0); segments.reserve(n / (epsilon * epsilon)); - auto ignore_last = *std::prev(last) == std::numeric_limits::max(); // max is reserved for padding - auto last_n = n - ignore_last; - last -= ignore_last; + if (*std::prev(last) == sentinel) + throw std::invalid_argument("The value " + std::to_string(sentinel) + " is reserved as a sentinel."); - auto build_level = [&](auto epsilon, auto in_fun, auto out_fun) { + auto build_level = [&](auto epsilon, auto in_fun, auto out_fun, size_t last_n) { auto n_segments = internal::make_segmentation_par(last_n, epsilon, in_fun, out_fun); - if (last_n > 1 && segments.back().slope == 0) { - // Here we need to ensure that keys > *(last-1) are approximated to a position == prev_level_size - segments.emplace_back(*std::prev(last) + 1, 0, last_n); - ++n_segments; + if (segments.back() == sentinel) + --n_segments; + else { + if (segments.back()(sentinel - 1) < last_n) + segments.emplace_back(*std::prev(last) + 1, 0, last_n); // Ensure keys > last are mapped to last_n + segments.emplace_back(sentinel, 0, last_n); } - segments.emplace_back(last_n); return n_segments; }; // Build first level - auto in_fun = [&](auto i) { - auto x = first[i]; - // Here there is an adjustment for inputs with duplicate keys: at the end of a run of duplicate keys equal - // to x=first[i] such that x+1!=first[i+1], we map the values x+1,...,first[i+1]-1 to their correct rank i - auto flag = i > 0 && i + 1u < (size_t)n && x == first[i - 1] && x != first[i + 1] && x + 1 != first[i + 1]; - return std::pair(x + flag, i); - }; + auto in_fun = [&](auto i) { return K(first[i]); }; auto out_fun = [&](auto cs) { segments.emplace_back(cs); }; - last_n = build_level(epsilon, in_fun, out_fun); - levels_offsets.push_back(levels_offsets.back() + last_n + 1); - levels_sizes.push_back(last_n); + auto last_n = build_level(epsilon, in_fun, out_fun, n); + levels_offsets.push_back(segments.size()); // Build upper levels while (epsilon_recursive && last_n > 1) { auto offset = levels_offsets[levels_offsets.size() - 2]; - auto in_fun_rec = [&](auto i) { return std::pair(segments[offset + i].key, i); }; - last_n = build_level(epsilon_recursive, in_fun_rec, out_fun); - levels_offsets.push_back(levels_offsets.back() + last_n + 1); - levels_sizes.push_back(last_n); + auto in_fun_rec = [&](auto i) { return segments[offset + i].key; }; + last_n = build_level(epsilon_recursive, in_fun_rec, out_fun, last_n); + levels_offsets.push_back(segments.size()); } - - levels_offsets.pop_back(); } /** @@ -135,27 +133,24 @@ class PGMIndex { */ auto segment_for_key(const K &key) const { if constexpr (EpsilonRecursive == 0) { - auto it = std::upper_bound(segments.begin(), segments.begin() + levels_sizes[0], key); - return it == segments.begin() ? it : std::prev(it); + return std::prev(std::upper_bound(segments.begin(), segments.begin() + segments_count(), key)); } - auto it = segments.begin() + levels_offsets.back(); - + auto it = segments.begin() + *(levels_offsets.end() - 2); for (auto l = int(height()) - 2; l >= 0; --l) { auto level_begin = segments.begin() + levels_offsets[l]; auto pos = std::min((*it)(key), std::next(it)->intercept); auto lo = level_begin + PGM_SUB_EPS(pos, EpsilonRecursive + 1); - auto level_size = levels_sizes[l]; static constexpr size_t linear_search_threshold = 8 * 64 / sizeof(Segment); if constexpr (EpsilonRecursive <= linear_search_threshold) { - for (; (size_t)std::distance(level_begin, lo) < level_size && std::next(lo)->key <= key; ++lo) + for (; std::next(lo)->key <= key; ++lo) continue; it = lo; } else { + auto level_size = levels_offsets[l + 1] - levels_offsets[l] - 1; auto hi = level_begin + PGM_ADD_EPS(pos, EpsilonRecursive, level_size); - it = std::upper_bound(lo, hi, key); - it = it == level_begin ? it : std::prev(it); + it = std::prev(std::upper_bound(lo, hi, key)); } } return it; @@ -183,11 +178,10 @@ class PGMIndex { template PGMIndex(RandomIt first, RandomIt last) : n(std::distance(first, last)), - first_key(n ? *first : 0), + first_key(n ? *first : K(0)), segments(), - levels_sizes(), levels_offsets() { - build(first, last, Epsilon, EpsilonRecursive, segments, levels_sizes, levels_offsets); + build(first, last, Epsilon, EpsilonRecursive, segments, levels_offsets); } /** @@ -198,10 +192,7 @@ class PGMIndex { ApproxPos search(const K &key) const { auto k = std::max(first_key, key); auto it = segment_for_key(k); - auto next = std::next(it); - auto pos = (*it)(k); - if (next!=segments.end()) - pos = std::min(pos, next->intercept); + auto pos = std::min((*it)(k), std::next(it)->intercept); auto lo = PGM_SUB_EPS(pos, Epsilon); auto hi = PGM_ADD_EPS(pos, Epsilon, n); return {pos, lo, hi}; @@ -211,48 +202,42 @@ class PGMIndex { * Returns the number of segments in the last level of the index. * @return the number of segments */ - size_t segments_count() const { - return segments.empty() ? 0 : levels_sizes.front(); - } + size_t segments_count() const { return segments.empty() ? 0 : levels_offsets[1] - 1; } /** * Returns the number of levels of the index. * @return the number of levels of the index */ - size_t height() const { - return levels_sizes.size(); - } + size_t height() const { return levels_offsets.size() - 1; } /** * Returns the size of the index in bytes. * @return the size of the index in bytes */ - size_t size_in_bytes() const { - return segments.size() * sizeof(Segment) + levels_sizes.size() * sizeof(size_t) + levels_offsets.size() * sizeof(size_t); - } + size_t size_in_bytes() const { return segments.size() * sizeof(Segment) + levels_offsets.size() * sizeof(size_t); } }; #pragma pack(push, 1) template struct PGMIndex::Segment { - K key; ///< The first key that the segment indexes. - Floating slope; ///< The slope of the segment. - int32_t intercept; ///< The intercept of the segment. + K key; ///< The first key that the segment indexes. + Floating slope; ///< The slope of the segment. + uint32_t intercept; ///< The intercept of the segment. Segment() = default; - Segment(K key, Floating slope, Floating intercept) : key(key), slope(slope), intercept(intercept) {}; - - explicit Segment(size_t n) : key(std::numeric_limits::max()), slope(), intercept(n) {}; + Segment(K key, Floating slope, uint32_t intercept) : key(key), slope(slope), intercept(intercept) {}; explicit Segment(const typename internal::OptimalPiecewiseLinearModel::CanonicalSegment &cs) : key(cs.get_first_x()) { auto[cs_slope, cs_intercept] = cs.get_floating_point_segment(key); if (cs_intercept > std::numeric_limits::max()) - throw std::overflow_error("Change the type of Segment::intercept to int64"); + throw std::overflow_error("Change the type of Segment::intercept to uint64"); + if (cs_intercept < 0) + throw std::overflow_error("Unexpected intercept < 0"); slope = cs_slope; - intercept = std::round(cs_intercept); + intercept = cs_intercept; } friend inline bool operator<(const Segment &s, const K &k) { return s.key < k; } @@ -267,8 +252,12 @@ struct PGMIndex::Segment { * @return the approximate position of the specified key */ inline size_t operator()(const K &k) const { - auto pos = int64_t(slope * (k - key)) + intercept; - return pos > 0 ? size_t(pos) : 0ull; + size_t pos; + if constexpr (std::is_same_v || std::is_same_v) + pos = size_t(slope * double(std::make_unsigned_t(k) - key)); + else + pos = size_t(slope * double(k - key)); + return pos + intercept; } }; diff --git a/third_party/pgm/include/pgm/pgm_index_dynamic.hpp b/third_party/pgm/include/pgm/pgm_index_dynamic.hpp index c22c329e25..1d28f6127c 100644 --- a/third_party/pgm/include/pgm/pgm_index_dynamic.hpp +++ b/third_party/pgm/include/pgm/pgm_index_dynamic.hpp @@ -15,16 +15,20 @@ #pragma once -#include -#include -#include -#include -#include +#include "pgm_index.hpp" +#include +#include +#include #include #include -#include -#include -#include "pgm_index.hpp" +#include +#include +#include +#include +#include +#include +#include +#include namespace pgm { @@ -33,125 +37,89 @@ namespace pgm { * @tparam K the type of a key * @tparam V the type of a value * @tparam PGMType the type of @ref PGMIndex to use in the container - * @tparam MinIndexedLevel the minimum level (of size 2^MinIndexedLevel) on which a @p PGMType index is constructed */ -template, uint8_t MinIndexedLevel = 18> +template> class DynamicPGMIndex { class ItemA; class ItemB; class Iterator; - template> - class DefaultInitAllocator; - - constexpr static uint8_t min_level = 6; ///< 2^min_level-1 is the size of the sorted buffer for new items. - constexpr static uint8_t fully_allocated_levels = std::max(15, min_level + 1); - constexpr static size_t buffer_max_size = (1ull << (min_level + 1)) - 1; + using Item = std::conditional_t || std::is_arithmetic_v, ItemA, ItemB>; + using Level = std::vector; - static_assert(min_level < MinIndexedLevel); - static_assert(fully_allocated_levels > min_level); - static_assert(2 * PGMType::epsilon_value < 1ul << MinIndexedLevel); - - using Item = std::conditional_t, ItemA, ItemB>; - using Level = std::vector>; - - uint8_t used_levels; ///< Equal to 1 + last level whose size is greater than 0, or = min_level if no data. - std::vector levels; ///< (i-min_level)th element is the data array on the ith level. - std::vector pgms; ///< (i-MinIndexedLevel)th element is the index on the ith level. + const uint8_t base; ///< base^i is the maximum size of the ith level. + const uint8_t min_level; ///< Levels 0..min_level are combined into one large level. + const uint8_t min_index_level; ///< Minimum level on which an index is constructed. + size_t buffer_max_size; ///< Size of the combined upper levels, i.e. max_size(0) + ... + max_size(min_level). + uint8_t used_levels; ///< Equal to 1 + last level whose size is greater than 0, or = min_level if no data. + std::vector levels; ///< (i-min_level)th element is the data array at the ith level. + std::vector pgms; ///< (i-min_index_level)th element is the index at the ith level. const Level &level(uint8_t level) const { return levels[level - min_level]; } - const PGMType &pgm(uint8_t level) const { return pgms[level - MinIndexedLevel]; } + const PGMType &pgm(uint8_t level) const { return pgms[level - min_index_level]; } Level &level(uint8_t level) { return levels[level - min_level]; } - PGMType &pgm(uint8_t level) { return pgms[level - MinIndexedLevel]; } - static uint8_t ceil_log2(size_t n) { return n <= 1 ? 0 : sizeof(unsigned long long) * 8 - __builtin_clzll(n - 1); } - - template - static OutIterator merge(In1 first1, In1 last1, In2 first2, In2 last2, OutIterator result) { - while (first1 != last1 && first2 != last2) { - if (*first2 < *first1) { - *result = *first2; - ++first2; - ++result; - } else if (*first1 < *first2) { - *result = *first1; - ++first1; - ++result; - } else if (SkipDeleted && first1->deleted()) { - ++first1; - ++first2; - } else { - *result = *first1; - ++first1; - ++first2; - ++result; - } - } - return std::copy(first2, last2, std::copy(first1, last1, result)); - } + PGMType &pgm(uint8_t level) { return pgms[level - min_index_level]; } + bool has_pgm(uint8_t level) const { return level >= min_index_level; } + size_t max_size(uint8_t level) const { return size_t(1) << (level * ceil_log2(base)); } + uint8_t max_fully_allocated_level() const { return min_level + 2; } + uint8_t ceil_log_base(size_t n) const { return (ceil_log2(n) + ceil_log2(base) - 1) / ceil_log2(base); } + constexpr static uint8_t ceil_log2(size_t n) { return n <= 1 ? 0 : sizeof(long long) * 8 - __builtin_clzll(n - 1); } void pairwise_merge(const Item &new_item, - uint8_t up_to_level, + uint8_t target, size_t size_hint, typename Level::iterator insertion_point) { - auto target = up_to_level + 1; - auto actual_size = size_t(1) << (1 + min_level); - assert((1ull << target) - level(target).size() >= actual_size); - - Level tmp_a(size_hint); + Level tmp_a(size_hint + level(target).size()); Level tmp_b(size_hint + level(target).size()); // Insert new_item in sorted order in the first level - const auto tmp1 = tmp_a.begin(); - const auto tmp2 = tmp_b.begin(); - const auto mod = (up_to_level - min_level) % 2; - - auto it = std::move(levels[0].begin(), insertion_point, mod ? tmp1 : tmp2); - *it = new_item; - ++it; - std::move(insertion_point, levels[0].end(), it); + auto alternate = true; + auto it = std::move(level(min_level).begin(), insertion_point, tmp_a.begin()); + *it++ = new_item; + it = std::move(insertion_point, level(min_level).end(), it); + auto tmp_size = std::distance(tmp_a.begin(), it); // Merge subsequent levels - uint8_t limit = level(target).empty() ? up_to_level : up_to_level + 1; - for (uint8_t i = 1 + min_level; i <= limit; ++i) { - bool alternate = (i - min_level) % 2 == mod; - auto tmp_it = alternate ? tmp1 : tmp2; - auto out_it = alternate ? tmp2 : tmp1; - decltype(out_it) out_end; - - bool can_delete_permanently = i == used_levels - 1; + uint8_t merge_limit = level(target).empty() ? target - 1 : target; + for (uint8_t i = 1 + min_level; i <= merge_limit; ++i, alternate = !alternate) { + auto tmp_begin = (alternate ? tmp_a : tmp_b).begin(); + auto tmp_end = tmp_begin + tmp_size; + auto out_begin = (alternate ? tmp_b : tmp_a).begin(); + decltype(out_begin) out_end; + + auto can_delete_permanently = i == used_levels - 1; if (can_delete_permanently) - out_end = merge(tmp_it, tmp_it + actual_size, level(i).begin(), level(i).end(), out_it); + out_end = merge(tmp_begin, tmp_end, level(i).begin(), level(i).end(), out_begin); else - out_end = merge(tmp_it, tmp_it + actual_size, level(i).begin(), level(i).end(), out_it); - actual_size = std::distance(out_it, out_end); + out_end = merge(tmp_begin, tmp_end, level(i).begin(), level(i).end(), out_begin); + tmp_size = std::distance(out_begin, out_end); // Empty this level and the corresponding index level(i).clear(); - if (i >= fully_allocated_levels) + if (i >= max_fully_allocated_level()) level(i).shrink_to_fit(); - if (i >= MinIndexedLevel) + if (has_pgm(i)) pgm(i) = PGMType(); } - tmp_b.resize(actual_size); - target = std::max(ceil_log2(actual_size), min_level + 1); - levels[0].resize(0); - levels[target - min_level] = std::move(tmp_b); + level(min_level).clear(); + level(target) = std::move(alternate ? tmp_a : tmp_b); + level(target).resize(tmp_size); // Rebuild index, if needed - if (target >= MinIndexedLevel) + if (has_pgm(target)) pgm(target) = PGMType(level(target).begin(), level(target).end()); } void insert(const Item &new_item) { - auto insertion_point = std::lower_bound(levels[0].begin(), levels[0].end(), new_item); - if (insertion_point != levels[0].end() && *insertion_point == new_item) { + auto insertion_point = lower_bound_bl(level(min_level).begin(), level(min_level).end(), new_item); + if (insertion_point != level(min_level).end() && *insertion_point == new_item) { *insertion_point = new_item; return; } - if (levels[0].size() < buffer_max_size) { - levels[0].insert(insertion_point, new_item); + if (level(min_level).size() < buffer_max_size) { + level(min_level).insert(insertion_point, new_item); used_levels = used_levels == min_level ? min_level + 1 : used_levels; return; } @@ -159,22 +127,21 @@ class DynamicPGMIndex { size_t slots_required = buffer_max_size + 1; uint8_t i; for (i = min_level + 1; i < used_levels; ++i) { - size_t slots_left_in_level = (1ull << i) - level(i).size(); + auto slots_left_in_level = max_size(i) - level(i).size(); if (slots_required <= slots_left_in_level) break; slots_required += level(i).size(); } - auto insertion_idx = std::distance(levels[0].begin(), insertion_point); auto need_new_level = i == used_levels; if (need_new_level) { ++used_levels; levels.emplace_back(); - if (i - MinIndexedLevel >= int(pgms.size())) + if (i - min_index_level >= int(pgms.size())) pgms.emplace_back(); } - pairwise_merge(new_item, i - 1, slots_required, levels[0].begin() + insertion_idx); + pairwise_merge(new_item, i, slots_required, insertion_point); } public: @@ -187,27 +154,47 @@ class DynamicPGMIndex { /** * Constructs an empty container. + * @param base determines the size of the ith level as base^i + * @param buffer_level determines the size of level 0, equal to the sum of base^i for i = 0, ..., buffer_level + * @param index_level the minimum level at which an index is constructed to speed up searches */ - DynamicPGMIndex() : used_levels(min_level), levels(32 - min_level), pgms() { + DynamicPGMIndex(uint8_t base = 8, uint8_t buffer_level = 0, uint8_t index_level = 0) + : base(base), + min_level(buffer_level ? buffer_level : ceil_log_base(128) - (base == 2)), + min_index_level(std::max(min_level + 1, index_level ? index_level : ceil_log_base(size_t(1) << 24))), + buffer_max_size(), + used_levels(min_level), + levels(), + pgms() { + if (base < 2 || (base & (base - 1u)) != 0) + throw std::invalid_argument("base must be a power of two"); + + for (auto j = 0; j <= min_level; ++j) + buffer_max_size += max_size(j); + + levels.resize(32 - used_levels); level(min_level).reserve(buffer_max_size); - for (uint8_t i = min_level + 1; i < fully_allocated_levels; ++i) - level(i).reserve(1ull << i); + for (uint8_t i = min_level + 1; i < max_fully_allocated_level(); ++i) + level(i).reserve(max_size(i)); } /** * Constructs the container on the sorted data in the range [first, last). * @tparam Iterator * @param first, last the range containing the sorted elements to be indexed + * @param base determines the size of the ith level as base^i + * @param buffer_level determines the size of level 0, equal to the sum of base^i for i = 0, ..., buffer_level + * @param index_level the minimum level at which an index is constructed to speed up searches */ template - DynamicPGMIndex(Iterator first, Iterator last) { + DynamicPGMIndex(Iterator first, Iterator last, uint8_t base = 8, uint8_t buffer_level = 0, uint8_t index_level = 0) + : DynamicPGMIndex(base, buffer_level, index_level) { size_t n = std::distance(first, last); - used_levels = std::max(ceil_log2(n), min_level) + 1; - levels = decltype(levels)(std::max(used_levels, 32) - min_level + 1); - + used_levels = std::max(ceil_log_base(n), min_level) + 1; + levels.resize(std::max(used_levels, 32) - min_level + 1); level(min_level).reserve(buffer_max_size); - for (uint8_t i = min_level + 1; i < fully_allocated_levels; ++i) - level(i).reserve(1ull << i); + for (uint8_t i = min_level + 1; i < max_fully_allocated_level(); ++i) + level(i).reserve(max_size(i)); if (n == 0) { used_levels = min_level; @@ -227,8 +214,8 @@ class DynamicPGMIndex { } target.resize(std::distance(target.begin(), out)); - if (used_levels - 1 >= MinIndexedLevel) { - pgms = decltype(pgms)(used_levels - MinIndexedLevel); + if (has_pgm(used_levels - 1)) { + pgms = decltype(pgms)(used_levels - min_index_level); pgm(used_levels - 1) = PGMType(target.begin(), target.end()); } } @@ -259,20 +246,76 @@ class DynamicPGMIndex { auto first = level(i).begin(); auto last = level(i).end(); - if (i >= MinIndexedLevel) { + if (has_pgm(i)) { auto range = pgm(i).search(key); first = level(i).begin() + range.lo; last = level(i).begin() + range.hi; } - auto it = std::lower_bound(first, last, key); + auto it = lower_bound_bl(first, last, key); if (it != level(i).end() && it->first == key) - return it->deleted() ? end() : iterator(this, it); + return it->deleted() ? end() : iterator(this, i, it); } return end(); } + /** + * Returns a copy of the elements with key between and including @p lo and @p hi. + * @param lo lower endpoint of the range query + * @param hi upper endpoint of the range query, must be greater than or equal to @p lo + * @return a vector of key-value pairs satisfying the range query + */ + std::vector> range(const K &lo, const K &hi) const { + if (lo > hi) + throw std::invalid_argument("lo > hi"); + + Level tmp_a; + Level tmp_b; + auto alternate = true; + + for (auto i = min_level; i < used_levels; ++i) { + if (level(i).empty()) + continue; + + auto lo_first = level(i).begin(); + auto lo_last = level(i).end(); + auto hi_first = level(i).begin(); + auto hi_last = level(i).end(); + if (has_pgm(i)) { + auto range = pgm(i).search(lo); + lo_first = level(i).begin() + range.lo; + lo_last = level(i).begin() + range.hi; + range = pgm(i).search(hi); + hi_first = level(i).begin() + range.lo; + hi_last = level(i).begin() + range.hi; + } + + auto it_lo = lower_bound_bl(lo_first, lo_last, lo); + auto it_hi = std::upper_bound(std::max(it_lo, hi_first), hi_last, hi); + auto range_size = std::distance(it_lo, it_hi); + if (range_size == 0) + continue; + + auto tmp_size = (alternate ? tmp_a : tmp_b).size(); + (alternate ? tmp_b : tmp_a).resize(tmp_size + range_size); + auto tmp_it = (alternate ? tmp_a : tmp_b).begin(); + auto out_it = (alternate ? tmp_b : tmp_a).begin(); + tmp_size = std::distance(out_it, merge(tmp_it, tmp_it + tmp_size, it_lo, it_hi, out_it)); + (alternate ? tmp_b : tmp_a).resize(tmp_size); + alternate = !alternate; + } + + std::vector> result; + result.reserve((alternate ? tmp_a : tmp_b).size()); + auto first = (alternate ? tmp_a : tmp_b).begin(); + auto last = (alternate ? tmp_a : tmp_b).end(); + for (auto it = first; it != last; ++it) + if (!it->deleted()) + result.emplace_back(it->first, it->second); + return result; + } + /** * Returns an iterator pointing to the first element that is not less than (i.e. greater or equal to) @p key. * @param key key value to compare the elements to @@ -281,7 +324,8 @@ class DynamicPGMIndex { iterator lower_bound(const K &key) const { typename Level::const_iterator lb; auto lb_set = false; - std::unordered_set deleted; + uint8_t lb_level; + std::set deleted; for (auto i = min_level; i < used_levels; ++i) { if (level(i).empty()) @@ -289,25 +333,29 @@ class DynamicPGMIndex { auto first = level(i).begin(); auto last = level(i).end(); - if (i >= MinIndexedLevel) { + if (has_pgm(i)) { auto range = pgm(i).search(key); first = level(i).begin() + range.lo; last = level(i).begin() + range.hi; } - auto it = std::lower_bound(first, last, key); - for (; it != level(i).end() && it->deleted(); ++it) - deleted.emplace(it->first); - - if (it != level(i).end() && it->first >= key && (!lb_set || it->first < lb->first) - && deleted.find(it->first) == deleted.end()) { - lb = it; - lb_set = true; + for (auto it = lower_bound_bl(first, last, key); + it != level(i).end() && (!lb_set || it->first < lb->first); ++it) { + if (it->deleted()) + deleted.emplace(it->first); + else if (deleted.find(it->first) == deleted.end()) { + if (it->first == key) + return iterator(this, i, it); + lb = it; + lb_level = i; + lb_set = true; + break; + } } } if (lb_set) - return iterator(this, lb); + return iterator(this, lb_level, lb); return end(); } @@ -327,7 +375,7 @@ class DynamicPGMIndex { * Returns an iterator to the end. * @return an iterator to the end */ - iterator end() const { return iterator(this, levels.back().end()); } + iterator end() const { return iterator(this, levels.size() - 1, levels.back().end()); } /** * Returns the number of elements with key that compares equal to the specified argument key, which is either 1 @@ -347,7 +395,7 @@ class DynamicPGMIndex { } /** - * Returns the size of the container in bytes. + * Returns the size of the container (data + index structure) in bytes. * @return the size of the container in bytes */ size_t size_in_bytes() const { @@ -370,29 +418,49 @@ class DynamicPGMIndex { private: - // from: https://stackoverflow.com/a/21028912 - template - class DefaultInitAllocator : public std::allocator { - typedef std::allocator_traits a_t; - - public: - template - struct rebind { - using other = DefaultInitAllocator >; - }; - - using A::A; - - template - void construct(U *ptr) noexcept(std::is_nothrow_default_constructible_v) { - ::new(static_cast(ptr)) U; + template + static OutIterator merge(In1 first1, In1 last1, In2 first2, In2 last2, OutIterator result) { + while (first1 != last1 && first2 != last2) { + if (*first2 < *first1) { + if constexpr (Move) *result = std::move(*first2); + else *result = *first2; + ++first2; + ++result; + } else if (*first1 < *first2) { + if constexpr (Move) *result = std::move(*first1); + else *result = *first1; + ++first1; + ++result; + } else if (SkipDeleted && first1->deleted()) { + ++first1; + ++first2; + } else { + if constexpr (Move) *result = std::move(*first1); + else *result = *first1; + ++first1; + ++first2; + ++result; + } } + if constexpr (Move) + return std::move(first2, last2, std::move(first1, last1, result)); + return std::copy(first2, last2, std::copy(first1, last1, result)); + } - template - void construct(U *ptr, Args &&... args) { - a_t::construct(static_cast(*this), ptr, std::forward(args)...); + template + static RandomIt lower_bound_bl(RandomIt first, RandomIt last, const K &x) { + if (first == last) + return first; + auto n = std::distance(first, last); + while (n > 1) { + auto half = n / 2; + __builtin_prefetch(&*(first + half / 2), 0, 0); + __builtin_prefetch(&*(first + half + half / 2), 0, 0); + first = first[half] < x ? first + half : first; + n -= half; } - }; + return first + (*first < x); + } }; namespace internal { @@ -400,20 +468,16 @@ namespace internal { /* LoserTree implementation adapted from Timo Bingmann's https://tlx.github.io and http://stxxl.org, and from * Johannes Singler's http://algo2.iti.uni-karlsruhe.de/singler/mcstl. These three libraries are distributed under the * Boost Software License 1.0. */ -template> +template class LoserTree { using Source = uint8_t; struct Loser { T key; ///< Copy of the current key in the sequence. - bool sup; ///< true iff this is a supremum sentinel. Source source; ///< Index of the sequence. }; - Source ik; ///< Number of nodes. - Source k; ///< Smallest power of 2 greater than ik. - Compare cmp; ///< The comparator object. - bool first_insert; ///< true iff still have to construct keys. + Source k; ///< Smallest power of 2 greater than the number of nodes. std::vector losers; ///< Vector of size 2k containing loser tree nodes. static uint64_t next_pow2(uint64_t x) { @@ -427,7 +491,7 @@ class LoserTree { auto left = init_winner(2 * root); auto right = init_winner(2 * root + 1); - if (losers[right].sup || (!losers[left].sup && !cmp(losers[right].key, losers[left].key))) { + if (losers[right].key >= losers[left].key) { losers[root] = losers[right]; return left; } else { @@ -440,57 +504,39 @@ class LoserTree { LoserTree() = default; - explicit LoserTree(const Source &ik, const Compare &cmp = Compare()) - : ik(ik), - k(next_pow2(ik)), - cmp(cmp), - first_insert(true), - losers(2 * k) { + explicit LoserTree(const Source &ik) : k(next_pow2(ik)), losers(2 * k) { for (auto i = ik - 1u; i < k; ++i) { - losers[i + k].sup = true; + losers[i + k].key = std::numeric_limits::max(); losers[i + k].source = std::numeric_limits::max(); } } /** Returns the index of the sequence with the smallest element. */ - Source min_source() const { return losers[0].source; } + Source min_source() const { + assert(losers[0].source != std::numeric_limits::max()); + return losers[0].source; + } - /** Inserts the initial element of the sequence source. If sup is true, a supremum sentinel is inserted. */ - void insert_start(const T *key_ptr, const Source &source, bool sup) { + /** Inserts the initial element of the sequence source. */ + void insert_start(const T *key_ptr, const Source &source) { Source pos = k + source; assert(pos < losers.size()); - assert(sup == (key_ptr == nullptr)); - - losers[pos].sup = sup; losers[pos].source = source; - - if (first_insert) { - for (auto i = 0u; i < k + k; ++i) - losers[i].key = key_ptr ? *key_ptr : T(); - first_insert = false; - } else - losers[pos].key = key_ptr ? *key_ptr : T(); + losers[pos].key = *key_ptr; } /** Deletes the smallest element and insert a new element in its place. */ - void delete_min_insert(const T *key_ptr, bool sup) { - assert(sup == (key_ptr == nullptr)); + void delete_min_insert(const T *key_ptr) { auto source = losers[0].source; - auto key = key_ptr ? *key_ptr : T(); - auto pos = (k + source) / 2; - - while (pos > 0) { - if ((sup && (!losers[pos].sup || losers[pos].source < source)) - || (!sup && !losers[pos].sup && (cmp(losers[pos].key, key) - || (!cmp(key, losers[pos].key) && losers[pos].source < source)))) { - std::swap(losers[pos].sup, sup); + auto key = key_ptr ? *key_ptr : std::numeric_limits::max(); + + for (auto pos = (k + source) / 2; pos > 0; pos /= 2) { + if (losers[pos].key < key || (key >= losers[pos].key && losers[pos].source < source)) { std::swap(losers[pos].source, source); std::swap(losers[pos].key, key); } - pos /= 2; } - losers[0].sup = sup; losers[0].source = source; losers[0].key = key; } @@ -501,26 +547,32 @@ class LoserTree { } // namespace internal -template -class DynamicPGMIndex::Iterator { +template +class DynamicPGMIndex::Iterator { friend class DynamicPGMIndex; - using internal_iterator = typename Level::const_iterator; - using it_pair = std::pair; - using dynamic_pgm_type = DynamicPGMIndex; + using level_iterator = typename Level::const_iterator; + using dynamic_pgm_type = DynamicPGMIndex; + + struct Cursor { + uint8_t level_number; + level_iterator iterator; + Cursor() = default; + Cursor(uint8_t level_number, const level_iterator iterator) : level_number(level_number), iterator(iterator) {} + }; const dynamic_pgm_type *super; ///< Pointer to the container that is being iterated. - internal_iterator it; ///< Iterator to the current element. + Cursor current; ///< Pair (level number, iterator to the current element). bool initialized; ///< true iff the members tree and iterators have been initialized. uint8_t unconsumed_count; ///< Number of iterators that have not yet reached the end. internal::LoserTree tree; ///< Tournament tree with one leaf for each iterator. - std::vector iterators; ///< Vector with pairs (current, end) iterators. + std::vector iterators; ///< Vector with pairs (level number, iterator). void lazy_initialize() { if (initialized) return; - // For each level create and position an iterator to the first key > it->first + // For each level create and position an iterator to the first key > current iterators.reserve(super->used_levels - super->min_level); for (uint8_t i = super->min_level; i < super->used_levels; ++i) { auto &level = super->level(i); @@ -529,20 +581,20 @@ class DynamicPGMIndex::Iterator { size_t lo = 0; size_t hi = level.size(); - if (i >= MinIndexedLevel) { - auto range = super->pgm(i).search(it->first); + if (super->has_pgm(i)) { + auto range = super->pgm(i).search(current.iterator->first); lo = range.lo; hi = range.hi; } - auto pos = std::upper_bound(level.begin() + lo, level.begin() + hi, *it); + auto pos = std::upper_bound(level.begin() + lo, level.begin() + hi, current.iterator->first); if (pos != level.end()) - iterators.emplace_back(pos, level.end()); + iterators.emplace_back(i, pos); } tree = decltype(tree)(iterators.size()); for (size_t i = 0; i < iterators.size(); ++i) - tree.insert_start(&iterators[i].first->first, i, false); + tree.insert_start(&iterators[i].iterator->first, i); tree.init(); initialized = true; @@ -557,36 +609,36 @@ class DynamicPGMIndex::Iterator { auto step = [&] { auto &it_min = iterators[tree.min_source()]; - auto result = it_min.first; - ++it_min.first; - if (it_min.first == it_min.second) { - tree.delete_min_insert(nullptr, true); + auto level_number = it_min.level_number; + auto result = it_min.iterator; + ++it_min.iterator; + if (it_min.iterator == super->level(level_number).end()) { + tree.delete_min_insert(nullptr); --unconsumed_count; } else - tree.delete_min_insert(&it_min.first->first, false); - return result; + tree.delete_min_insert(&it_min.iterator->first); + return Cursor(level_number, result); }; - internal_iterator tmp_it; + Cursor tmp; do { - tmp_it = step(); - while (unconsumed_count > 0 && iterators[tree.min_source()].first->first == tmp_it->first) + tmp = step(); + while (unconsumed_count > 0 && iterators[tree.min_source()].iterator->first == tmp.iterator->first) step(); - } while (unconsumed_count > 0 && tmp_it->deleted()); + } while (unconsumed_count > 0 && tmp.iterator->deleted()); - if (tmp_it->deleted()) + if (tmp.iterator->deleted()) *this = super->end(); else - it = tmp_it; + current = tmp; } - Iterator() = default; - Iterator(const dynamic_pgm_type *p, const internal_iterator it) - : super(p), it(it), initialized(), unconsumed_count(), tree(), iterators() {}; + Iterator(const dynamic_pgm_type *p, uint8_t level_number, const level_iterator it) + : super(p), current(level_number, it), initialized(), unconsumed_count(), tree(), iterators() {}; public: - using difference_type = ssize_t; + using difference_type = typename decltype(levels)::difference_type; using value_type = const Item; using pointer = const Item *; using reference = const Item &; @@ -599,56 +651,65 @@ class DynamicPGMIndex::Iterator { } Iterator operator++(int) { - Iterator i(it); + Iterator i(current); ++i; return i; } - reference operator*() const { return *it; } - pointer operator->() const { return &*it; }; - bool operator==(const Iterator &rhs) const { return it == rhs.it; } - bool operator!=(const Iterator &rhs) const { return it != rhs.it; } + reference operator*() const { return *current.iterator; } + pointer operator->() const { return &*current.iterator; }; + + bool operator==(const Iterator &rhs) const { + return current.level_number == rhs.current.level_number && current.iterator == rhs.current.iterator; + } + + bool operator!=(const Iterator &rhs) const { return !(*this == rhs); } }; #pragma pack(push, 1) -template -class DynamicPGMIndex::ItemA { - friend class DynamicPGMIndex; - static V tombstone; +template +class DynamicPGMIndex::ItemA { + const static V tombstone; - ItemA() = default; - explicit ItemA(const K &key) : first(key), second(tombstone) {} - explicit ItemA(const K &key, const V &value) : first(key), second(value) {} + template, int> = 0> + static V get_tombstone() { return new std::remove_pointer_t(); } - bool deleted() const { return this->second == tombstone; } + template, int> = 0> + static V get_tombstone() { return std::numeric_limits::max(); } public: K first; V second; + ItemA() { /* do not (default-)initialize for a more efficient std::vector::resize */ } + explicit ItemA(const K &key) : first(key), second(tombstone) {} + explicit ItemA(const K &key, const V &value) : first(key), second(value) { + if (second == tombstone) + throw std::invalid_argument("The specified value is reserved and cannot be used."); + } + operator K() const { return first; } + bool deleted() const { return this->second == tombstone; } }; -template -V DynamicPGMIndex::ItemA::tombstone = new std::remove_pointer_t(); +template +const V DynamicPGMIndex::ItemA::tombstone = get_tombstone(); -template -class DynamicPGMIndex::ItemB { - friend class DynamicPGMIndex; +template +class DynamicPGMIndex::ItemB { bool flag; - ItemB() = default; - explicit ItemB(const K &key) : flag(true), first(key), second() {} - explicit ItemB(const K &key, const V &value) : flag(false), first(key), second(value) {} - - bool deleted() const { return flag; } - public: K first; V second; + ItemB() = default; + explicit ItemB(const K &key) : flag(true), first(key), second() {} + explicit ItemB(const K &key, const V &value) : flag(false), first(key), second(value) {} + operator K() const { return first; } + bool deleted() const { return flag; } }; #pragma pack(pop) diff --git a/third_party/pgm/include/pgm/pgm_index_variants.hpp b/third_party/pgm/include/pgm/pgm_index_variants.hpp index 79037db111..a8d8ff96d3 100644 --- a/third_party/pgm/include/pgm/pgm_index_variants.hpp +++ b/third_party/pgm/include/pgm/pgm_index_variants.hpp @@ -15,22 +15,35 @@ #pragma once -#include -#include -#include -#include -#include +#include "morton_nd.hpp" +#include "piecewise_linear_model.hpp" +#include "pgm_index.hpp" +#include "sdsl.hpp" + #include #include #include -#include "sdsl.hpp" -#include "morton_nd.hpp" -#include "pgm_index.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace pgm { -/** Computes the smallest integral value not less than x / y, where x and y must be positive integers. */ -#define CEIL_UINT_DIV(x, y) ((x) / (y) + ((x) % (y) != 0)) +/** Computes the smallest integral value not less than x / y, where y must be a positive integer. */ +#define CEIL_INT_DIV(x, y) ((x) / (y) + ((x) % (y) > 0)) /** Computes the number of bits needed to store x, that is, 0 if x is 0, 1 + floor(log2(x)) otherwise. */ #define BIT_WIDTH(x) ((x) == 0 ? 0 : 64 - __builtin_clzll(x)) @@ -62,9 +75,13 @@ class CompressedPGMIndex { K first_key; ///< The smallest element in the data. Floating root_slope; ///< The slope of the root segment. int64_t root_intercept; ///< The intercept of the root segment. + size_t root_range; ///< The size of the level below the root segment. std::vector slopes_table; ///< The vector containing the slopes used by the segments in the index. std::vector levels; ///< The levels composing the compressed index. + /// Sentinel value to avoid bounds checking. + static constexpr K sentinel = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() + : std::numeric_limits::max(); using floating_pair = std::pair; using canonical_segment = typename internal::OptimalPiecewiseLinearModel::CanonicalSegment; @@ -96,24 +113,19 @@ class CompressedPGMIndex { std::vector segments; segments.reserve(n / (Epsilon * Epsilon)); - auto ignore_last = *std::prev(last) == std::numeric_limits::max(); // max is reserved for padding - auto last_n = n - ignore_last; - last -= ignore_last; + if (*std::prev(last) == sentinel) + throw std::invalid_argument("The value " + std::to_string(sentinel) + " is reserved as a sentinel."); // Build first level - auto in_fun = [this, first](auto i) { - auto x = first[i]; - auto flag = i > 0 && i + 1u < n && x == first[i - 1] && x != first[i + 1] && x + 1 != first[i + 1]; - return std::pair(x + flag, i); - }; - auto out_fun = [&, this](auto cs) { segments.emplace_back(cs); }; - last_n = internal::make_segmentation_par(last_n, Epsilon, in_fun, out_fun); + auto in_fun = [&](auto i) { return first[i]; }; + auto out_fun = [&](auto cs) { segments.emplace_back(cs); }; + auto last_n = internal::make_segmentation_par(n, Epsilon, in_fun, out_fun); levels_offsets.push_back(levels_offsets.back() + last_n); // Build upper levels while (EpsilonRecursive && last_n > 1) { auto offset = levels_offsets[levels_offsets.size() - 2]; - auto in_fun_rec = [&, this](auto i) { return std::pair(segments[offset + i].get_first_x(), i); }; + auto in_fun_rec = [&](auto i) { return segments[offset + i].get_first_x(); }; last_n = internal::make_segmentation(last_n, EpsilonRecursive, in_fun_rec, out_fun); levels_offsets.push_back(levels_offsets.back() + last_n); } @@ -123,14 +135,16 @@ class CompressedPGMIndex { slopes_table = tmp_table; // Build levels + auto n_levels = levels_offsets.size() - 1; first_key = *first; if constexpr (EpsilonRecursive > 0) { auto root = *std::prev(levels_offsets.end(), 2); std::tie(root_slope, root_intercept) = segments[root].get_floating_point_segment(first_key); + root_range = n_levels == 1 ? n : levels_offsets[n_levels - 1] - levels_offsets[n_levels - 2]; } - levels.reserve(levels_offsets.size() - 2); - for (auto i = EpsilonRecursive == 0 ? 1 : int(levels_offsets.size()) - 2; i > 0; --i) { + levels.reserve(n_levels - 1); + for (int i = EpsilonRecursive == 0 ? 1 : n_levels - 1; i > 0; --i) { auto l = levels_offsets[i - 1]; auto r = levels_offsets[i]; auto prev_level_size = i == 1 ? n : l - levels_offsets[i - 2]; @@ -163,8 +177,7 @@ class CompressedPGMIndex { if constexpr (EpsilonRecursive == 0) { auto &level = levels.front(); auto it = std::upper_bound(level.keys.begin(), level.keys.begin() + level.size(), key); - auto i = std::distance(level.keys.begin(), it); - i = i == 0 ? 0 : i - 1; + auto i = std::distance(level.keys.begin(), it) - 1; auto pos = std::min(level(slopes_table, i, k), level.get_intercept(i + 1)); auto lo = PGM_SUB_EPS(pos, Epsilon); auto hi = PGM_ADD_EPS(pos, Epsilon, n); @@ -172,9 +185,9 @@ class CompressedPGMIndex { } auto p = int64_t(root_slope * (k - first_key)) + root_intercept; - auto pos = std::min(p > 0 ? size_t(p) : 0ull, levels.front().size()); + auto pos = std::min(p > 0 ? size_t(p) : 0ull, root_range); - for (auto &level : levels) { + for (const auto &level : levels) { auto lo = level.keys.begin() + PGM_SUB_EPS(pos, EpsilonRecursive + 1); static constexpr size_t linear_search_threshold = 8 * 64 / sizeof(K); @@ -183,8 +196,7 @@ class CompressedPGMIndex { continue; } else { auto hi = level.keys.begin() + PGM_ADD_EPS(pos, EpsilonRecursive, level.size()); - auto it = std::upper_bound(lo, hi, k); - lo == level.keys.begin() ? it : std::prev(it); + auto it = std::prev(std::upper_bound(lo, hi, k)); } auto i = std::distance(level.keys.begin(), lo); @@ -294,28 +306,27 @@ struct CompressedPGMIndex::CompressedLev keys.emplace_back(it->get_first_x()); if (need_extra_segment) keys.emplace_back(last_key + 1); - keys.emplace_back(std::numeric_limits::max()); + keys.emplace_back(sentinel); // Compress and store intercepts - sdsl::bit_vector intercept_bv(prev_level_size - intercept_offset + 1); - intercept_bv[prev_level_size - intercept_offset] = true; - for (auto it = first_intercept; it != last_intercept; ++it) { - auto idx = std::min(prev_level_size - 1, *it) - intercept_offset; - intercept_bv[idx] = true; - } + auto max_intercept = prev_level_size - intercept_offset + 2; + auto intercepts_count = std::distance(first_intercept, last_intercept) + need_extra_segment + 1; + sdsl::sd_vector_builder builder(max_intercept, intercepts_count); + builder.set(0); + for (auto it = first_intercept + 1; it != last_intercept; ++it) + builder.set(std::clamp(*it, *(it - 1) + 1, prev_level_size - 1) - intercept_offset); if (need_extra_segment) - intercept_bv[prev_level_size + 1 - intercept_offset] = true; - compressed_intercepts = sdsl::sd_vector<>(intercept_bv); + builder.set(max_intercept - 2); + builder.set(max_intercept - 1); + compressed_intercepts = sdsl::sd_vector<>(builder); sdsl::util::init_support(sel1, &compressed_intercepts); // Compress and store slopes_map - size_t i = 0; - size_t map_size = std::distance(first_slope, last_slope) + need_extra_segment; + auto map_size = std::distance(first_slope, last_slope) + need_extra_segment; slopes_map = sdsl::int_vector<>(map_size, 0, sdsl::bits::hi(slopes_table.size() - 1) + 1); - for (auto it = first_slope; it != last_slope; ++it) - slopes_map[i++] = *it; + std::copy(first_slope, last_slope, slopes_map.begin()); if (need_extra_segment) - slopes_map[slopes_map.size() - 1] = slopes_table[*std::prev(last_slope)]; + slopes_map.back() = slopes_table[*std::prev(last_slope)]; } inline size_t operator()(const std::vector &slopes, size_t i, K k) const { @@ -341,51 +352,63 @@ struct CompressedPGMIndex::CompressedLev }; /** - * A simple variant of @ref OneLevelPGMIndex that builds a top-level lookup table to speed up the search on the - * segments. + * A simple variant of @ref OneLevelPGMIndex that uses length reduction to speed up the search on the segments. * - * The size of the top-level table is specified as a constructor argument. Additionally, the @p TopLevelBitSize template - * argument allows to specify the bit-size of the memory cells in the top-level table. It can be set either to a power - * of two or to 0. If set to 0, the bit-size of the cells will be determined dynamically so that the table is - * bit-compressed. + * Segments are assigned to a number of buckets specified with the @p TopLevelSize template argument. A top-level table + * of size @p TopLevelSize stores the position of the first segment in each bucket. The @p TopLevelBitSize template + * argument allows to specify the bit-size of the memory cells in the top-level table. If set to 0, the bit-size of the + * cells will be determined dynamically so that the table is bit-compressed. * * @tparam K the type of the indexed keys * @tparam Epsilon controls the size of the returned search range - * @tparam TopLevelBitSize the bit-size of the cells in the top-level table, must be either 0 or a power of two + * @tparam TopLevelSize the number of cells allocated for the top-level table + * @tparam TopLevelBitSize the bit-size of the cells in the top-level table * @tparam Floating the floating-point type to use for slopes */ -template +template class BucketingPGMIndex { protected: - static_assert(Epsilon > 0); - static_assert(TopLevelBitSize == 0 || (TopLevelBitSize & (TopLevelBitSize - 1u)) == 0); + static_assert(Epsilon > 0 && TopLevelSize > 0); using Segment = typename PGMIndex::Segment; + static constexpr bool pow_two_top_level = (TopLevelSize & (TopLevelSize - 1u)) == 0; size_t n; ///< The number of elements this index was built on. K first_key; ///< The smallest element. + K last_key; ///< The largest element. std::vector segments; ///< The segments composing the index. sdsl::int_vector top_level; ///< The structure on the segment. K step; - template - void build_top_level(RandomIt, RandomIt last, size_t top_level_size) { + void build_top_level() { + // Compute the size of the partitioned universe + size_t actual_top_level_size = TopLevelSize + 2; + if constexpr (pow_two_top_level) { + step = K(1) << (sizeof(K) * CHAR_BIT - BIT_WIDTH(TopLevelSize) + 1); + actual_top_level_size = CEIL_INT_DIV(last_key - first_key, step) + 2; + } else + step = std::max(CEIL_INT_DIV(last_key - first_key, TopLevelSize), 1); + + // Allocate the top-level table auto log_segments = (size_t) BIT_WIDTH(segments.size()); if constexpr (TopLevelBitSize == 0) - top_level = sdsl::int_vector<>(top_level_size, segments.size(), log_segments); + top_level = sdsl::int_vector<>(actual_top_level_size, 0, log_segments); else { - if (log_segments > TopLevelBitSize) - throw std::invalid_argument("The value TopLevelBitSize=" + std::to_string(TopLevelBitSize) + - " is too low. Try to set it to " + std::to_string(TopLevelBitSize << 1)); - top_level = sdsl::int_vector(top_level_size, segments.size(), TopLevelBitSize); + if (TopLevelBitSize < log_segments) + throw std::invalid_argument("TopLevelBitSize must be >=" + std::to_string(log_segments)); + top_level = sdsl::int_vector(actual_top_level_size, 0, TopLevelBitSize); } - step = (size_t) CEIL_UINT_DIV(*std::prev(last), top_level_size); - for (auto i = 0ull, k = 1ull; i < top_level_size - 1; ++i) { - while (k < segments.size() && segments[k].key < (i + 1) * step) + // Fill the top-level table + for (uint64_t i = 1, k = 1; i < actual_top_level_size - 1; ++i) { + K upper_bound; + if (__builtin_mul_overflow(K(i), step, &upper_bound)) + upper_bound = std::numeric_limits::max(); + while (k < segments.size() && (segments[k].key - first_key) < upper_bound) ++k; top_level[i] = k; } + top_level[actual_top_level_size - 1] = segments.size(); } /** @@ -394,9 +417,13 @@ class BucketingPGMIndex { * @return an iterator to the segment responsible for the given key */ auto segment_for_key(const K &key) const { - auto j = std::min(key / step, top_level.size() - 1); - auto first = segments.begin() + (key < step ? 1 : top_level[j - 1]); - auto last = segments.begin() + top_level[j]; + size_t j; + if constexpr (pow_two_top_level) + j = (key - first_key) >> (sizeof(K) * CHAR_BIT - BIT_WIDTH(TopLevelSize) + 1); + else + j = (key - first_key) / step; + auto first = segments.begin() + top_level[j]; + auto last = segments.begin() + top_level[j + 1]; return std::prev(std::upper_bound(first, last, key)); } @@ -410,28 +437,27 @@ class BucketingPGMIndex { BucketingPGMIndex() = default; /** - * Constructs the index on the given sorted vector, with the specified top level size. + * Constructs the index on the given sorted vector. * @param data the vector of keys, must be sorted - * @param top_level_size the number of cells allocated for the top-level table */ - explicit BucketingPGMIndex(const std::vector &data, size_t top_level_size) - : BucketingPGMIndex(data.begin(), data.end(), top_level_size) {} + BucketingPGMIndex(const std::vector &data) : BucketingPGMIndex(data.begin(), data.end()) {} /** - * Constructs the index on the sorted keys in the range [first, last), with the specified top level size. + * Constructs the index on the sorted keys in the range [first, last). * @param first, last the range containing the sorted keys to be indexed - * @param top_level_size the number of cells allocated for the top-level table */ template - BucketingPGMIndex(RandomIt first, RandomIt last, size_t top_level_size) + BucketingPGMIndex(RandomIt first, RandomIt last) : n(std::distance(first, last)), - first_key(n ? *first : 0), + first_key(n ? *first : K(0)), + last_key(n ? *(last - 1) : K(0)), segments(), top_level() { - std::vector sizes; + if (n == 0) + return; std::vector offsets; - PGMIndex::build(first, last, Epsilon, 0, segments, sizes, offsets); - build_top_level(first, last, top_level_size); + PGMIndex::build(first, last, Epsilon, 0, segments, offsets); + build_top_level(); } /** @@ -440,9 +466,12 @@ class BucketingPGMIndex { * @return a struct with the approximate position and bounds of the range */ ApproxPos search(const K &key) const { - auto k = std::max(first_key, key); - auto it = segment_for_key(k); - auto pos = std::min((*it)(k), std::next(it)->intercept); + if (__builtin_expect(key < first_key, 0)) + return {0, 0, 0}; + if (__builtin_expect(key > last_key, 0)) + return {n, n, n}; + auto it = segment_for_key(key); + auto pos = std::min((*it)(key), std::next(it)->intercept); auto lo = PGM_SUB_EPS(pos, Epsilon); auto hi = PGM_ADD_EPS(pos, Epsilon, n); return {pos, lo, hi}; @@ -469,7 +498,7 @@ class BucketingPGMIndex { * @return the size of the index in bytes */ size_t size_in_bytes() const { - return segments.size() * sizeof(Segment) + top_level.size() * top_level.width(); + return segments.size() * sizeof(Segment) + top_level.size() * top_level.width() / CHAR_BIT; } }; @@ -502,11 +531,10 @@ class EliasFanoPGMIndex { } }; - size_t n; ///< The number of elements this index was built on. - K first_key; ///< The smallest segment key. - std::vector segments; ///< The segments composing the index. - sdsl::sd_vector<> ef; ///< The Elias-Fano structure on the segment. - sdsl::sd_vector<>::rank_1_type rank; ///< The rank1 structure. + size_t n; ///< The number of elements this index was built on. + K first_key; ///< The smallest segment key. + std::vector segments; ///< The segments composing the index. + sdsl::sd_vector<> ef; ///< The Elias-Fano structure on the segment. public: @@ -530,16 +558,15 @@ class EliasFanoPGMIndex { template EliasFanoPGMIndex(RandomIt first, RandomIt last) : n(std::distance(first, last)), - first_key(n ? *first : 0), + first_key(n ? *first : K(0)), segments(), ef() { if (n == 0) return; std::vector tmp; - std::vector sizes; std::vector offsets; - PGMIndex::build(first, last, Epsilon, 0, tmp, sizes, offsets); + PGMIndex::build(first, last, Epsilon, 0, tmp, offsets); segments.reserve(tmp.size()); for (auto &x: tmp) { @@ -548,7 +575,6 @@ class EliasFanoPGMIndex { } ef = decltype(ef)(tmp.begin(), std::prev(tmp.end())); - sdsl::util::init_support(rank, &ef); } /** @@ -558,7 +584,7 @@ class EliasFanoPGMIndex { */ ApproxPos search(const K &key) const { auto k = std::max(first_key, key); - auto[r, origin] = rank.pred(k - first_key); + auto[r, origin] = pred(k - first_key); auto pos = std::min(segments[r](origin + first_key, k), segments[r + 1].intercept); auto lo = PGM_SUB_EPS(pos, Epsilon); auto hi = PGM_ADD_EPS(pos, Epsilon, n); @@ -588,6 +614,40 @@ class EliasFanoPGMIndex { size_t size_in_bytes() const { return segments.size() * sizeof(SegmentData) + sdsl::size_in_bytes(ef); } + +private: + + std::pair pred(uint64_t i) const { + if (i > ef.size()) { + auto j = ef.low.size(); + return {j - 1, ef.low[j - 1] + ((ef.high_1_select(j) + 1 - j) << (ef.wl))}; + } + ++i; + + auto high_val = (i >> (ef.wl)); + auto sel_high = ef.high_0_select(high_val + 1); + auto rank_hi = sel_high - high_val; + if (0 == rank_hi) + return {0, ef.low[0] + (high_val << ef.wl)}; + + auto rank_lo = high_val == 0 ? 0 : ef.high_0_select(high_val) - high_val + 1; + auto val_low = i & sdsl::bits::lo_set[ef.wl]; + auto count = rank_hi - rank_lo; + while (count > 0) { + auto step = count / 2; + auto mid = rank_lo + step; + if (ef.low[mid] < val_low) { + rank_lo = mid + 1; + count -= step + 1; + } else { + count = step; + } + } + --rank_lo; + sel_high -= rank_hi - rank_lo; + auto h = ef.high[sel_high] ? high_val : sdsl::bits::prev(ef.high.data(), sel_high) - rank_lo; + return {rank_lo, ef.low[rank_lo] + (h << ef.wl)}; + } }; /** @@ -644,7 +704,7 @@ class MappedPGMIndex : public PGMIndex { auto in_data = map_file(in_filename, in_bytes); this->n = in_bytes / sizeof(K); this->template build(in_data, in_data + this->n, Epsilon, EpsilonRecursive, - this->segments, this->levels_sizes, this->levels_offsets); + this->segments, this->levels_offsets); serialize_and_map(in_data, in_data + this->n, out_filename); unmap_file(in_data, in_bytes); } @@ -662,7 +722,6 @@ class MappedPGMIndex : public PGMIndex { read_member(header_bytes, in); read_member(this->n, in); read_member(this->first_key, in); - read_container(this->levels_sizes, in); read_container(this->levels_offsets, in); read_container(this->segments, in); file_bytes = header_bytes + this->n * sizeof(K); @@ -752,7 +811,6 @@ class MappedPGMIndex : public PGMIndex { header_bytes += write_member(header_bytes, out); header_bytes += write_member(this->n, out); header_bytes += write_member(this->first_key, out); - header_bytes += write_container(this->levels_sizes, out); header_bytes += write_container(this->levels_offsets, out); header_bytes += write_container(this->segments, out); for (auto it = first; it != last; ++it) @@ -817,6 +875,8 @@ class MappedPGMIndex : public PGMIndex { #ifdef MORTON_ND_BMI2_ENABLED +#include + /** * A multidimensional container that uses a @ref PGMIndex for fast orthogonal range queries. * @@ -913,6 +973,87 @@ class MultidimensionalPGMIndex { * @return an iterator pointing to an element inside the query hyperrectangle */ iterator range(const value_type &min, const value_type &max) { return iterator(this, min, max); } + + /** + * (approximate) k-nearest neighbor query. + * Returns @p k nearest points from query point @p p. + * + * @param p the query point. + * @param k the number of nearest points. + * @return a vector of k nearest points. + */ + std::vector knn(const value_type &p, uint32_t k){ + // to access coordinate of point dynamically + using swallow = int[]; + auto sequence = std::make_index_sequence{}; + + // return euclidean distance between given point and query point p + auto dist_from_p = [&](value_type point, std::index_sequence){ + uint64_t squared_sum = 0; + squared_sum = (std::pow((int64_t)std::get(point) - (int64_t)std::get(p), 2) + ... ); + return std::sqrt(squared_sum); + }; + + // return first point of range query for knn query + auto k_range_first = [&](uint64_t dist, std::index_sequence) -> value_type{ + value_type point; + swallow{ + (std::get(point) = std::max((int64_t)std::get(p) - dist, 0), 0)... + }; + return point; + }; + + // return end point of range query for knn query + auto k_range_end = [&](uint64_t dist, std::index_sequence) -> value_type{ + value_type point; + swallow{ + (std::get(point) = std::min(std::get(p) + dist, this->data.size()), 0)... + }; + return point; + }; + + // for debug + auto print_point = [&](value_type point, std::index_sequence){ + std::cout << "("; + swallow{ + (std::cout << std::get(point) << " , ", 0)... + }; + std::cout << ")"; + }; + + // get 2k points around zp to make temporary answer + auto zp = encode(p); + auto range = pgm.search(zp); + auto it = std::lower_bound(data.begin() + range.lo, data.begin() + range.hi, zp); + + std::vector tmp_ans; + for (auto i = it - k >= data.begin() ? it - k : data.begin(); i != it + k && i != data.end(); ++i) + tmp_ans.push_back(morton::Decode(*i)); + + std::sort(tmp_ans.begin(), tmp_ans.end(), [&](auto const& lhs, auto const& rhs) { + double dist_l = dist_from_p(lhs, sequence); + double dist_r = dist_from_p(rhs, sequence); + return dist_l < dist_r; + }); + + // calc distance of k nearest point in tmp_ans, and get a range(hyperrectangle) that contains more than k points around p + uint64_t k_range_dist = dist_from_p(tmp_ans[k - 1], sequence) + 1; + value_type first = k_range_first(k_range_dist, sequence); + value_type end = k_range_end(k_range_dist, sequence); + + // execute range query and get k nearest points + std::vector ans; + for (auto it = this->range(first, end); it != this->end(); ++it) + ans.push_back(*it); + + std::sort(ans.begin(), ans.end(), [&](auto const& lhs, auto const& rhs) { + double dist_l = dist_from_p(lhs, sequence); + double dist_r = dist_from_p(rhs, sequence); + return dist_l < dist_r; + }); + + return std::vector {ans.begin(), ans.begin() + k}; + } private: @@ -923,7 +1064,7 @@ class MultidimensionalPGMIndex { public: using value_type = multidimensional_pgm_type::value_type; - using difference_type = ssize_t; + using difference_type = typename internal_iterator::difference_type; using pointer = const value_type *; using reference = const value_type &; using iterator_category = std::forward_iterator_tag; @@ -967,7 +1108,7 @@ class MultidimensionalPGMIndex { RangeIterator(internal_iterator it) : RangeIterator(it, morton::Decode(*it)) {} - RangeIterator(internal_iterator it, const value_type &p) : it(it), p(p), miss(-1) {} + RangeIterator(internal_iterator it, const value_type &p) : p(p), it(it), miss(-1) {} RangeIterator(const decltype(super) super, const value_type &min, const value_type &max) : super(super), diff --git a/third_party/pgm/include/pgm/piecewise_linear_model.hpp b/third_party/pgm/include/pgm/piecewise_linear_model.hpp index adbc087557..b5e8587e92 100644 --- a/third_party/pgm/include/pgm/piecewise_linear_model.hpp +++ b/third_party/pgm/include/pgm/piecewise_linear_model.hpp @@ -15,41 +15,31 @@ #pragma once +#include #include +#include +#include +#include #include -#include #include #include +#include +#include #ifdef _OPENMP #include #else #pragma message ("Compilation with -fopenmp is optional but recommended") -typedef int omp_int_t; -inline omp_int_t omp_get_max_threads() { return 1; } +#define omp_get_num_procs() 1 +#define omp_get_max_threads() 1 #endif namespace pgm::internal { -#ifdef _MSC_VER - -template -class T_IS_POD -{ - static_assert(sizeof(sizeof(T) < 8), "key_should_be_POD"); -}; - -template -using LargeSigned = typename std::conditional_t, - long double, - std::conditional_t<(sizeof(T) < 8), int64_t, long double>>; -#else - template using LargeSigned = typename std::conditional_t, long double, std::conditional_t<(sizeof(T) < 8), int64_t, __int128>>; -#endif template class OptimalPiecewiseLinearModel { @@ -68,42 +58,16 @@ class OptimalPiecewiseLinearModel { explicit operator long double() const { return dy / (long double) dx; } }; - struct StoredPoint { - X x; - Y y; - }; - struct Point { X x{}; - SY y{}; - - Slope operator-(const Point &p) const { - SX dx = SX(x) - p.x; - return {dx, y - p.y}; - } - }; - - template - struct Hull : private std::vector { - const SY epsilon; - - explicit Hull(SY epsilon) : std::vector(), epsilon(Upper ? epsilon : -epsilon) {} - - Point operator[](size_t i) const { - auto &p = std::vector::operator[](i); - return {p.x, SY(p.y) + epsilon}; - } + Y y{}; - void clear() { std::vector::clear(); } - void resize(size_t n) { std::vector::resize(n); } - void reserve(size_t n) { std::vector::reserve(n); } - size_t size() const { return std::vector::size(); } - void push(X x, Y y) { std::vector::emplace_back(StoredPoint{x, y}); }; + Slope operator-(const Point &p) const { return {SX(x) - p.x, SY(y) - p.y}; } }; const Y epsilon; - Hull lower; - Hull upper; + std::vector lower; + std::vector upper; X first_x = 0; X last_x = 0; size_t lower_start = 0; @@ -114,14 +78,14 @@ class OptimalPiecewiseLinearModel { auto cross(const Point &O, const Point &A, const Point &B) const { auto OA = A - O; auto OB = B - O; - return (OA.dx * OB.dy) - (OA.dy * OB.dx); + return OA.dx * OB.dy - OA.dy * OB.dx; } public: class CanonicalSegment; - explicit OptimalPiecewiseLinearModel(Y epsilon) : epsilon(epsilon), lower(epsilon), upper(epsilon) { + explicit OptimalPiecewiseLinearModel(Y epsilon) : epsilon(epsilon), lower(), upper() { if (epsilon < 0) throw std::invalid_argument("epsilon cannot be negative"); @@ -134,8 +98,10 @@ class OptimalPiecewiseLinearModel { throw std::logic_error("Points must be increasing by x."); last_x = x; - Point p1{x, SY(y) + epsilon}; - Point p2{x, SY(y) - epsilon}; + auto max_y = std::numeric_limits::max(); + auto min_y = std::numeric_limits::lowest(); + Point p1{x, y >= max_y - epsilon ? max_y : y + epsilon}; + Point p2{x, y <= min_y + epsilon ? min_y : y - epsilon}; if (points_in_hull == 0) { first_x = x; @@ -143,8 +109,8 @@ class OptimalPiecewiseLinearModel { rectangle[1] = p2; upper.clear(); lower.clear(); - upper.push(x, y); - lower.push(x, y); + upper.push_back(p1); + lower.push_back(p2); upper_start = lower_start = 0; ++points_in_hull; return true; @@ -153,8 +119,8 @@ class OptimalPiecewiseLinearModel { if (points_in_hull == 1) { rectangle[2] = p2; rectangle[3] = p1; - upper.push(x, y); - lower.push(x, y); + upper.push_back(p1); + lower.push_back(p2); ++points_in_hull; return true; } @@ -174,7 +140,7 @@ class OptimalPiecewiseLinearModel { auto min = lower[lower_start] - p1; auto min_i = lower_start; for (auto i = lower_start + 1; i < lower.size(); i++) { - auto val = (lower[i] - p1); + auto val = lower[i] - p1; if (val > min) break; min = val; @@ -190,7 +156,7 @@ class OptimalPiecewiseLinearModel { for (; end >= upper_start + 2 && cross(upper[end - 2], upper[end - 1], p1) <= 0; --end) continue; upper.resize(end); - upper.push(x, y); + upper.push_back(p1); } if (p2 - rectangle[0] > slope1) { @@ -198,7 +164,7 @@ class OptimalPiecewiseLinearModel { auto max = upper[upper_start] - p2; auto max_i = upper_start; for (auto i = upper_start + 1; i < upper.size(); i++) { - auto val = (upper[i] - p2); + auto val = upper[i] - p2; if (val < max) break; max = val; @@ -214,7 +180,7 @@ class OptimalPiecewiseLinearModel { for (; end >= lower_start + 2 && cross(lower[end - 2], lower[end - 1], p2) >= 0; --end) continue; lower.resize(end); - lower.push(x, y); + lower.push_back(p2); } ++points_in_hull; @@ -276,10 +242,19 @@ class OptimalPiecewiseLinearModel::CanonicalSegment { return {i_x, i_y}; } - std::pair get_floating_point_segment(const X &origin) const { + std::pair get_floating_point_segment(const X &origin) const { if (one_point()) return {0, (rectangle[0].y + rectangle[1].y) / 2}; + if constexpr (std::is_integral_v && std::is_integral_v) { + auto slope = rectangle[3] - rectangle[1]; + auto intercept_n = slope.dy * (SX(origin) - rectangle[1].x); + auto intercept_d = slope.dx; + auto rounding_term = ((intercept_n < 0) ^ (intercept_d < 0) ? -1 : +1) * intercept_d / 2; + auto intercept = (intercept_n + rounding_term) / intercept_d + rectangle[1].y; + return {static_cast(slope), intercept}; + } + auto[i_x, i_y] = get_intersection(); auto[min_slope, max_slope] = get_slope_range(); auto slope = (min_slope + max_slope) / 2.; @@ -298,70 +273,86 @@ class OptimalPiecewiseLinearModel::CanonicalSegment { }; template -size_t make_segmentation(size_t n, size_t epsilon, Fin in, Fout out) { - if (n == 0) - return 0; - - using X = typename std::invoke_result_t::first_type; - using Y = typename std::invoke_result_t::second_type; +size_t make_segmentation(size_t n, size_t start, size_t end, size_t epsilon, Fin in, Fout out) { + using K = typename std::invoke_result_t; size_t c = 0; - size_t start = 0; - auto p = in(0); - - OptimalPiecewiseLinearModel opt(epsilon); - opt.add_point(p.first, p.second); - - for (size_t i = 1; i < n; ++i) { - auto next_p = in(i); - if (i != start && next_p.first == p.first) - continue; - p = next_p; - if (!opt.add_point(p.first, p.second)) { + OptimalPiecewiseLinearModel opt(epsilon); + auto add_point = [&](K x, size_t y) { + if (!opt.add_point(x, y)) { out(opt.get_segment()); - start = i; - --i; + opt.add_point(x, y); ++c; } + }; + + add_point(in(start), start); + for (size_t i = start + 1; i < end - 1; ++i) { + if (in(i) == in(i - 1)) { + // Here there is an adjustment for inputs with duplicate keys: at the end of a run of duplicate keys equal + // to x=in(i) such that x+1!=in(i+1), we map the values x+1,...,in(i+1)-1 to their correct rank i. + // For floating-point keys, the value x+1 above is replaced by the next representable value following x. + if constexpr (std::is_floating_point_v) { + K next; + if ((next = std::nextafter(in(i), std::numeric_limits::infinity())) < in(i + 1)) + add_point(next, i); + } else { + if (in(i) + 1 < in(i + 1)) + add_point(in(i) + 1, i); + } + } else { + add_point(in(i), i); + } + } + if (in(end - 1) != in(end - 2)) + add_point(in(end - 1), end - 1); + + if (end == n) { + // Ensure values greater than the last one are mapped to n + if constexpr (std::is_floating_point_v) { + add_point(std::nextafter(in(n - 1), std::numeric_limits::infinity()), n); + } else { + add_point(in(n - 1) + 1, n); + } } out(opt.get_segment()); return ++c; } +template +size_t make_segmentation(size_t n, size_t epsilon, Fin in, Fout out) { + return make_segmentation(n, 0, n, epsilon, in, out); +} + template size_t make_segmentation_par(size_t n, size_t epsilon, Fin in, Fout out) { - auto parallelism = std::min(omp_get_max_threads(), 20); + auto parallelism = std::min(std::min(omp_get_num_procs(), omp_get_max_threads()), 20); auto chunk_size = n / parallelism; auto c = 0ull; if (parallelism == 1 || n < 1ull << 15) return make_segmentation(n, epsilon, in, out); - using X = typename std::invoke_result_t::first_type; - using Y = typename std::invoke_result_t::second_type; - using canonical_segment = typename OptimalPiecewiseLinearModel::CanonicalSegment; + using K = typename std::invoke_result_t; + using canonical_segment = typename OptimalPiecewiseLinearModel::CanonicalSegment; std::vector> results(parallelism); #pragma omp parallel for reduction(+:c) num_threads(parallelism) -#ifdef _MSC_VER - for (auto i = 0ll; i < parallelism; ++i) { -#else - for (auto i = 0ull; i < parallelism; ++i) { -#endif + for (auto i = 0; i < parallelism; ++i) { auto first = i * chunk_size; auto last = i == parallelism - 1 ? n : first + chunk_size; if (first > 0) { for (; first < last; ++first) - if (in(first).first != in(first - 1).first) + if (in(first) != in(first - 1)) break; if (first == last) continue; } - auto in_fun = [in, first](auto j) { return in(first + j); }; + auto in_fun = [in](auto j) { return in(j); }; auto out_fun = [&results, i](const auto &cs) { results[i].emplace_back(cs); }; results[i].reserve(chunk_size / (epsilon > 0 ? epsilon * epsilon : 16)); - c += make_segmentation(last - first, epsilon, in_fun, out_fun); + c += make_segmentation(n, first, last, epsilon, in_fun, out_fun); } for (auto &v : results) @@ -371,21 +362,4 @@ size_t make_segmentation_par(size_t n, size_t epsilon, Fin in, Fout out) { return c; } -template -auto make_segmentation(RandomIt first, RandomIt last, size_t epsilon) { - using key_type = typename RandomIt::value_type; - using canonical_segment = typename OptimalPiecewiseLinearModel::CanonicalSegment; - using pair_type = typename std::pair; - - size_t n = std::distance(first, last); - std::vector out; - out.reserve(epsilon > 0 ? n / (epsilon * epsilon) : n / 16); - - auto in_fun = [first](auto i) { return pair_type(first[i], i); }; - auto out_fun = [&out](const auto &cs) { out.push_back(cs); }; - make_segmentation(n, epsilon, in_fun, out_fun); - - return out; } - -} \ No newline at end of file diff --git a/third_party/pgm/include/pgm/sdsl.hpp b/third_party/pgm/include/pgm/sdsl.hpp index a5d3599a8d..a038ab669b 100644 --- a/third_party/pgm/include/pgm/sdsl.hpp +++ b/third_party/pgm/include/pgm/sdsl.hpp @@ -10154,16 +10154,13 @@ class sd_vector { sd_vector(const bit_vector& bv) { - m_size = bv.size(); + m_size = bv.size(); size_type m = util::cnt_one_bits(bv); - uint8_t logm = bits::hi(m) + 1; - uint8_t logn = bits::hi(m_size) + 1; - if (logm == logn) { - --logm; // to ensure logn-logm > 0 - } - m_wl = logn - logm; - m_low = int_vector<>(m, 0, m_wl); - bit_vector high = bit_vector(m + (1ULL << logm), 0); // + std::pair params = get_params(m_size, m); + m_wl = params.first; + m_low = int_vector<>(m, 0, params.first); + bit_vector high = bit_vector(params.second, 0); + const uint64_t* bvp = bv.data(); for (size_type i = 0, mm = 0, last_high = 0, highpos = 0; i < (bv.size() + 63) / 64; ++i, ++bvp) { @@ -10198,19 +10195,13 @@ class sd_vector { if (begin == end) { return; } - if (!is_sorted(begin, end)) { - throw std::runtime_error("sd_vector: source list is not sorted."); - } - size_type m = std::distance(begin, end); - m_size = *(end - 1) + 1; - uint8_t logm = bits::hi(m) + 1; - uint8_t logn = bits::hi(m_size) + 1; - if (logm == logn) { - --logm; // to ensure logn-logm > 0 - } - m_wl = logn - logm; - m_low = int_vector<>(m, 0, m_wl); - bit_vector high = bit_vector(m + (1ULL << logm), 0); + size_type m = std::distance(begin, end); + m_size = *(end - 1) + 1; + std::pair params = get_params(m_size, m); + m_wl = params.first; + m_low = int_vector<>(m, 0, params.first); + bit_vector high = bit_vector(params.second, 0); + auto itr = begin; size_type mm = 0, last_high = 0, highpos = 0; while (itr != end) { @@ -10246,6 +10237,37 @@ class sd_vector { builder = sd_vector_builder(); } + // Returns `(low.width(), high.size())`. + // + // This is based on: + // + // Ma, Puglisi, Raman, Zhukova: + // On Elias-Fano for Rank Queries in FM-Indexes. + // DCC 2021. + // + // Implementation credit: Jouni Siren, https://github.com/vgteam/sdsl-lite + static std::pair get_params(size_type universe, size_type ones) + { + size_type low_width = 1; + // Multisets with too many ones will have width 1. + if (ones > 0 && ones <= universe) { + double ideal_width = std::log2((static_cast(universe) * std::log(2.0)) / static_cast(ones)); + low_width = std::round(std::max(ideal_width, 1.0)); + } + size_type buckets = get_buckets(universe, low_width); + return std::pair(low_width, ones + buckets); + } + + // Returns the number of buckets. + static size_type get_buckets(size_type universe, size_type low_width) + { + size_type buckets = universe >> low_width; + if ((universe & bits::lo_set[low_width]) != 0) { + buckets++; + } + return buckets; + } + //! Accessing the i-th element of the original bit_vector /*! \param i An index i with \f$ 0 \leq i < size() \f$. * \return The i-th bit of the original bit_vector @@ -10452,33 +10474,6 @@ class rank_support_sd { return rank_support_sd_trait::adjust_rank(rank_low + 1, i); } - std::pair pred(size_type i) const - { - assert(m_v != nullptr); - if (i > m_v->size()) { - auto j = m_v->low.size(); - return {j - 1, m_v->low[j-1] + ((m_v->high_1_select(j) + 1 - j) << (m_v->wl))}; - } - ++i; - // split problem in two parts: - // (1) find >= - size_type high_val = (i >> (m_v->wl)); - size_type sel_high = m_v->high_0_select(high_val + 1); - size_type rank_low = sel_high - high_val; // - if (0 == rank_low) - return {rank_support_sd_trait::adjust_rank(0, i), m_v->low[0] + (high_val << m_v->wl)}; - size_type val_low = i & bits::lo_set[ m_v->wl ]; - // now since rank_low > 0 => sel_high > 0 - do { - if (!sel_high) { - return {rank_support_sd_trait::adjust_rank(0, i), m_v->low[rank_low] + (high_val < m_v->wl)}; - } - --sel_high; --rank_low; - } while (m_v->high[sel_high] and m_v->low[rank_low] >= val_low); - auto h = m_v->high[sel_high] ? high_val : bits::prev(m_v->high.data(), sel_high) - rank_low; - return {rank_support_sd_trait::adjust_rank(rank_low, i), m_v->low[rank_low] + (h << m_v->wl)}; - } - size_type operator()(size_type i) const { return rank(i); } size_type size() const { return m_v->size(); } @@ -10806,13 +10801,10 @@ inline sd_vector_builder::sd_vector_builder(size_type n, size_type m) "sd_vector_builder: requested capacity is larger than vector size."); } - size_type logm = bits::hi(m_capacity) + 1, logn = bits::hi(m_size) + 1; - if (logm == logn) { - logm--; // to ensure logn-logm > 0 - } - m_wl = logn - logm; - m_low = int_vector<>(m_capacity, 0, m_wl); - m_high = bit_vector(m_capacity + (1ULL << logm), 0); + std::pair params = sd_vector<>::get_params(m_size, m_capacity); + m_wl = params.first; + m_low = int_vector<>(m_capacity, 0, params.first); + m_high = bit_vector(params.second, 0); } template <> diff --git a/third_party/versions b/third_party/versions index 837b928480..2580c48105 100644 --- a/third_party/versions +++ b/third_party/versions @@ -14,4 +14,5 @@ iresearch self-maintained thrift v0.19.0 xor_singleheader v1.0.3 base64 commit-id: d354224643b1b1343cf4944c5cd2ff94e33c3768 -oatpp commit-id: 17ef2a7f6c8a932498799b2a5ae5aab2869975c7 \ No newline at end of file +oatpp commit-id: 17ef2a7f6c8a932498799b2a5ae5aab2869975c7 +PGM-index commit-id: f578e68414c60f9869c5611575143645f75e0ce1 \ No newline at end of file From ece2013df873e290fa96271f473270b0c83ce219 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Mon, 22 Apr 2024 11:46:53 +0800 Subject: [PATCH 3/9] update secondary_index_pgm.cppm update unit test for pgm --- .../secondary_index/secondary_index_pgm.cppm | 14 ---- .../storage/secondary_index/test_pgm.cpp | 77 +++++++++++++++++++ 2 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 src/unit_test/storage/secondary_index/test_pgm.cpp diff --git a/src/storage/secondary_index/secondary_index_pgm.cppm b/src/storage/secondary_index/secondary_index_pgm.cppm index ddb06dbada..28cc7d4f23 100644 --- a/src/storage/secondary_index/secondary_index_pgm.cppm +++ b/src/storage/secondary_index/secondary_index_pgm.cppm @@ -37,7 +37,6 @@ using PGMIndexType = PGMIndex; // size_t n; ///< The number of elements this index was built on. // K first_key; ///< The smallest element. K is the type of key. // std::vector segments; ///< The segments composing the index. -// std::vector levels_sizes; ///< The number of segment in each level, in reverse order. // std::vector levels_offsets; ///< The starting position of each level in segments[], in reverse order. // Segment member objects: @@ -75,13 +74,6 @@ public: this->segments.resize(save_size); file_handler.Read(this->segments.data(), save_size * sizeof(typename decltype(this->segments)::value_type)); } - { - // load std::vector levels_sizes - u32 save_levels_sizes_size; - file_handler.Read(&save_levels_sizes_size, sizeof(save_levels_sizes_size)); - this->levels_sizes.resize(save_levels_sizes_size); - file_handler.Read(this->levels_sizes.data(), save_levels_sizes_size * sizeof(typename decltype(this->levels_sizes)::value_type)); - } { // load std::vector levels_offsets u32 save_levels_offsets_size; @@ -108,12 +100,6 @@ public: file_handler.Write(&save_size, sizeof(save_size)); file_handler.Write(this->segments.data(), save_size * sizeof(typename decltype(this->segments)::value_type)); } - { - // save std::vector levels_sizes - u32 save_levels_sizes_size = this->levels_sizes.size(); - file_handler.Write(&save_levels_sizes_size, sizeof(save_levels_sizes_size)); - file_handler.Write(this->levels_sizes.data(), save_levels_sizes_size * sizeof(typename decltype(this->levels_sizes)::value_type)); - } { // save std::vector levels_offsets u32 save_levels_offsets_size = this->levels_offsets.size(); diff --git a/src/unit_test/storage/secondary_index/test_pgm.cpp b/src/unit_test/storage/secondary_index/test_pgm.cpp new file mode 100644 index 0000000000..b14656c726 --- /dev/null +++ b/src/unit_test/storage/secondary_index/test_pgm.cpp @@ -0,0 +1,77 @@ +// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "unit_test/base_test.h" +#include +#include +import stl; +import third_party; + +using namespace infinity; + +constexpr uint32_t TestNum = 200; + +template +struct PGMT; + +template + requires std::integral || std::same_as +struct PGMT { + using type = PGMIndex; +}; + +template + requires std::same_as +struct PGMT { + using type = PGMIndex; +}; + +template +class TestPGM : public BaseTest { +public: + using PGM = PGMT::type; + std::array data; + uint32_t cnt = TestNum; + std::unique_ptr pgm; + + void SetUp() override { + std::random_device rd; + std::mt19937 gen(rd()); + if constexpr (std::integral) { + std::uniform_int_distribution distrib(std::numeric_limits::lowest(), std::numeric_limits::max() - 1); + std::generate(data.begin(), data.end(), [&] { return distrib(gen); }); + } else if constexpr (std::floating_point) { + std::uniform_real_distribution distrib(std::numeric_limits::lowest() / 2, std::numeric_limits::max() / 2); + std::generate(data.begin(), data.end(), [&] { return distrib(gen); }); + } else { + static_assert(false, "Unsupported type"); + } + std::sort(data.begin(), data.end()); + cnt = std::unique(data.begin(), data.end()) - data.begin(); + pgm = std::make_unique(data.begin(), data.begin() + cnt); + } + void TearDown() override {} +}; + +using TestPGMTypes = ::testing::Types; + +TYPED_TEST_SUITE(TestPGM, TestPGMTypes); + +TYPED_TEST(TestPGM, TestPGMTypeSupport) { + for (uint32_t i = 0; i < this->cnt; ++i) { + auto [pos, lo, hi] = this->pgm->search(this->data[i]); + EXPECT_LE(lo, i); + EXPECT_GE(hi, i); + } +} From b613d9ee958437e9250c63aec32b76a32f735490 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Tue, 23 Apr 2024 20:41:31 +0800 Subject: [PATCH 4/9] fix memory secondary index --- src/executor/operator/physical_index_scan.cpp | 442 +++++++++++------- .../operator/physical_index_scan.cppm | 3 +- .../secondary_index_file_worker.cpp | 114 +++-- .../secondary_index_file_worker.cppm | 60 ++- src/storage/column_vector/bitmask.cpp | 11 + src/storage/column_vector/bitmask.cppm | 1 + src/storage/meta/entry/chunk_index_entry.cpp | 63 +++ src/storage/meta/entry/chunk_index_entry.cppm | 20 +- .../meta/entry/segment_index_entry.cpp | 125 ++--- .../meta/entry/segment_index_entry.cppm | 10 + src/storage/meta/entry/table_entry.cpp | 21 +- src/storage/meta/entry/table_index_entry.cpp | 11 +- .../secondary_index/common_query_filter.cpp | 3 +- .../secondary_index/secondary_index_data.cpp | 379 ++++----------- .../secondary_index/secondary_index_data.cppm | 121 +---- .../secondary_index_in_mem.cpp | 149 ++++++ .../secondary_index_in_mem.cppm | 42 ++ 17 files changed, 878 insertions(+), 697 deletions(-) create mode 100644 src/storage/secondary_index/secondary_index_in_mem.cpp create mode 100644 src/storage/secondary_index/secondary_index_in_mem.cppm diff --git a/src/executor/operator/physical_index_scan.cpp b/src/executor/operator/physical_index_scan.cpp index 9b70d57325..dabab0e1af 100644 --- a/src/executor/operator/physical_index_scan.cpp +++ b/src/executor/operator/physical_index_scan.cpp @@ -14,6 +14,7 @@ module; +#include #include #include @@ -32,9 +33,10 @@ import secondary_index_scan_execute_expression; import logical_type; import table_index_entry; import segment_index_entry; +import chunk_index_entry; +import secondary_index_in_mem; import segment_entry; import fast_rough_filter; -// TODO:use bitset import bitmask; import filter_value_type_classification; @@ -99,7 +101,231 @@ bool PhysicalIndexScan::Execute(QueryContext *query_context, OperatorState *oper return true; } -// TODO: replace bitmask with bitmap +template +struct TrunkReader { + virtual ~TrunkReader() = default; + virtual u32 GetResultCnt(const FilterIntervalRangeT &interval_range) = 0; + virtual void OutPut(std::variant, Bitmask> &selected_rows_) = 0; +}; + +template +struct TrunkReaderT final : public TrunkReader { + using KeyType = ConvertToOrderedType; + static constexpr u32 data_pair_size = sizeof(KeyType) + sizeof(u32); + const u32 segment_row_count_; + SharedPtr chunk_index_entry_; + u32 begin_pos_ = 0; + u32 end_pos_ = 0; + TrunkReaderT(const u32 segment_row_count, const SharedPtr &chunk_index_entry) + : segment_row_count_(segment_row_count), chunk_index_entry_(chunk_index_entry) {} + u32 GetResultCnt(const FilterIntervalRangeT &interval_range) override { + static_assert(std::is_same_v::T>); + BufferHandle index_handle_head = chunk_index_entry_->GetIndex(); + auto index = static_cast(index_handle_head.GetData()); + u32 index_data_num = index->GetChunkRowCount(); + auto [begin_val, end_val] = interval_range.GetRange(); + // 1. search PGM and get approximate search range + // result: + // 1. size_t pos_; ///< The approximate position of the key. + // 2. size_t lower_bound_; ///< The lower bound of the range. + // 3. size_t upper_bound_; ///< The upper bound of the range. + // NOTICE: PGM return a range [lower_bound_, upper_bound_) which must include **one** key when the key exists + // NOTICE: but the range may not include the complete [start, end] range + auto [begin_approx_pos, begin_lower, begin_upper] = index->SearchPGM(&begin_val); + auto [end_approx_pos, end_lower, end_upper] = index->SearchPGM(&end_val); + u32 begin_pos = begin_lower; + u32 end_pos = std::min(end_upper, index_data_num - 1); + if (end_pos < begin_pos) { + return 0; + } + const auto column_data_type = chunk_index_entry_->segment_index_entry_->table_index_entry()->column_def()->type(); + const auto index_part_num = chunk_index_entry_->GetPartNum(); + // 2. find the exact range + // 2.1 find the exact begin_pos which is the first position that index_key >= begin_val + u32 begin_part_id = begin_pos / 8192; + u32 begin_part_offset = begin_pos % 8192; + auto index_handle_b = chunk_index_entry_->GetIndexPartAt(begin_part_id); + auto index_data_b = index_handle_b.GetData(); + auto index_key_b_ptr = [&index_data_b](u32 i) -> KeyType { + KeyType key = {}; + std::memcpy(&key, static_cast(index_data_b) + i * data_pair_size, sizeof(KeyType)); + return key; + }; + auto begin_part_size = chunk_index_entry_->GetPartRowCount(begin_part_id); + if (index_key_b_ptr(begin_part_offset) < begin_val) { + // search forward + while (index_key_b_ptr(begin_part_offset) < begin_val) { + if (++begin_part_offset == begin_part_size) { + if (++begin_part_id >= index_part_num) { + // nothing found + return 0; + } + index_handle_b = chunk_index_entry_->GetIndexPartAt(begin_part_id); + index_data_b = index_handle_b.GetData(); + begin_part_size = chunk_index_entry_->GetPartRowCount(begin_part_id); + begin_part_offset = 0; + } + } + } else { + // search backward + auto test_begin_part_id = begin_part_id; + auto test_begin_part_offset = begin_part_offset; + while (index_key_b_ptr(test_begin_part_offset) >= begin_val) { + // keep valid begin_pos + begin_part_id = test_begin_part_id; + begin_part_offset = test_begin_part_offset; + if (test_begin_part_offset-- == 0) { + if (test_begin_part_id-- == 0) { + // left bound is the leftmost + break; + } + index_handle_b = chunk_index_entry_->GetIndexPartAt(test_begin_part_id); + index_data_b = index_handle_b.GetData(); + begin_part_size = chunk_index_entry_->GetPartRowCount(test_begin_part_id); + test_begin_part_offset = begin_part_size - 1; + } + } + // recover valid pointers + index_handle_b = chunk_index_entry_->GetIndexPartAt(begin_part_id); + index_data_b = index_handle_b.GetData(); + begin_part_size = chunk_index_entry_->GetPartRowCount(begin_part_id); + } + // update begin_pos + begin_pos = begin_part_id * 8192 + begin_part_offset; + // 2.2 find the exact end_pos which is the first position that index_key > end_val (or the position past the end) + u32 end_part_id = end_pos / 8192; + u32 end_part_offset = end_pos % 8192; + auto index_handle_e = chunk_index_entry_->GetIndexPartAt(end_part_id); + auto index_data_e = index_handle_e.GetData(); + auto index_key_e_ptr = [&index_data_e](u32 i) -> KeyType { + KeyType key = {}; + std::memcpy(&key, static_cast(index_data_e) + i * data_pair_size, sizeof(KeyType)); + return key; + }; + auto end_part_size = chunk_index_entry_->GetPartRowCount(end_part_id); + if (index_key_e_ptr(end_part_offset) <= end_val) { + // search forward + while (index_key_e_ptr(end_part_offset) <= end_val) { + if (++end_part_offset == end_part_size) { + if (++end_part_id >= index_part_num) { + // right bound is the rightmost + // recover end_part_id and keep end_part_offset + // they will be used to calculate end_pos + --end_part_id; + break; + } + index_handle_e = chunk_index_entry_->GetIndexPartAt(end_part_id); + index_data_e = index_handle_e.GetData(); + end_part_size = chunk_index_entry_->GetPartRowCount(end_part_id); + end_part_offset = 0; + } + } + } else { + // search backward + auto test_end_part_id = end_part_id; + auto test_end_part_offset = end_part_offset; + while (index_key_e_ptr(test_end_part_offset) > end_val) { + end_part_id = test_end_part_id; + end_part_offset = test_end_part_offset; + if (test_end_part_offset-- == 0) { + if (test_end_part_id-- == 0) { + // nothing found + return 0; + } + index_handle_e = chunk_index_entry_->GetIndexPartAt(test_end_part_id); + index_data_e = index_handle_e.GetData(); + // no need to update end_part_size + test_end_part_offset = chunk_index_entry_->GetPartRowCount(test_end_part_id) - 1; + } + } + // does not need to recover valid values like index_handle_e, index_data_e, index_key_e_ptr, end_part_size + } + // update end_pos + end_pos = end_part_id * 8192 + end_part_offset; + // 3. now we know result size + if (end_pos <= begin_pos) { + // nothing found + return 0; + } + // have result + begin_pos_ = begin_pos; + end_pos_ = end_pos; + const u32 result_size = end_pos - begin_pos; + return result_size; + } + void OutPut(std::variant, Bitmask> &selected_rows_) override { + const u32 begin_pos = begin_pos_; + const u32 end_pos = end_pos_; + const u32 result_size = end_pos - begin_pos; + u32 begin_part_id = begin_pos / 8192; + u32 begin_part_offset = begin_pos % 8192; + auto index_handle_b = chunk_index_entry_->GetIndexPartAt(begin_part_id); + auto index_data_b = index_handle_b.GetData(); + auto index_offset_b_ptr = [&index_data_b](const u32 i) -> u32 { + u32 result = 0; + std::memcpy(&result, static_cast(index_data_b) + i * data_pair_size + sizeof(KeyType), sizeof(u32)); + return result; + }; + auto begin_part_size = chunk_index_entry_->GetPartRowCount(begin_part_id); + // output result + std::visit(Overload{[&](Vector &selected_rows) { + for (u32 i = 0; i < result_size; ++i) { + if (begin_part_offset == begin_part_size) { + index_handle_b = chunk_index_entry_->GetIndexPartAt(++begin_part_id); + index_data_b = index_handle_b.GetData(); + begin_part_size = chunk_index_entry_->GetPartRowCount(begin_part_id); + begin_part_offset = 0; + } + selected_rows.push_back(index_offset_b_ptr(begin_part_offset)); + ++begin_part_offset; + } + }, + [&](Bitmask &bitmask) { + for (u32 i = 0; i < result_size; ++i) { + if (begin_part_offset == begin_part_size) { + index_handle_b = chunk_index_entry_->GetIndexPartAt(++begin_part_id); + index_data_b = index_handle_b.GetData(); + begin_part_size = chunk_index_entry_->GetPartRowCount(begin_part_id); + begin_part_offset = 0; + } + bitmask.SetTrue(index_offset_b_ptr(begin_part_offset)); + ++begin_part_offset; + } + }}, + selected_rows_); + } +}; + +template +struct TrunkReaderM final : public TrunkReader { + using KeyType = ConvertToOrderedType; + const u32 segment_row_count_; + SharedPtr memory_secondary_index_; + Pair, Bitmask>> result_cache_; + TrunkReaderM(const u32 segment_row_count, const SharedPtr &memory_secondary_index) + : segment_row_count_(segment_row_count), memory_secondary_index_(memory_secondary_index) {} + u32 GetResultCnt(const FilterIntervalRangeT &interval_range) override { + auto [begin_val, end_val] = interval_range.GetRange(); + Tuple arg_tuple = {segment_row_count_, begin_val, end_val}; + result_cache_ = memory_secondary_index_->RangeQuery(&arg_tuple); + return result_cache_.first; + } + void OutPut(std::variant, Bitmask> &selected_rows_) override { + std::visit(Overload{[&](Vector &selected_rows, const Vector &result) { + selected_rows.insert(selected_rows.end(), result.begin(), result.end()); + }, + [](Vector &, const Bitmask &) { UnrecoverableError("TrunkReaderM::OutPut(): result count error."); }, + [&](Bitmask &bitmask, const Vector &result) { + for (const auto offset : result) { + bitmask.SetTrue(offset); + } + }, + [&](Bitmask &bitmask, const Bitmask &result) { bitmask.MergeOr(result); }}, + selected_rows_, + result_cache_.second); + } +}; + // Vector: selected rows in segment (used when selected_num <= (segment_row_cnt / 32)), i.e. size(Vector) <= size(Bitmask) // Bitmask: selected rows in segment (used when selected_num > (segment_row_cnt / 32)) struct FilterResult { @@ -227,194 +453,46 @@ struct FilterResult { template inline void - ExecuteSingleRangeT(const FilterIntervalRangeT &interval_range, SegmentIndexEntry &index_entry, SegmentID segment_id) { - using T = FilterIntervalRangeT::T; - BufferHandle index_handle_head = index_entry.GetIndex(); - auto index = static_cast(index_handle_head.GetData()); - auto index_part_capacity = index->GetPartCapacity(); - auto index_part_num = index->GetPartNum(); - auto index_data_num = index->GetDataNum(); - if (index_data_num != SegmentRowActualCount()) { - if (index_data_num < SegmentRowActualCount()) { - UnrecoverableError("FilterResult::ExecuteSingleRange(): index_data_num < SegmentRowActualCount(). index error."); - } else { - LOG_INFO(fmt::format("FilterResult::ExecuteSingleRange(): index_data_num: {}, SegmentRowActualCount(): {}. Some rows are deleted.", - index_data_num, - SegmentRowActualCount())); - } - } - auto [begin_val, end_val] = interval_range.GetRange(); - // 1. search PGM and get approximate search range - // result: - // 1. size_t pos_; ///< The approximate position of the key. - // 2. size_t lower_bound_; ///< The lower bound of the range. - // 3. size_t upper_bound_; ///< The upper bound of the range. - // NOTICE: PGM return a range [lower_bound_, upper_bound_) which must include **one** key when the key exists - // NOTICE: but the range may not include the complete [start, end] range - auto [begin_approx_pos, begin_lower, begin_upper] = index->SearchPGM(&begin_val); - auto [end_approx_pos, end_lower, end_upper] = index->SearchPGM(&end_val); - u32 begin_pos = begin_lower; - u32 end_pos = std::min(end_upper, index_data_num - 1); - if (end_pos < begin_pos) { - return SetEmptyResult(); - } - // 2. find the exact range - // 2.1 find the exact begin_pos which is the first position that index_key >= begin_val - u32 begin_part_id = begin_pos / index_part_capacity; - u32 begin_part_offset = begin_pos % index_part_capacity; - auto index_handle_b = index_entry.GetIndexPartAt(begin_part_id); - auto index_data_b = static_cast(index_handle_b.GetData()); - auto index_key_b_ptr = static_cast(index_data_b->GetColumnKeyData()); - auto begin_part_size = index_data_b->GetPartSize(); - if (index_data_b->GetPartId() != begin_part_id) { - UnrecoverableError("FilterResult::ExecuteSingleRange(): index_data_b->GetPartId() error."); - } - if (index_key_b_ptr[begin_part_offset] < begin_val) { - // search forward - while (index_key_b_ptr[begin_part_offset] < begin_val) { - if (++begin_part_offset == begin_part_size) { - if (++begin_part_id >= index_part_num) { - // nothing found - return SetEmptyResult(); - } - index_handle_b = index_entry.GetIndexPartAt(begin_part_id); - index_data_b = static_cast(index_handle_b.GetData()); - index_key_b_ptr = static_cast(index_data_b->GetColumnKeyData()); - begin_part_size = index_data_b->GetPartSize(); - begin_part_offset = 0; - } - } - } else { - // search backward - auto test_begin_part_id = begin_part_id; - auto test_begin_part_offset = begin_part_offset; - while (index_key_b_ptr[test_begin_part_offset] >= begin_val) { - // keep valid begin_pos - begin_part_id = test_begin_part_id; - begin_part_offset = test_begin_part_offset; - if (test_begin_part_offset-- == 0) { - if (test_begin_part_id-- == 0) { - // left bound is the leftmost - break; - } - index_handle_b = index_entry.GetIndexPartAt(test_begin_part_id); - index_data_b = static_cast(index_handle_b.GetData()); - index_key_b_ptr = static_cast(index_data_b->GetColumnKeyData()); - begin_part_size = index_data_b->GetPartSize(); - test_begin_part_offset = begin_part_size - 1; - } + ExecuteSingleRangeT(const FilterIntervalRangeT &interval_range, SegmentIndexEntry &index_entry, const TxnTimeStamp ts) { + Vector>> trunk_readers; + Tuple>, SharedPtr> chunks_snapshot = index_entry.GetSecondaryIndexSnapshot(); + const u32 segment_row_count = SegmentRowCount(); + auto &[chunk_index_entries, memory_secondary_index] = chunks_snapshot; + for (const auto &chunk_index_entry : chunk_index_entries) { + if (chunk_index_entry->CheckVisible(ts)) { + trunk_readers.emplace_back(MakeUnique>(segment_row_count, chunk_index_entry)); } - if (test_begin_part_id != begin_part_id) { - // recover valid pointers - index_handle_b = index_entry.GetIndexPartAt(begin_part_id); - index_data_b = static_cast(index_handle_b.GetData()); - index_key_b_ptr = static_cast(index_data_b->GetColumnKeyData()); - begin_part_size = index_data_b->GetPartSize(); - } - } - // update begin_pos - begin_pos = begin_part_id * index_part_capacity + begin_part_offset; - // 2.2 find the exact end_pos which is the first position that index_key > end_val (or the position past the end) - u32 end_part_id = end_pos / index_part_capacity; - u32 end_part_offset = end_pos % index_part_capacity; - auto index_handle_e = index_entry.GetIndexPartAt(end_part_id); - auto index_data_e = static_cast(index_handle_e.GetData()); - auto index_key_e_ptr = static_cast(index_data_e->GetColumnKeyData()); - auto end_part_size = index_data_e->GetPartSize(); - if (index_data_e->GetPartId() != end_part_id) { - UnrecoverableError("FilterResult::ExecuteSingleRange(): index_data_e->GetPartId() error."); } - if (index_key_e_ptr[end_part_offset] <= end_val) { - // search forward - while (index_key_e_ptr[end_part_offset] <= end_val) { - if (++end_part_offset == end_part_size) { - if (++end_part_id >= index_part_num) { - // right bound is the rightmost - // recover end_part_id and keep end_part_offset - // they will be used to calculate end_pos - --end_part_id; - break; - } - index_handle_e = index_entry.GetIndexPartAt(end_part_id); - index_data_e = static_cast(index_handle_e.GetData()); - index_key_e_ptr = static_cast(index_data_e->GetColumnKeyData()); - end_part_size = index_data_e->GetPartSize(); - end_part_offset = 0; - } - } - } else { - // search backward - auto test_end_part_id = end_part_id; - auto test_end_part_offset = end_part_offset; - while (index_key_e_ptr[test_end_part_offset] > end_val) { - end_part_id = test_end_part_id; - end_part_offset = test_end_part_offset; - if (test_end_part_offset-- == 0) { - if (test_end_part_id-- == 0) { - // nothing found - return SetEmptyResult(); - } - index_handle_e = index_entry.GetIndexPartAt(test_end_part_id); - index_data_e = static_cast(index_handle_e.GetData()); - index_key_e_ptr = static_cast(index_data_e->GetColumnKeyData()); - // no need to update end_part_size - test_end_part_offset = index_data_e->GetPartSize() - 1; - } - } - // does not need to recover valid values like index_handle_e, index_data_e, index_key_e_ptr, end_part_size + if (memory_secondary_index) { + trunk_readers.emplace_back(MakeUnique>(segment_row_count, memory_secondary_index)); } - // update end_pos - end_pos = end_part_id * index_part_capacity + end_part_offset; - // 3. now we know result size - if (end_pos <= begin_pos) { - // nothing found - return SetEmptyResult(); + u32 result_size = 0; + for (auto &trunk_reader : trunk_readers) { + result_size += trunk_reader->GetResultCnt(interval_range); } - u32 result_size = end_pos - begin_pos; // use array or bitmask for result // use array when result_size <= 1024 or size of array (u32 type) <= size of bitmask - bool use_array = result_size <= 1024 or result_size <= (std::bit_ceil(SegmentRowCount()) / 32); - // 4. output result - auto index_offset_b_ptr = static_cast(index_data_b->GetColumnOffsetData()); + const bool use_array = result_size <= 1024 or result_size <= (std::bit_ceil(SegmentRowCount()) / 32); + // output result if (use_array) { - auto &selected_rows = selected_rows_.emplace>(); - selected_rows.reserve(result_size); - for (u32 i = 0; i < result_size; ++i) { - if (begin_part_offset == begin_part_size) { - index_handle_b = index_entry.GetIndexPartAt(++begin_part_id); - index_data_b = static_cast(index_handle_b.GetData()); - index_offset_b_ptr = static_cast(index_data_b->GetColumnOffsetData()); - begin_part_size = index_data_b->GetPartSize(); - begin_part_offset = 0; - } - selected_rows.emplace_back(index_offset_b_ptr[begin_part_offset]); - ++begin_part_offset; - } - // need to sort - std::sort(selected_rows.begin(), selected_rows.end()); - return; + auto &vec = selected_rows_.emplace>(); + vec.reserve(result_size); } else { auto &bitmask = selected_rows_.emplace(); bitmask.Initialize(std::bit_ceil(SegmentRowCount())); bitmask.SetAllFalse(); - for (u32 i = 0; i < result_size; ++i) { - if (begin_part_offset == begin_part_size) { - index_handle_b = index_entry.GetIndexPartAt(++begin_part_id); - index_data_b = static_cast(index_handle_b.GetData()); - index_offset_b_ptr = static_cast(index_data_b->GetColumnOffsetData()); - begin_part_size = index_data_b->GetPartSize(); - begin_part_offset = 0; - } - bitmask.SetTrue(index_offset_b_ptr[begin_part_offset]); - ++begin_part_offset; - } - return; } + for (auto &trunk_reader : trunk_readers) { + trunk_reader->OutPut(selected_rows_); + } + std::visit(Overload{[](Vector &selected_rows) { std::sort(selected_rows.begin(), selected_rows.end()); }, [](Bitmask &) {}}, + selected_rows_); } inline void ExecuteSingleRange(const HashMap &column_index_map, const FilterExecuteSingleRange &single_range, - SegmentID segment_id) { + SegmentID segment_id, + const TxnTimeStamp ts) { // step 1. check if range is empty if (single_range.IsEmpty()) { return SetEmptyResult(); @@ -426,7 +504,7 @@ struct FilterResult { // step 3. search index auto &interval_range_variant = single_range.GetIntervalRange(); std::visit(Overload{[&](const FilterIntervalRangeT &interval_range) { - ExecuteSingleRangeT(interval_range, index_entry, segment_id); + ExecuteSingleRangeT(interval_range, index_entry, ts); }, [](const std::monostate &empty) { UnrecoverableError("FilterResult::ExecuteSingleRange(): class member interval_range_ not initialized!"); @@ -519,7 +597,8 @@ FilterResult SolveSecondaryIndexFilterInner(const Vector &fil const HashMap &column_index_map, const SegmentID segment_id, const u32 segment_row_count, - const u32 segment_row_actual_count) { + const u32 segment_row_actual_count, + const TxnTimeStamp ts) { Vector result_stack; // execute filter_execute_command_ (Reverse Polish notation) for (auto const &elem : filter_execute_command) { @@ -549,7 +628,7 @@ FilterResult SolveSecondaryIndexFilterInner(const Vector &fil }, [&](const FilterExecuteSingleRange &single_range) { result_stack.emplace_back(segment_row_count, segment_row_actual_count); - result_stack.back().ExecuteSingleRange(column_index_map, single_range, segment_id); + result_stack.back().ExecuteSingleRange(column_index_map, single_range, segment_id, ts); }}, elem); } @@ -564,12 +643,14 @@ std::variant, Bitmask> SolveSecondaryIndexFilter(const Vector &column_index_map, const SegmentID segment_id, const u32 segment_row_count, - const u32 segment_row_actual_count) { + const u32 segment_row_actual_count, + const TxnTimeStamp ts) { if (filter_execute_command.empty()) { // return all true return std::variant, Bitmask>(std::in_place_type); } - auto result = SolveSecondaryIndexFilterInner(filter_execute_command, column_index_map, segment_id, segment_row_count, segment_row_actual_count); + auto result = + SolveSecondaryIndexFilterInner(filter_execute_command, column_index_map, segment_id, segment_row_count, segment_row_actual_count, ts); return std::move(result.selected_rows_); } @@ -627,7 +708,8 @@ void PhysicalIndexScan::ExecuteInternal(QueryContext *query_context, IndexScanOp // prepare filter for deleted rows DeleteFilter delete_filter(segment_entry, begin_ts, segment_entry->row_count(begin_ts)); // output - auto result = SolveSecondaryIndexFilterInner(filter_execute_command_, column_index_map_, segment_id, segment_row_count, segment_row_actual_count); + const auto result = + SolveSecondaryIndexFilterInner(filter_execute_command_, column_index_map_, segment_id, segment_row_count, segment_row_actual_count, begin_ts); result.Output(output_data_blocks, segment_id, delete_filter); LOG_TRACE(fmt::format("IndexScan: job number: {}, segment_ids.size(): {}, finished", next_idx, segment_ids.size())); diff --git a/src/executor/operator/physical_index_scan.cppm b/src/executor/operator/physical_index_scan.cppm index b8156a449f..d96c80a702 100644 --- a/src/executor/operator/physical_index_scan.cppm +++ b/src/executor/operator/physical_index_scan.cppm @@ -109,6 +109,7 @@ export std::variant, Bitmask> SolveSecondaryIndexFilter(const Vector const HashMap &column_index_map, const SegmentID segment_id, const u32 segment_row_count, - const u32 segment_row_actual_count); + const u32 segment_row_actual_count, + const TxnTimeStamp ts); } // namespace infinity diff --git a/src/storage/buffer/file_worker/secondary_index_file_worker.cpp b/src/storage/buffer/file_worker/secondary_index_file_worker.cpp index 8f32fee77e..66c19e7c47 100644 --- a/src/storage/buffer/file_worker/secondary_index_file_worker.cpp +++ b/src/storage/buffer/file_worker/secondary_index_file_worker.cpp @@ -41,17 +41,8 @@ void SecondaryIndexFileWorker::AllocateInMemory() { if (data_) [[unlikely]] { UnrecoverableError("AllocateInMemory: Already allocated."); } else if (auto &data_type = column_def_->type(); data_type->CanBuildSecondaryIndex()) [[likely]] { - if (worker_id_ == 0) { - data_ = static_cast(new SecondaryIndexDataHead(part_capacity_, row_count_, data_type)); - } else { - if (u32 previous_rows = (worker_id_ - 1) * part_capacity_; previous_rows < row_count_) [[likely]] { - auto part_size = std::min(part_capacity_, row_count_ - previous_rows); - data_ = static_cast(new SecondaryIndexDataPart(worker_id_ - 1, part_size)); - } else { - UnrecoverableError(fmt::format("AllocateInMemory: previous_rows: {} >= row_count_: {}.", previous_rows, row_count_)); - } - } - LOG_TRACE(fmt::format("Finished AllocateInMemory() by worker_id: {}.", worker_id_)); + data_ = static_cast(GetSecondaryIndexData(data_type, row_count_, true)); + LOG_TRACE("Finished AllocateInMemory()."); } else { UnrecoverableError(fmt::format("Cannot build secondary index on data type: {}", data_type->ToString())); } @@ -59,15 +50,10 @@ void SecondaryIndexFileWorker::AllocateInMemory() { void SecondaryIndexFileWorker::FreeInMemory() { if (data_) [[likely]] { - if (worker_id_ == 0) { - auto index = static_cast(data_); - delete index; - } else { - auto index = static_cast(data_); - delete index; - } + auto index = static_cast(data_); + delete index; data_ = nullptr; - LOG_TRACE(fmt::format("Finished FreeInMemory() by worker_id: {}, deleted data_ ptr.", worker_id_)); + LOG_TRACE("Finished FreeInMemory(), deleted data_ ptr."); } else { UnrecoverableError("FreeInMemory: Data is not allocated."); } @@ -75,15 +61,10 @@ void SecondaryIndexFileWorker::FreeInMemory() { void SecondaryIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success) { if (data_) [[likely]] { - if (worker_id_ == 0) { - auto index = static_cast(data_); - index->SaveIndexInner(*file_handler_); - } else { - auto index = static_cast(data_); - index->SaveIndexInner(*file_handler_); - } + auto index = static_cast(data_); + index->SaveIndexInner(*file_handler_); prepare_success = true; - LOG_TRACE(fmt::format("Finished WriteToFileImpl(bool &prepare_success) by worker_id: {}.", worker_id_)); + LOG_TRACE("Finished WriteToFileImpl(bool &prepare_success)."); } else { UnrecoverableError("WriteToFileImpl: data_ is nullptr"); } @@ -91,16 +72,75 @@ void SecondaryIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_succ void SecondaryIndexFileWorker::ReadFromFileImpl() { if (!data_) [[likely]] { - if (worker_id_ == 0) { - auto index = new SecondaryIndexDataHead(); - index->ReadIndexInner(*file_handler_); - data_ = static_cast(index); - } else { - auto index = new SecondaryIndexDataPart(); - index->ReadIndexInner(*file_handler_); - data_ = static_cast(index); - } - LOG_TRACE(fmt::format("Finished ReadFromFileImpl() by worker_id: {}.", worker_id_)); + auto index = GetSecondaryIndexData(column_def_->type(), row_count_, false); + index->ReadIndexInner(*file_handler_); + data_ = static_cast(index); + LOG_TRACE("Finished ReadFromFileImpl()."); + } else { + UnrecoverableError("ReadFromFileImpl: data_ is not nullptr"); + } +} + +SecondaryIndexFileWorkerParts::SecondaryIndexFileWorkerParts(SharedPtr file_dir, + SharedPtr file_name, + SharedPtr index_base, + SharedPtr column_def, + u32 row_count, + u32 part_id) + : IndexFileWorker(file_dir, file_name, index_base, column_def), row_count_(row_count), part_id_(part_id) { + data_pair_size_ = GetSecondaryIndexDataPairSize(column_def_->type()); +} + +SecondaryIndexFileWorkerParts::~SecondaryIndexFileWorkerParts() { + if (data_ != nullptr) { + FreeInMemory(); + data_ = nullptr; + } +} + +void SecondaryIndexFileWorkerParts::AllocateInMemory() { + if (row_count_ < part_id_ * 8192) { + UnrecoverableError(fmt::format("AllocateInMemory: row_count_: {} < part_id_ * 8192: {}", row_count_, part_id_ * 8192)); + } + if (data_) [[unlikely]] { + UnrecoverableError("AllocateInMemory: Already allocated."); + } else if (auto &data_type = column_def_->type(); data_type->CanBuildSecondaryIndex()) [[likely]] { + data_ = static_cast(new char[part_row_count_ * data_pair_size_]); + LOG_TRACE("Finished AllocateInMemory()."); + } else { + UnrecoverableError(fmt::format("Cannot build secondary index on data type: {}", data_type->ToString())); + } +} + +void SecondaryIndexFileWorkerParts::FreeInMemory() { + if (data_) [[likely]] { + delete[] static_cast(data_); + data_ = nullptr; + LOG_TRACE("Finished FreeInMemory(), deleted data_ ptr."); + } else { + UnrecoverableError("FreeInMemory: Data is not allocated."); + } +} + +void SecondaryIndexFileWorkerParts::WriteToFileImpl(bool to_spill, bool &prepare_success) { + if (data_) [[likely]] { + file_handler_->Write(data_, part_row_count_ * data_pair_size_); + prepare_success = true; + LOG_TRACE("Finished WriteToFileImpl(bool &prepare_success)."); + } else { + UnrecoverableError("WriteToFileImpl: data_ is nullptr"); + } +} + +void SecondaryIndexFileWorkerParts::ReadFromFileImpl() { + if (row_count_ < part_id_ * 8192) { + UnrecoverableError(fmt::format("ReadFromFileImpl: row_count_: {} < part_id_ * 8192: {}", row_count_, part_id_ * 8192)); + } + if (!data_) [[likely]] { + const u32 read_bytes = part_row_count_ * data_pair_size_; + data_ = static_cast(new char[read_bytes]); + file_handler_->Read(data_, read_bytes); + LOG_TRACE("Finished ReadFromFileImpl()."); } else { UnrecoverableError("ReadFromFileImpl: data_ is not nullptr"); } diff --git a/src/storage/buffer/file_worker/secondary_index_file_worker.cppm b/src/storage/buffer/file_worker/secondary_index_file_worker.cppm index 54a8621e82..e53ae5f325 100644 --- a/src/storage/buffer/file_worker/secondary_index_file_worker.cppm +++ b/src/storage/buffer/file_worker/secondary_index_file_worker.cppm @@ -28,44 +28,60 @@ import column_def; namespace infinity { export struct CreateSecondaryIndexParam : public CreateIndexParam { - // when create index file worker, we should always use the row_count_ - // because the actual_row_count_ will reduce when we delete rows - // which will cause the index file worker count to be inconsistent when we read the index file - const u32 row_count_{}; // rows in the segment, include the deleted rows - const u32 part_capacity_{}; // split sorted index data into parts - CreateSecondaryIndexParam(SharedPtr index_base, SharedPtr column_def, u32 row_count, u32 part_capacity) - : CreateIndexParam(index_base, column_def), row_count_(row_count), part_capacity_(part_capacity) {} + const u32 row_count_{}; // rows in the segment, include the deleted rows + CreateSecondaryIndexParam(SharedPtr index_base, SharedPtr column_def, u32 row_count) + : CreateIndexParam(index_base, column_def), row_count_(row_count) {} }; -// SecondaryIndexFileWorker includes two types of data: -// 1. SecondaryIndexDataHead (when worker_id_ == 0) -// 2. SecondaryIndexDataPart (when worker_id_ > 0) +// pgm index export class SecondaryIndexFileWorker final : public IndexFileWorker { public: explicit SecondaryIndexFileWorker(SharedPtr file_dir, SharedPtr file_name, SharedPtr index_base, SharedPtr column_def, - u32 worker_id, - u32 row_count, - u32 part_capacity) - : IndexFileWorker(file_dir, file_name, index_base, column_def), worker_id_(worker_id), row_count_(row_count), part_capacity_(part_capacity) {} + u32 row_count) + : IndexFileWorker(file_dir, file_name, index_base, column_def), row_count_(row_count) {} - ~SecondaryIndexFileWorker() final; + ~SecondaryIndexFileWorker() override; -public: - void AllocateInMemory() final; + void AllocateInMemory() override; - void FreeInMemory() final; + void FreeInMemory() override; protected: - void WriteToFileImpl(bool to_spill, bool &prepare_success) final; + void WriteToFileImpl(bool to_spill, bool &prepare_success) override; - void ReadFromFileImpl() final; + void ReadFromFileImpl() override; - const u32 worker_id_{}; const u32 row_count_{}; - const u32 part_capacity_{}; +}; + +// row_count * pair +export class SecondaryIndexFileWorkerParts final : public IndexFileWorker { +public: + explicit SecondaryIndexFileWorkerParts(SharedPtr file_dir, + SharedPtr file_name, + SharedPtr index_base, + SharedPtr column_def, + u32 row_count, + u32 part_id); + + ~SecondaryIndexFileWorkerParts() override; + + void AllocateInMemory() override; + + void FreeInMemory() override; + +protected: + void WriteToFileImpl(bool to_spill, bool &prepare_success) override; + + void ReadFromFileImpl() override; + + const u32 row_count_; + const u32 part_id_; + u32 part_row_count_ = std::min(8192, row_count_ - part_id_ * 8192); + u32 data_pair_size_ = 0; }; } // namespace infinity \ No newline at end of file diff --git a/src/storage/column_vector/bitmask.cpp b/src/storage/column_vector/bitmask.cpp index c42bbf5e29..fa00dbac44 100644 --- a/src/storage/column_vector/bitmask.cpp +++ b/src/storage/column_vector/bitmask.cpp @@ -325,4 +325,15 @@ SharedPtr Bitmask::ReadAdv(char *&ptr, i32) { return bitmask; } +Bitmask &Bitmask::operator=(Bitmask &&right) { + if (this != &right) { + data_ptr_ = right.data_ptr_; + buffer_ptr = std::move(right.buffer_ptr); + count_ = right.count_; + right.data_ptr_ = nullptr; + right.count_ = 0; + } + return *this; +} + } // namespace infinity diff --git a/src/storage/column_vector/bitmask.cppm b/src/storage/column_vector/bitmask.cppm index 66acebe6c9..828bc7c014 100644 --- a/src/storage/column_vector/bitmask.cppm +++ b/src/storage/column_vector/bitmask.cppm @@ -77,6 +77,7 @@ public: bool operator==(const Bitmask &other) const; bool operator!=(const Bitmask &other) const { return !(*this == other); } + Bitmask &operator=(Bitmask &&right); // Estimated serialized size in bytes i32 GetSizeInBytes() const; diff --git a/src/storage/meta/entry/chunk_index_entry.cpp b/src/storage/meta/entry/chunk_index_entry.cpp index 5bcacd30d1..5eb9fe891c 100644 --- a/src/storage/meta/entry/chunk_index_entry.cpp +++ b/src/storage/meta/entry/chunk_index_entry.cpp @@ -39,6 +39,8 @@ import buffer_handle; import infinity_exception; import index_defines; import local_file_system; +import secondary_index_file_worker; +import column_def; namespace infinity { @@ -112,6 +114,33 @@ SharedPtr ChunkIndexEntry::NewFtChunkIndexEntry(SegmentIndexEnt return chunk_index_entry; } +SharedPtr ChunkIndexEntry::NewSecondaryIndexChunkIndexEntry(ChunkID chunk_id, + SegmentIndexEntry *segment_index_entry, + const String &base_name, + RowID base_rowid, + u32 row_count, + BufferManager *buffer_mgr) { + auto chunk_index_entry = MakeShared(chunk_id, segment_index_entry, base_name, base_rowid, row_count); + const auto &index_dir = segment_index_entry->index_dir(); + assert(index_dir.get() != nullptr); + if (buffer_mgr != nullptr) { + SegmentID segment_id = segment_index_entry->segment_id(); + auto secondary_index_file_name = MakeShared(IndexFileName(segment_id, chunk_id)); + const auto &index_base = segment_index_entry->table_index_entry()->table_index_def(); + const auto &column_def = segment_index_entry->table_index_entry()->column_def(); + auto file_worker = MakeUnique(index_dir, secondary_index_file_name, index_base, column_def, row_count); + chunk_index_entry->buffer_obj_ = buffer_mgr->AllocateBufferObject(std::move(file_worker)); + const u32 part_cnt = (row_count + 8191) / 8192; + for (u32 i = 0; i < part_cnt; ++i) { + auto part_name = MakeShared(fmt::format("{}_part{}", *secondary_index_file_name, i)); + auto part_file_worker = MakeUnique(index_dir, part_name, index_base, column_def, row_count, i); + BufferObj *part_ptr = buffer_mgr->AllocateBufferObject(std::move(part_file_worker)); + chunk_index_entry->part_buffer_objs_.push_back(part_ptr); + } + } + return chunk_index_entry; +} + SharedPtr ChunkIndexEntry::NewReplayChunkIndexEntry(ChunkID chunk_id, SegmentIndexEntry *segment_index_entry, CreateIndexParam *param, @@ -127,6 +156,15 @@ SharedPtr ChunkIndexEntry::NewReplayChunkIndexEntry(ChunkID chu auto column_length_file_name = MakeShared(base_name + LENGTH_SUFFIX); auto file_worker = MakeUnique(index_dir, column_length_file_name, row_count * sizeof(u32)); chunk_index_entry->buffer_obj_ = buffer_mgr->GetBufferObject(std::move(file_worker)); + } else if (param->index_base_->index_type_ == IndexType::kSecondary) { + const auto &index_dir = segment_index_entry->index_dir(); + SegmentID segment_id = segment_index_entry->segment_id(); + auto secondary_index_file_name = MakeShared(IndexFileName(segment_id, chunk_id)); + const auto &index_base = segment_index_entry->table_index_entry()->table_index_def(); + const auto &column_def = segment_index_entry->table_index_entry()->column_def(); + auto file_worker = MakeUnique(index_dir, secondary_index_file_name, index_base, column_def, row_count); + chunk_index_entry->buffer_obj_ = buffer_mgr->GetBufferObject(std::move(file_worker)); + chunk_index_entry->LoadPartsReader(buffer_mgr); } else { const auto &index_dir = segment_index_entry->index_dir(); const auto &index_base = param->index_base_; @@ -171,6 +209,9 @@ void ChunkIndexEntry::Cleanup() { if (buffer_obj_) { buffer_obj_->PickForCleanup(); } + for (BufferObj *part_buffer_obj : part_buffer_objs_) { + part_buffer_obj->PickForCleanup(); + } TableIndexEntry *table_index_entry = segment_index_entry_->table_index_entry(); const auto &index_dir = segment_index_entry_->index_dir(); const IndexBase *index_base = table_index_entry->index_base(); @@ -194,6 +235,28 @@ void ChunkIndexEntry::SaveIndexFile() { return; } buffer_obj_->Save(); + for (BufferObj *part_buffer_obj : part_buffer_objs_) { + part_buffer_obj->Save(); + } } +void ChunkIndexEntry::LoadPartsReader(BufferManager *buffer_mgr) { + const auto &index_dir = segment_index_entry_->index_dir(); + SegmentID segment_id = segment_index_entry_->segment_id(); + String secondary_index_file_name = IndexFileName(segment_id, chunk_id_); + const auto &index_base = segment_index_entry_->table_index_entry()->table_index_def(); + const auto &column_def = segment_index_entry_->table_index_entry()->column_def(); + const u32 part_cnt = (row_count_ + 8191) / 8192; + part_buffer_objs_.clear(); + part_buffer_objs_.reserve(part_cnt); + for (u32 i = 0; i < part_cnt; ++i) { + auto part_name = MakeShared(fmt::format("{}_part{}", secondary_index_file_name, i)); + auto part_file_worker = MakeUnique(index_dir, std::move(part_name), index_base, column_def, row_count_, i); + BufferObj *part_ptr = buffer_mgr->GetBufferObject(std::move(part_file_worker)); + part_buffer_objs_.push_back(part_ptr); + } +} + +BufferHandle ChunkIndexEntry::GetIndexPartAt(u32 i) { return part_buffer_objs_.at(i)->Load(); } + } // namespace infinity diff --git a/src/storage/meta/entry/chunk_index_entry.cppm b/src/storage/meta/entry/chunk_index_entry.cppm index a93c4a3be0..ff2bf9ec19 100644 --- a/src/storage/meta/entry/chunk_index_entry.cppm +++ b/src/storage/meta/entry/chunk_index_entry.cppm @@ -14,13 +14,13 @@ module; -#include "type/complex/row_id.h" #include export module chunk_index_entry; import stl; import third_party; +import internal_types; import base_entry; import meta_entry_interface; import cleanup_scanner; @@ -65,6 +65,13 @@ public: static SharedPtr NewFtChunkIndexEntry(SegmentIndexEntry *segment_index_entry, const String &base_name, RowID base_rowid, u32 row_count, BufferManager *buffer_mgr); + static SharedPtr NewSecondaryIndexChunkIndexEntry(ChunkID chunk_id, + SegmentIndexEntry *segment_index_entry, + const String &base_name, + RowID base_rowid, + u32 row_count, + BufferManager *buffer_mgr); + static SharedPtr NewReplayChunkIndexEntry(ChunkID chunk_id, SegmentIndexEntry *segment_index_entry, CreateIndexParam *param, @@ -77,8 +84,16 @@ public: void SetRowCount(u32 row_count) { row_count_ = row_count; } + u32 GetRowCount() const { return row_count_; } + + inline u32 GetPartNum() const { return (row_count_ + 8191) / 8192; } + + inline u32 GetPartRowCount(const u32 part_id) const { return std::min(8192, row_count_ - part_id * 8192); } + BufferHandle GetIndex(); + BufferHandle GetIndexPartAt(u32 i); + nlohmann::json Serialize(); static SharedPtr @@ -90,6 +105,8 @@ public: void SaveIndexFile(); + void LoadPartsReader(BufferManager *buffer_mgr); + BufferObj *GetBufferObj() { return buffer_obj_; } void DeprecateChunk(TxnTimeStamp commit_ts) { @@ -120,6 +137,7 @@ public: private: BufferObj *buffer_obj_{}; + Vector part_buffer_objs_; }; } // namespace infinity diff --git a/src/storage/meta/entry/segment_index_entry.cpp b/src/storage/meta/entry/segment_index_entry.cpp index 36325b2456..58c42c7241 100644 --- a/src/storage/meta/entry/segment_index_entry.cpp +++ b/src/storage/meta/entry/segment_index_entry.cpp @@ -14,7 +14,6 @@ module; -#include "type/complex/row_id.h" #include #include #include @@ -58,6 +57,7 @@ import chunk_index_entry; import abstract_hnsw; import block_column_iter; import txn_store; +import secondary_index_in_mem; namespace infinity { @@ -162,9 +162,16 @@ Vector> SegmentIndexEntry::CreateFileWorkers(SharedPt // reference file_worker will be invalidated when vector_file_worker is resized const auto index_base = param->index_base_; const auto column_def = param->column_def_; - if (index_base->index_type_ == IndexType::kFullText || index_base->index_type_ == IndexType::kHnsw) { - // fulltext doesn't use BufferManager - return vector_file_worker; + switch (index_base->index_type_) { + case IndexType::kHnsw: + case IndexType::kFullText: + case IndexType::kSecondary: { + // these indexes don't use BufferManager + return vector_file_worker; + } + default: { + break; + } } auto file_name = MakeShared(IndexFileName(segment_id)); @@ -186,24 +193,6 @@ Vector> SegmentIndexEntry::CreateFileWorkers(SharedPt } break; } - case IndexType::kSecondary: { - auto create_secondary_param = static_cast(param); - auto const row_count = create_secondary_param->row_count_; - auto const part_capacity = create_secondary_param->part_capacity_; - // now we can only use row_count to calculate the part_num - // because the actual_row_count will reduce when we delete rows - // consider the timestamp, actual_row_count may be less than, equal to or greater than rows we can actually read - u32 part_num = (row_count + part_capacity - 1) / part_capacity; - vector_file_worker.resize(part_num + 1); - // cannot use invalid file_worker - vector_file_worker[0] = MakeUnique(index_dir, file_name, index_base, column_def, 0, row_count, part_capacity); - for (u32 i = 1; i <= part_num; ++i) { - auto part_file_name = MakeShared(fmt::format("{}_part{}", *file_name, i)); - vector_file_worker[i] = - MakeUnique(index_dir, part_file_name, index_base, column_def, i, row_count, part_capacity); - } - break; - } default: { UniquePtr err_msg = MakeUnique(fmt::format("File worker isn't implemented: {}", IndexInfo::IndexTypeToString(index_base->index_type_))); @@ -313,8 +302,17 @@ void SegmentIndexEntry::MemIndexInsert(SharedPtr block_entry, memory_hnsw_indexer_->SetRowCount(row_cnt); break; } - case IndexType::kIVFFlat: case IndexType::kSecondary: { + if (memory_secondary_index_.get() == nullptr) { + std::unique_lock lck(rw_locker_); + memory_secondary_index_ = SecondaryIndexInMem::NewSecondaryIndexInMem(column_def, begin_row_id); + } + auto block_id = block_entry->block_id(); + BlockColumnEntry *block_column_entry = block_entry->GetColumnBlockEntry(column_id); + memory_secondary_index_->Insert(block_id, block_column_entry, buffer_manager, row_offset, row_count); + break; + } + case IndexType::kIVFFlat: { UniquePtr err_msg = MakeUnique(fmt::format("{} realtime index is not supported yet", IndexInfo::IndexTypeToString(index_base->index_type_))); LOG_WARN(*err_msg); @@ -363,6 +361,17 @@ SharedPtr SegmentIndexEntry::MemIndexDump(bool spill) { } return chunk_index_entry; } + case IndexType::kSecondary: { + if (memory_secondary_index_.get() == nullptr) { + return nullptr; + } + std::unique_lock lck(rw_locker_); + SharedPtr chunk_index_entry = memory_secondary_index_->Dump(this, buffer_manager_); + chunk_index_entry->SaveIndexFile(); + chunk_index_entries_.push_back(chunk_index_entry); + memory_secondary_index_.reset(); + return chunk_index_entry; + } default: { return nullptr; } @@ -388,7 +397,23 @@ void SegmentIndexEntry::MemIndexLoad(const String &base_name, RowID base_row_id) memory_indexer_->Load(); } -u32 SegmentIndexEntry::MemIndexRowCount() { return memory_indexer_.get() == nullptr ? 0 : memory_indexer_->GetDocCount(); } +u32 SegmentIndexEntry::MemIndexRowCount() { + const IndexBase *index_base = table_index_entry_->index_base(); + switch (index_base->index_type_) { + case IndexType::kFullText: { + return memory_indexer_.get() ? memory_indexer_->GetDocCount() : 0; + } + case IndexType::kHnsw: { + return memory_hnsw_indexer_.get() ? memory_hnsw_indexer_->GetRowCount() : 0; + } + case IndexType::kSecondary: { + return memory_secondary_index_.get() ? memory_secondary_index_->GetRowCount() : 0; + } + default: { + return 0; + } + } +} void SegmentIndexEntry::PopulateEntirely(const SegmentEntry *segment_entry, Txn *txn, const PopulateEntireConfig &config) { TxnTimeStamp begin_ts = txn->BeginTS(); @@ -483,8 +508,21 @@ void SegmentIndexEntry::PopulateEntirely(const SegmentEntry *segment_entry, Txn } break; } - case IndexType::kIVFFlat: - case IndexType::kSecondary: { // TODO + case IndexType::kSecondary: { + u32 seg_id = segment_entry->segment_id(); + RowID base_row_id(seg_id, 0); + memory_secondary_index_ = SecondaryIndexInMem::NewSecondaryIndexInMem(column_def, base_row_id); + u64 column_id = column_def->id(); + auto block_entry_iter = BlockEntryIter(segment_entry); + for (const auto *block_entry = block_entry_iter.Next(); block_entry != nullptr; block_entry = block_entry_iter.Next()) { + const auto block_id = block_entry->block_id(); + BlockColumnEntry *block_column_entry = block_entry->GetColumnBlockEntry(column_id); + memory_secondary_index_->Insert(block_id, block_column_entry, buffer_mgr, 0, block_entry->row_count()); + } + MemIndexDump(); + break; + } + case IndexType::kIVFFlat: { // TODO UniquePtr err_msg = MakeUnique(fmt::format("{} PopulateEntirely is not supported yet", IndexInfo::IndexTypeToString(index_base->index_type_))); LOG_WARN(*err_msg); @@ -549,32 +587,7 @@ Status SegmentIndexEntry::CreateIndexPrepare(const SegmentEntry *segment_entry, break; } case IndexType::kSecondary: { - auto &data_type = column_def->type(); - if (!(data_type->CanBuildSecondaryIndex())) { - UnrecoverableError(fmt::format("Cannot build secondary index on data type: {}", data_type->ToString())); - } - // 1. build secondary index by merge sort - u32 part_capacity = DEFAULT_BLOCK_CAPACITY; - // fetch the row_count from segment_entry - auto secondary_index_builder = GetSecondaryIndexDataBuilder(data_type, segment_entry->row_count(), part_capacity); - secondary_index_builder->LoadSegmentData(segment_entry, buffer_mgr, column_def->id(), begin_ts, check_ts); - secondary_index_builder->StartOutput(); - // 2. output into SecondaryIndexDataPart - { - u32 part_num = GetIndexPartNum(); - for (u32 part_id = 0; part_id < part_num; ++part_id) { - BufferHandle buffer_handle_part = GetIndexPartAt(part_id); - auto secondary_index_part = static_cast(buffer_handle_part.GetDataMut()); - secondary_index_builder->OutputToPart(secondary_index_part); - } - } - // 3. output into SecondaryIndexDataHead - { - BufferHandle buffer_handle_head = GetIndex(); - auto secondary_index_head = static_cast(buffer_handle_head.GetDataMut()); - secondary_index_builder->OutputToHeader(secondary_index_head); - } - secondary_index_builder->EndOutput(); + PopulateEntirely(segment_entry, txn, populate_entire_config); break; } default: { @@ -730,8 +743,7 @@ SegmentIndexEntry::GetCreateIndexParam(SharedPtr index_base, SizeT se return MakeUnique(index_base, column_def); } case IndexType::kSecondary: { - u32 part_capacity = DEFAULT_BLOCK_CAPACITY; - return MakeUnique(index_base, column_def, seg_row_count, part_capacity); + return MakeUnique(index_base, column_def, seg_row_count); } default: { UniquePtr err_msg = @@ -833,6 +845,11 @@ SharedPtr SegmentIndexEntry::CreateChunkIndexEntry(SharedPtr SegmentIndexEntry::CreateSecondaryIndexChunkIndexEntry(RowID base_rowid, u32 row_count, BufferManager *buffer_mgr) { + ChunkID chunk_id = this->GetNextChunkID(); + return ChunkIndexEntry::NewSecondaryIndexChunkIndexEntry(chunk_id, this, "", base_rowid, row_count, buffer_mgr); +} + void SegmentIndexEntry::AddChunkIndexEntry(SharedPtr chunk_index_entry) { std::shared_lock lock(rw_locker_); chunk_index_entries_.push_back(chunk_index_entry); diff --git a/src/storage/meta/entry/segment_index_entry.cppm b/src/storage/meta/entry/segment_index_entry.cppm index 27dea41b45..ead32661ef 100644 --- a/src/storage/meta/entry/segment_index_entry.cppm +++ b/src/storage/meta/entry/segment_index_entry.cppm @@ -44,6 +44,7 @@ struct TableEntry; class BufferManager; struct SegmentEntry; struct TableEntry; +class SecondaryIndexInMem; export struct PopulateEntireConfig { bool prepare_; @@ -174,6 +175,11 @@ public: return {chunk_index_entries_, memory_hnsw_indexer_}; } + Tuple>, SharedPtr> GetSecondaryIndexSnapshot() { + std::shared_lock lock(rw_locker_); + return {chunk_index_entries_, memory_secondary_index_}; + } + Pair GetFulltextColumnLenInfo() { std::shared_lock lock(rw_locker_); if (ft_column_len_sum_ == 0 && memory_indexer_.get() != nullptr) { @@ -190,6 +196,9 @@ public: public: SharedPtr CreateChunkIndexEntry(SharedPtr column_def, RowID base_rowid, BufferManager *buffer_mgr); + SharedPtr + CreateSecondaryIndexChunkIndexEntry(RowID base_rowid, u32 row_count, BufferManager *buffer_mgr); + void AddChunkIndexEntry(SharedPtr chunk_index_entry); SharedPtr AddFtChunkIndexEntry(const String &base_name, RowID base_rowid, u32 row_count); @@ -234,6 +243,7 @@ private: Vector> chunk_index_entries_{}; SharedPtr memory_hnsw_indexer_{}; SharedPtr memory_indexer_{}; + SharedPtr memory_secondary_index_{}; u64 ft_column_len_sum_{}; // increase only u32 ft_column_len_cnt_{}; // increase only diff --git a/src/storage/meta/entry/table_entry.cpp b/src/storage/meta/entry/table_entry.cpp index 83bc75b7da..a42a3099a3 100644 --- a/src/storage/meta/entry/table_entry.cpp +++ b/src/storage/meta/entry/table_entry.cpp @@ -327,11 +327,19 @@ void TableEntry::Import(SharedPtr segment_entry, Txn *txn) { if (!status.ok()) continue; const IndexBase *index_base = table_index_entry->index_base(); - if (index_base->index_type_ != IndexType::kFullText && index_base->index_type_ != IndexType::kHnsw) { - UniquePtr err_msg = - MakeUnique(fmt::format("{} realtime index is not supported yet", IndexInfo::IndexTypeToString(index_base->index_type_))); - LOG_WARN(*err_msg); - continue; + switch (index_base->index_type_) { + case IndexType::kFullText: + case IndexType::kSecondary: + case IndexType::kHnsw: { + // support realtime index + break; + } + default: { + UniquePtr err_msg = + MakeUnique(fmt::format("{} realtime index is not supported yet", IndexInfo::IndexTypeToString(index_base->index_type_))); + LOG_WARN(*err_msg); + continue; + } } PopulateEntireConfig populate_entire_config{.prepare_ = false, .check_ts_ = false}; SharedPtr segment_index_entry = table_index_entry->PopulateEntirely(segment_entry.get(), txn, populate_entire_config); @@ -618,7 +626,8 @@ void TableEntry::MemIndexInsert(Txn *txn, Vector &append_ranges) { const IndexBase *index_base = table_index_entry->index_base(); switch (index_base->index_type_) { case IndexType::kHnsw: - case IndexType::kFullText: { + case IndexType::kFullText: + case IndexType::kSecondary: { for (auto &[seg_id, ranges] : seg_append_ranges) { MemIndexInsertInner(table_index_entry, txn, seg_id, ranges); } diff --git a/src/storage/meta/entry/table_index_entry.cpp b/src/storage/meta/entry/table_index_entry.cpp index 720362020d..0399d24a14 100644 --- a/src/storage/meta/entry/table_index_entry.cpp +++ b/src/storage/meta/entry/table_index_entry.cpp @@ -267,8 +267,15 @@ SharedPtr TableIndexEntry::MemIndexDump(TxnIndexStore *txn_inde } SharedPtr TableIndexEntry::PopulateEntirely(SegmentEntry *segment_entry, Txn *txn, const PopulateEntireConfig &config) { - if (index_base_->index_type_ != IndexType::kFullText && index_base_->index_type_ != IndexType::kHnsw) { - return nullptr; + switch (index_base_->index_type_) { + case IndexType::kHnsw: + case IndexType::kFullText: + case IndexType::kSecondary: { + break; + } + default: { + return nullptr; + } } auto create_index_param = SegmentIndexEntry::GetCreateIndexParam(index_base_, segment_entry->row_capacity(), column_def_); u32 segment_id = segment_entry->segment_id(); diff --git a/src/storage/secondary_index/common_query_filter.cpp b/src/storage/secondary_index/common_query_filter.cpp index 5fe272fd70..7be0893135 100644 --- a/src/storage/secondary_index/common_query_filter.cpp +++ b/src/storage/secondary_index/common_query_filter.cpp @@ -144,7 +144,8 @@ void CommonQueryFilter::BuildFilter(u32 task_id, TxnTimeStamp begin_ts, BufferMa secondary_index_column_index_map_, segment_id, segment_row_count, - segment_actual_row_count); + segment_actual_row_count, + begin_ts); if (std::visit(Overload{[](const Vector &v) -> bool { return v.empty(); }, [](const Bitmask &) -> bool { return false; }}, result_elem)) { // empty result return; diff --git a/src/storage/secondary_index/secondary_index_data.cpp b/src/storage/secondary_index/secondary_index_data.cpp index 161ac21ef6..dfa0d3603e 100644 --- a/src/storage/secondary_index/secondary_index_data.cpp +++ b/src/storage/secondary_index/secondary_index_data.cpp @@ -26,365 +26,168 @@ import index_base; import file_system; import file_system_type; import infinity_exception; -import column_vector; import third_party; -import segment_iter; -import buffer_manager; import secondary_index_pgm; import logger; +import chunk_index_entry; +import buffer_handle; namespace infinity { -template -inline void LoadFromSegmentColumnIterator(OneColumnIterator &iter, - UniquePtr[]> &sorted_key_offset_pair, - const u32 full_data_num, - u32 &data_num) { - if (data_num != 0) { - UnrecoverableError("LoadFromSegmentColumnIterator(): data_num is not initially 0"); - } - while (true) { - auto pair_opt = iter.Next(); - if (!pair_opt) { - break; - } - if (data_num >= full_data_num) { - UnrecoverableError("LoadFromSegmentColumnIterator(): segment row count more than expected"); - } - auto &[val_ptr, offset] = pair_opt.value(); // val_ptr is const RawValueType * type, offset is SegmentOffset type - sorted_key_offset_pair[data_num++] = {ConvertToOrderedKeyValue(*val_ptr), offset}; - } - // finally, sort - std::sort(sorted_key_offset_pair.get(), sorted_key_offset_pair.get() + data_num); -} - -// usage: -// 1. AppendColumnVector(): merge sort, collect all values of the column in the segment -// 2.1. OutputToPart(): copy sorted (key, offset) pairs into several SecondaryIndexDataPart structures. -// 2.2. OutputToHeader(): create PGM index in SecondaryIndexDataHead. template -class SecondaryIndexDataBuilder final : public SecondaryIndexDataBuilderBase { -public: - using KeyType = ConvertToOrderedType; - using OffsetType = SegmentOffset; - using KeyOffsetPair = Pair; +class SecondaryIndexDataT final : public SecondaryIndexData { + using OrderedKeyType = ConvertToOrderedType; + // sorted values in chunk + // only for build and save + // should not be loaded from file + bool need_save_ = false; + UniquePtr key_; + UniquePtr offset_; - explicit SecondaryIndexDataBuilder(u32 full_data_num, u32 part_capacity) : full_data_num_(full_data_num), output_part_capacity_(part_capacity) { - output_part_num_ = (full_data_num + output_part_capacity_ - 1) / output_part_capacity_; - index_key_type_ = GetLogicalType; - output_offset_type_ = LogicalType::kInteger; // used to store u32 offsets - sorted_key_offset_pair_ = MakeUniqueForOverwrite(full_data_num_); - } - - ~SecondaryIndexDataBuilder() final = default; +public: + static constexpr u32 PairSize = sizeof(OrderedKeyType) + sizeof(SegmentOffset); - void - LoadSegmentData(const SegmentEntry *segment_entry, BufferManager *buffer_mgr, ColumnID column_id, TxnTimeStamp begin_ts, bool check_ts) final { - static_assert(std::is_same_v, "OffsetType != SegmentOffset, need to fix"); - if (check_ts) { - OneColumnIterator iter(segment_entry, buffer_mgr, column_id, begin_ts); - return LoadFromSegmentColumnIterator(iter, sorted_key_offset_pair_, full_data_num_, data_num_); - } else { - OneColumnIterator iter(segment_entry, buffer_mgr, column_id, begin_ts); - return LoadFromSegmentColumnIterator(iter, sorted_key_offset_pair_, full_data_num_, data_num_); + SecondaryIndexDataT(const u32 chunk_row_count, const bool allocate) : SecondaryIndexData(chunk_row_count) { + pgm_index_ = GenerateSecondaryPGMIndex(); + if (allocate) { + need_save_ = true; + LOG_TRACE(fmt::format("SecondaryIndexDataT(): Allocate space for chunk_row_count_: {}", chunk_row_count_)); + key_ = MakeUnique(chunk_row_count_); + offset_ = MakeUnique(chunk_row_count_); } } - void StartOutput() final { - output_row_progress_ = 0; - output_part_progress_ = 0; - // initialize sorted_keys_ - sorted_keys_ = MakeUniqueForOverwrite(data_num_); - LOG_TRACE(fmt::format("StartOutput(), output_row_progress_: {}, data_num_: {}.", output_row_progress_, data_num_)); - } - - void EndOutput() final { - if (output_row_progress_ != data_num_) { - UnrecoverableError("EndOutput(): output is not complete: output_row_progress_ != data_num_."); - } - if (output_part_progress_ != output_part_num_ + 1) { - UnrecoverableError("EndOutput(): output is not complete: output_part_progress_ != output_part_num_ + 1."); + void SaveIndexInner(FileHandler &file_handler) const override { + if (!need_save_) { + UnrecoverableError("SaveIndexInner(): error: SecondaryIndexDataT is not allocated."); } - // destroy sorted_keys_ - sorted_keys_.reset(); - LOG_TRACE(fmt::format("EndOutput(), output_row_progress_: {}, data_num_: {}.", output_row_progress_, data_num_)); + pgm_index_->SaveIndex(file_handler); } - void OutputToHeader(SecondaryIndexDataHead *index_head) final { - if (output_part_progress_ != output_part_num_) { - UnrecoverableError( - "OutputToHeader(): error: output_part_progress_ != output_part_num_, need to call OutputToHeader() after OutputToPart()."); - } - if (output_row_progress_ != data_num_) { - UnrecoverableError("OutputToHeader(): error: output_row_progress_ != data_num_, need to call OutputToHeader() after OutputToPart()."); - } - // 1. metadata - { - if (index_head->full_data_num_ != full_data_num_) { - UnrecoverableError("OutputToHeader(): error: index_head->full_data_num_ != full_data_num_"); - } - if (index_head->data_num_ != 0) { - UnrecoverableError("OutputToHeader(): index_head->data_num_ already exist"); - } - index_head->data_num_ = data_num_; - if (index_head->part_capacity_ != output_part_capacity_) { - UnrecoverableError("OutputToHeader(): error: index_head->part_capacity_ != output_part_capacity_"); - } - if (index_head->part_num_ != output_part_num_) { - UnrecoverableError("OutputToHeader(): error: index_head->part_num_ != output_part_num_"); - } - // type of key and offset - index_head->data_type_key_ = index_key_type_; - index_head->data_type_offset_ = output_offset_type_; - } - // 2. pgm - sorted_key_offset_pair_.reset(); // release some memory - { - index_head->pgm_index_ = GenerateSecondaryPGMIndex(); - index_head->pgm_index_->BuildIndex(data_num_, sorted_keys_.get()); - LOG_TRACE("OutputToHeader(): Successfully built pgm index."); - } - // 3. finish - ++output_part_progress_; - // now output_part_progress_ = output_part_num_ + 1 - index_head->loaded_ = true; - LOG_TRACE(fmt::format("OutputToHeader(), output_row_progress_: {}, data_num_: {}.", output_row_progress_, data_num_)); - } + void ReadIndexInner(FileHandler &file_handler) override { pgm_index_->LoadIndex(file_handler); } - void OutputToPart(SecondaryIndexDataPart *index_part) final { - if (output_part_progress_ != index_part->part_id_) { - UnrecoverableError("OutputToPart(): error: unexpected index_part->part_id_ value"); + void InsertData(void *ptr, SharedPtr &chunk_index) override { + if (!need_save_) { + UnrecoverableError("InsertData(): error: SecondaryIndexDataT is not allocated."); } - if (auto expect_size = std::min(output_part_capacity_, data_num_ - output_row_progress_); expect_size != index_part->part_size_) { - if (index_part->part_size_ < expect_size) { - UnrecoverableError("OutputToPart(): error: index_part->part_size_"); - } else { - LOG_INFO(fmt::format("OutputToPart(): index_part->part_size_: {}, expect_size: {}. Maybe some rows are deleted.", - index_part->part_size_, - expect_size)); - index_part->part_size_ = expect_size; - } + auto map_ptr = static_cast *>(ptr); + if (!map_ptr) { + UnrecoverableError("InsertData(): error: map_ptr type error."); } - // check empty part - if (index_part->part_size_ == 0) { - index_part->loaded_ = true; - ++output_part_progress_; - LOG_TRACE(fmt::format("OutputToPart(), output_row_progress_: {}, data_num_: {}.", output_row_progress_, data_num_)); - return; + if (map_ptr->size() != chunk_row_count_) { + UnrecoverableError(fmt::format("InsertData(): error: map size: {} != chunk_row_count_: {}", map_ptr->size(), chunk_row_count_)); } - // 1. metadata - { - index_part->data_type_key_ = index_key_type_; - index_part->data_type_offset_ = output_offset_type_; + u32 i = 0; + for (const auto &[key, offset] : *map_ptr) { + key_[i] = key; + offset_[i] = offset; + ++i; } - // 2. raw data - { - auto data_type_key = MakeShared(index_part->data_type_key_); - index_part->column_key_ = MakeUnique(std::move(data_type_key)); - index_part->column_key_->Initialize(); - auto data_type_offset = MakeShared(index_part->data_type_offset_); - index_part->column_offset_ = MakeUnique(data_type_offset); - index_part->column_offset_->Initialize(); - auto key_ptr = reinterpret_cast(index_part->column_key_->data()); - auto offset_ptr = reinterpret_cast(index_part->column_offset_->data()); - static_assert(sizeof(IntegerT) == sizeof(OffsetType), "sizeof(IntegerT) != sizeof(u32), cannot use IntegerT as offset type"); - for (u32 i = 0; i < index_part->part_size_; ++i) { - auto [key, offset] = sorted_key_offset_pair_[output_row_progress_ + i]; - key_ptr[i] = key; - offset_ptr[i] = offset; // store u32 into IntegerT ColumnVector + if (i != chunk_row_count_) { + UnrecoverableError(fmt::format("InsertData(): error: i: {} != chunk_row_count_: {}", i, chunk_row_count_)); + } + const u32 part_num = chunk_index->GetPartNum(); + for (u32 part_id = 0; part_id < part_num; ++part_id) { + const u32 part_row_count = chunk_index->GetPartRowCount(part_id); + const u32 part_offset = part_id * 8192; + BufferHandle handle = chunk_index->GetIndexPartAt(part_id); + auto data_ptr = static_cast(handle.GetDataMut()); + for (u32 j = 0; j < part_row_count; ++j) { + const u32 index = part_offset + j; + const OrderedKeyType key = key_[index]; + const u32 offset = offset_[index]; + std::memcpy(data_ptr + j * PairSize, &key, sizeof(OrderedKeyType)); + std::memcpy(data_ptr + j * PairSize + sizeof(OrderedKeyType), &offset, sizeof(SegmentOffset)); } - // copy to sorted_keys_ - // used in OutputToHeader() - std::copy(key_ptr, key_ptr + index_part->part_size_, sorted_keys_.get() + output_row_progress_); - // finalize - index_part->column_key_->Finalize(index_part->part_size_); - index_part->column_offset_->Finalize(index_part->part_size_); } - // 3. finish - index_part->loaded_ = true; - output_row_progress_ += index_part->part_size_; - ++output_part_progress_; - LOG_TRACE(fmt::format("OutputToPart(), output_row_progress_: {}, data_num_: {}.", output_row_progress_, data_num_)); + pgm_index_->BuildIndex(chunk_row_count_, key_.get()); } - -private: - const u32 full_data_num_{}; // number of rows in the segment, include those deleted - u32 data_num_{}; // number of rows in the segment, except those deleted, start from 0, grow during input - LogicalType index_key_type_ = LogicalType::kInvalid; // type of ordered keys stored in the raw index - UniquePtr sorted_key_offset_pair_; // size: full_data_num_. Will be destroyed in OutputToHeader(). - -private: - // record output progress - u32 output_part_capacity_{}; // number of rows in each full output part - u32 output_part_num_{}; // number of output parts - u32 output_row_progress_{}; // record output progress - u32 output_part_progress_{}; // record output progress - LogicalType output_offset_type_ = LogicalType::kInvalid; // type of offset stored in the output part - UniquePtr sorted_keys_; // for pgm. Will be created in StartOutput(). }; -UniquePtr GetSecondaryIndexDataBuilder(const SharedPtr &data_type, u32 full_data_num, u32 part_capacity) { +SecondaryIndexData *GetSecondaryIndexData(const SharedPtr &data_type, const u32 chunk_row_count, const bool allocate) { if (!(data_type->CanBuildSecondaryIndex())) { UnrecoverableError(fmt::format("Cannot build secondary index on data type: {}", data_type->ToString())); - return {}; + return nullptr; } switch (data_type->type()) { case LogicalType::kTinyInt: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kSmallInt: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kInteger: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kBigInt: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kFloat: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kDouble: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kDate: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kTime: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kDateTime: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } case LogicalType::kTimestamp: { - return MakeUnique>(full_data_num, part_capacity); + return new SecondaryIndexDataT(chunk_row_count, allocate); } default: { UnrecoverableError(fmt::format("Need to add secondary index support for data type: {}", data_type->ToString())); - return {}; + return nullptr; } } } -void SecondaryIndexDataHead::SaveIndexInner(FileHandler &file_handler) const { - if (!loaded_) { - UnrecoverableError("SaveIndexInner(): error: SecondaryIndexDataHead is not loaded"); - } - file_handler.Write(&part_capacity_, sizeof(part_capacity_)); - file_handler.Write(&part_num_, sizeof(part_num_)); - file_handler.Write(&full_data_num_, sizeof(full_data_num_)); - file_handler.Write(&data_num_, sizeof(data_num_)); - file_handler.Write(&data_type_raw_, sizeof(data_type_raw_)); - file_handler.Write(&data_type_key_, sizeof(data_type_key_)); - file_handler.Write(&data_type_offset_, sizeof(data_type_offset_)); - // pgm - pgm_index_->SaveIndex(file_handler); - LOG_TRACE("SaveIndexInner() done."); -} - -void SecondaryIndexDataHead::ReadIndexInner(FileHandler &file_handler) { - if (loaded_) { - UnrecoverableError("SecondaryIndexDataHead is already loaded"); +u32 GetSecondaryIndexDataPairSize(const SharedPtr &data_type) { + if (!(data_type->CanBuildSecondaryIndex())) { + UnrecoverableError(fmt::format("Cannot build secondary index on data type: {}", data_type->ToString())); + return 0; } - file_handler.Read(&part_capacity_, sizeof(part_capacity_)); - file_handler.Read(&part_num_, sizeof(part_num_)); - file_handler.Read(&full_data_num_, sizeof(full_data_num_)); - file_handler.Read(&data_num_, sizeof(data_num_)); - file_handler.Read(&data_type_raw_, sizeof(data_type_raw_)); - file_handler.Read(&data_type_key_, sizeof(data_type_key_)); - file_handler.Read(&data_type_offset_, sizeof(data_type_offset_)); - // initialize pgm - switch (data_type_key_) { + switch (data_type->type()) { case LogicalType::kTinyInt: { - pgm_index_ = GenerateSecondaryPGMIndex(); - break; + return SecondaryIndexDataT::PairSize; } case LogicalType::kSmallInt: { - pgm_index_ = GenerateSecondaryPGMIndex(); - break; + return SecondaryIndexDataT::PairSize; } case LogicalType::kInteger: { - pgm_index_ = GenerateSecondaryPGMIndex(); - break; + return SecondaryIndexDataT::PairSize; } case LogicalType::kBigInt: { - pgm_index_ = GenerateSecondaryPGMIndex(); - break; + return SecondaryIndexDataT::PairSize; } case LogicalType::kFloat: { - pgm_index_ = GenerateSecondaryPGMIndex(); - break; + return SecondaryIndexDataT::PairSize; } case LogicalType::kDouble: { - pgm_index_ = GenerateSecondaryPGMIndex(); - break; + return SecondaryIndexDataT::PairSize; + } + case LogicalType::kDate: { + return SecondaryIndexDataT::PairSize; + } + case LogicalType::kTime: { + return SecondaryIndexDataT::PairSize; + } + case LogicalType::kDateTime: { + return SecondaryIndexDataT::PairSize; + } + case LogicalType::kTimestamp: { + return SecondaryIndexDataT::PairSize; } default: { - UnrecoverableError(fmt::format("Need to add support for data type: {}", DataType(data_type_key_).ToString())); + UnrecoverableError(fmt::format("Need to add secondary index support for data type: {}", data_type->ToString())); + return 0; } } - // load pgm - pgm_index_->LoadIndex(file_handler); - // update loaded_ - loaded_ = true; - LOG_TRACE("ReadIndexInner() done."); -} - -void SecondaryIndexDataPart::SaveIndexInner(FileHandler &file_handler) const { - if (!loaded_) { - UnrecoverableError("SaveIndexInner(): error: SecondaryIndexDataPart is not loaded"); - } - file_handler.Write(&part_id_, sizeof(part_id_)); - file_handler.Write(&part_size_, sizeof(part_size_)); - if (part_size_ == 0) { - return; - } - file_handler.Write(&data_type_key_, sizeof(data_type_key_)); - file_handler.Write(&data_type_offset_, sizeof(data_type_offset_)); - // key - if (u32 key_cnt = column_key_->Size(); key_cnt != part_size_) { - UnrecoverableError("SaveIndexInner(): error: column_key_ size != part_size_."); - } - file_handler.Write(column_key_->data(), part_size_ * (column_key_->data_type_size_)); - // offset - if (u32 offset_cnt = column_offset_->Size(); offset_cnt != part_size_) { - UnrecoverableError("SaveIndexInner(): error: column_offset_ size != part_size_."); - } - file_handler.Write(column_offset_->data(), part_size_ * (column_offset_->data_type_size_)); - LOG_TRACE(fmt::format("SaveIndexInner() done. part_id_: {}.", part_id_)); -} - -void SecondaryIndexDataPart::ReadIndexInner(FileHandler &file_handler) { - if (loaded_) { - UnrecoverableError("SecondaryIndexDataPart is already loaded"); - } - file_handler.Read(&part_id_, sizeof(part_id_)); - file_handler.Read(&part_size_, sizeof(part_size_)); - if (part_size_ == 0) { - return; - } - // key type - file_handler.Read(&data_type_key_, sizeof(data_type_key_)); - auto data_type_key = MakeShared(data_type_key_); - // offset type - file_handler.Read(&data_type_offset_, sizeof(data_type_offset_)); - if (data_type_offset_ != LogicalType::kInteger) { - UnrecoverableError("ReadIndexInner(): data_type_offset_ != LogicalType::kInteger"); - } - auto data_type_offset = MakeShared(data_type_offset_); - // key - column_key_ = MakeUnique(std::move(data_type_key)); - column_key_->Initialize(); - file_handler.Read(column_key_->data(), part_size_ * (column_key_->data_type_size_)); - column_key_->Finalize(part_size_); - // offset - column_offset_ = MakeUnique(std::move(data_type_offset)); - column_offset_->Initialize(); - file_handler.Read(column_offset_->data(), part_size_ * (column_offset_->data_type_size_)); - column_offset_->Finalize(part_size_); - // update loaded_ - loaded_ = true; - LOG_TRACE(fmt::format("ReadIndexInner() done. part_id_: {}.", part_id_)); } } // namespace infinity \ No newline at end of file diff --git a/src/storage/secondary_index/secondary_index_data.cppm b/src/storage/secondary_index/secondary_index_data.cppm index 58dc140cce..2580f2f715 100644 --- a/src/storage/secondary_index/secondary_index_data.cppm +++ b/src/storage/secondary_index/secondary_index_data.cppm @@ -14,19 +14,15 @@ module; -#include - export module secondary_index_data; import stl; - import default_values; import file_system; import file_system_type; import infinity_exception; import column_vector; import third_party; -import buffer_manager; import secondary_index_pgm; import logical_type; import internal_types; @@ -34,6 +30,7 @@ import data_type; import segment_entry; namespace infinity { +class ChunkIndexEntry; template concept KeepOrderedSelf = IsAnyOf; @@ -91,7 +88,7 @@ ConvertToOrderedType ConvertToOrderedKeyValue(RawValueType value) } template -LogicalType GetLogicalType = kInvalid; +LogicalType GetLogicalType = LogicalType::kInvalid; template <> LogicalType GetLogicalType = LogicalType::kFloat; @@ -111,122 +108,36 @@ LogicalType GetLogicalType = LogicalType::kInteger; template <> LogicalType GetLogicalType = LogicalType::kBigInt; -export class SecondaryIndexDataHead; - -export class SecondaryIndexDataPart; - -class SecondaryIndexDataBuilderBase { -public: - SecondaryIndexDataBuilderBase() = default; - virtual ~SecondaryIndexDataBuilderBase() = default; - virtual void - LoadSegmentData(const SegmentEntry *segment_entry, BufferManager *buffer_mgr, ColumnID column_id, TxnTimeStamp begin_ts, bool check_ts) = 0; - virtual void StartOutput() = 0; - virtual void EndOutput() = 0; - virtual void OutputToHeader(SecondaryIndexDataHead *index_head) = 0; - virtual void OutputToPart(SecondaryIndexDataPart *index_part) = 0; -}; - -// create a secondary index on each segment -// now only support index for single column -// now only support create index for POD type with size <= sizeof(i64) -// need to convert values in column into ordered number type -// data_num : number of rows in the segment, except those deleted -export UniquePtr -GetSecondaryIndexDataBuilder(const SharedPtr &data_type, u32 full_data_num, u32 part_capacity); - -// includes: metadata and PGM index -class SecondaryIndexDataHead { - friend class SecondaryIndexDataBuilderBase; - template - friend class SecondaryIndexDataBuilder; - -private: - bool loaded_{false}; // whether data of this part is in memory - u32 part_capacity_{}; // number of rows in each full part - u32 part_num_{}; // number of parts - u32 full_data_num_{}; // number of rows in the segment (include those deleted) - u32 data_num_{}; // number of rows in the segment (except those deleted), init as 0 - // sorted values in segment - LogicalType data_type_raw_ = LogicalType::kInvalid; // type of original data - LogicalType data_type_key_ = LogicalType::kInvalid; // type of data stored in the raw index - // offset of each value in segment - // type: IntegerT (its size matches with the type u32 of the segment_offset_) - LogicalType data_type_offset_ = LogicalType::kInvalid; +export class SecondaryIndexData { +protected: + u32 chunk_row_count_ = 0; // pgm index + // will always be loaded UniquePtr pgm_index_; public: - // will be called when an old index is loaded - // used in SecondaryIndexFileWorker::ReadFromFileImpl() - SecondaryIndexDataHead() = default; - - // will be called when a new index is created - // used in SecondaryIndexFileWorker::AllocateInMemory() - explicit SecondaryIndexDataHead(u32 part_capacity, u32 full_data_num, const SharedPtr &data_type) - : part_capacity_(part_capacity), full_data_num_(full_data_num) { - part_num_ = (full_data_num + part_capacity_ - 1) / part_capacity_; - data_type_raw_ = data_type->type(); - } + explicit SecondaryIndexData(u32 chunk_row_count) : chunk_row_count_(chunk_row_count) {} - ~SecondaryIndexDataHead() = default; + virtual ~SecondaryIndexData() = default; - [[nodiscard]] u32 GetPartCapacity() const { return part_capacity_; } - [[nodiscard]] u32 GetPartNum() const { return part_num_; } - [[nodiscard]] u32 GetDataNum() const { return data_num_; } - - [[nodiscard]] auto SearchPGM(const void *val_ptr) const { + [[nodiscard]] inline auto SearchPGM(const void *val_ptr) const { if (!pgm_index_) { UnrecoverableError("Not initialized yet."); } return pgm_index_->SearchIndex(val_ptr); } - void SaveIndexInner(FileHandler &file_handler) const; + [[nodiscard]] inline u32 GetChunkRowCount() const { return chunk_row_count_; } - void ReadIndexInner(FileHandler &file_handler); -}; + virtual void SaveIndexInner(FileHandler &file_handler) const = 0; -// an index may include several parts -// includes: a part of keys and corresponding offsets in the segment -class SecondaryIndexDataPart { - friend class SecondaryIndexDataBuilderBase; - template - friend class SecondaryIndexDataBuilder; - -private: - bool loaded_{false}; // whether data of this part is in memory - u32 part_id_{}; // id of this part - u32 part_size_{}; // number of rows in this part - // data type - LogicalType data_type_key_ = LogicalType::kInvalid; - LogicalType data_type_offset_ = LogicalType::kInvalid; - // key-offset pairs - UniquePtr column_key_; - UniquePtr column_offset_; + virtual void ReadIndexInner(FileHandler &file_handler) = 0; -public: - // will be called when an old index is loaded - // used in SecondaryIndexFileWorker::ReadFromFileImpl() - SecondaryIndexDataPart() = default; - - // will be called when a new index is created - // used in SecondaryIndexFileWorker::AllocateInMemory() - explicit SecondaryIndexDataPart(u32 part_id, u32 part_size) : part_id_(part_id), part_size_(part_size) {} - - ~SecondaryIndexDataPart() = default; - - [[nodiscard]] u32 GetPartId() const { return part_id_; } - - [[nodiscard]] u32 GetPartSize() const { return part_size_; } - - [[nodiscard]] const void *GetColumnKeyData() const { return column_key_->data(); } - - [[nodiscard]] const void *GetColumnOffsetData() const { return column_offset_->data(); } + virtual void InsertData(void *ptr, SharedPtr &chunk_index) = 0; +}; - void SaveIndexInner(FileHandler &file_handler) const; +export SecondaryIndexData *GetSecondaryIndexData(const SharedPtr &data_type, u32 chunk_row_count, bool allocate); - void ReadIndexInner(FileHandler &file_handler); -}; +export u32 GetSecondaryIndexDataPairSize(const SharedPtr &data_type); } // namespace infinity \ No newline at end of file diff --git a/src/storage/secondary_index/secondary_index_in_mem.cpp b/src/storage/secondary_index/secondary_index_in_mem.cpp new file mode 100644 index 0000000000..d7245099c2 --- /dev/null +++ b/src/storage/secondary_index/secondary_index_in_mem.cpp @@ -0,0 +1,149 @@ +// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module; + +#include +#include +module secondary_index_in_mem; + +import stl; +import logical_type; +import internal_types; +import column_def; +import bitmask; +import default_values; +import buffer_manager; +import block_column_entry; +import block_column_iter; +import infinity_exception; +import secondary_index_data; +import chunk_index_entry; +import segment_index_entry; +import buffer_handle; + +namespace infinity { + +template +class SecondaryIndexInMemT final : public SecondaryIndexInMem { + using KeyType = ConvertToOrderedType; + const RowID begin_row_id_; + const u32 max_size_; + std::shared_mutex map_mutex_; + MultiMap in_mem_secondary_index_; + +public: + explicit SecondaryIndexInMemT(const RowID begin_row_id, const u32 max_size) : begin_row_id_(begin_row_id), max_size_(max_size) {} + u32 GetRowCount() const override { return in_mem_secondary_index_.size(); } + void Insert(const u16 block_id, BlockColumnEntry *block_column_entry, BufferManager *buffer_manager, u32 row_offset, u32 row_count) override { + MemIndexInserterIter iter(block_id * DEFAULT_BLOCK_CAPACITY, block_column_entry, buffer_manager, row_offset, row_count); + InsertInner(iter); + } + SharedPtr Dump(SegmentIndexEntry *segment_index_entry, BufferManager *buffer_mgr) override { + std::shared_lock lock(map_mutex_); + u32 row_count = GetRowCount(); + auto new_chunk_index_entry = segment_index_entry->CreateSecondaryIndexChunkIndexEntry(begin_row_id_, row_count, buffer_mgr); + BufferHandle handle = new_chunk_index_entry->GetIndex(); + auto data_ptr = static_cast(handle.GetDataMut()); + data_ptr->InsertData(&in_mem_secondary_index_, new_chunk_index_entry); + return new_chunk_index_entry; + } + Pair, Bitmask>> RangeQuery(const void *input) override { + const auto &[segment_row_count, b, e] = *static_cast *>(input); + return RangeQueryInner(segment_row_count, b, e); + } + +private: + void InsertInner(auto &iter) { + std::unique_lock lock(map_mutex_); + while (true) { + auto opt = iter.Next(); + if (!opt.has_value()) { + break; + } + const auto &[v_ptr, offset] = opt.value(); + const KeyType key = ConvertToOrderedKeyValue(*v_ptr); + in_mem_secondary_index_.emplace(key, offset); + } + } + + Pair, Bitmask>> RangeQueryInner(const u32 segment_row_count, const KeyType b, const KeyType e) { + std::shared_lock lock(map_mutex_); + const auto begin = in_mem_secondary_index_.lower_bound(b); + const auto end = in_mem_secondary_index_.upper_bound(e); + const u32 result_size = std::distance(begin, end); + Pair, Bitmask>> result_var; + result_var.first = result_size; + // use array or bitmask for result + // use array when result_size <= 1024 or size of array (u32 type) <= size of bitmask + if (result_size <= 1024 or result_size <= std::bit_ceil(segment_row_count) / 32) { + auto &result = result_var.second.emplace>(); + result.reserve(result_size); + for (auto it = begin; it != end; ++it) { + result.push_back(it->second); + } + } else { + auto &result = result_var.second.emplace(); + result.Initialize(segment_row_count); + result.SetAllFalse(); + for (auto it = begin; it != end; ++it) { + result.SetTrue(it->second); + } + } + return result_var; + } +}; + +SharedPtr SecondaryIndexInMem::NewSecondaryIndexInMem(const SharedPtr &column_def, RowID begin_row_id, u32 max_size) { + if (!column_def->type()->CanBuildSecondaryIndex()) { + UnrecoverableError("Column type can't build secondary index"); + } + switch (column_def->type()->type()) { + case LogicalType::kTinyInt: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kSmallInt: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kInteger: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kBigInt: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kFloat: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kDouble: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kDate: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kTime: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kDateTime: { + return MakeShared>(begin_row_id, max_size); + } + case LogicalType::kTimestamp: { + return MakeShared>(begin_row_id, max_size); + } + default: { + return nullptr; + } + } +} + +} // namespace infinity \ No newline at end of file diff --git a/src/storage/secondary_index/secondary_index_in_mem.cppm b/src/storage/secondary_index/secondary_index_in_mem.cppm new file mode 100644 index 0000000000..1b11d7e6bd --- /dev/null +++ b/src/storage/secondary_index/secondary_index_in_mem.cppm @@ -0,0 +1,42 @@ +// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module; + +export module secondary_index_in_mem; + +import stl; + +namespace infinity { + +struct Bitmask; +struct RowID; +struct BlockColumnEntry; +class BufferManager; +class ColumnDef; +class ChunkIndexEntry; +class SegmentIndexEntry; + +export class SecondaryIndexInMem { +public: + virtual ~SecondaryIndexInMem() = default; + virtual u32 GetRowCount() const = 0; + virtual void Insert(u16 block_id, BlockColumnEntry *block_column_entry, BufferManager *buffer_manager, u32 row_offset, u32 row_count) = 0; + virtual SharedPtr Dump(SegmentIndexEntry *segment_index_entry, BufferManager *buffer_mgr) = 0; + virtual Pair, Bitmask>> RangeQuery(const void *input) = 0; + + static SharedPtr NewSecondaryIndexInMem(const SharedPtr &column_def, RowID begin_row_id, u32 max_size = 5 << 20); +}; + +} // namespace infinity \ No newline at end of file From 99f0ad7fcccf36cd29167d4f8030ee5fa81d8f6c Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Mon, 6 May 2024 17:18:31 +0800 Subject: [PATCH 5/9] add slt --- test/sql/dql/index_scan_insert.slt | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 test/sql/dql/index_scan_insert.slt diff --git a/test/sql/dql/index_scan_insert.slt b/test/sql/dql/index_scan_insert.slt new file mode 100644 index 0000000000..c56085cf06 --- /dev/null +++ b/test/sql/dql/index_scan_insert.slt @@ -0,0 +1,37 @@ +statement ok +DROP TABLE IF EXISTS date_index_scan_insert; + +statement ok +CREATE TABLE date_index_scan_insert (i INTEGER, d1 DATE, d2 DATE); + +statement ok +CREATE INDEX date_index_scan_insert_d1 ON date_index_scan_insert(d1); + +statement ok +INSERT INTO date_index_scan_insert VALUES + (2222, DATE '2022-1-31', DATE '2023-1-31'), + (1, DATE '1970-1-1', DATE '2970-1-1'), + (11, DATE '1870-11-1', DATE '2570-1-1'), + (111, DATE '6570-11-1', DATE '5570-6-21'); + +query I +EXPLAIN SELECT * FROM date_index_scan_insert WHERE d1 >= DATE '1970-1-1'; +---- + PROJECT (4) + - table index: #4 + - expressions: [i (#0), d1 (#1), d2 (#2)] + -> INDEX SCAN (6) + - table name: date_index_scan_insert(default_db.date_index_scan_insert) + - table index: #1 + - filter: d1 (#1.1) >= 1970-01-01 + - output_columns: [__rowid] + +query II +SELECT * FROM date_index_scan_insert WHERE d1 >= DATE '1970-1-1'; +---- +2222 2022-01-31 2023-01-31 +1 1970-01-01 2970-01-01 +111 6570-11-01 5570-06-21 + +statement ok +DROP TABLE date_index_scan_insert; From 50801d399a5955ffe69707b407359e224e2a0f0e Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Mon, 6 May 2024 18:48:17 +0800 Subject: [PATCH 6/9] merge secondary index --- .../meta/entry/segment_index_entry.cpp | 25 +++++ src/storage/meta/entry/table_entry.cpp | 3 +- .../secondary_index/secondary_index_data.cpp | 97 ++++++++++++++++++- .../secondary_index/secondary_index_data.cppm | 3 + 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/src/storage/meta/entry/segment_index_entry.cpp b/src/storage/meta/entry/segment_index_entry.cpp index 58c42c7241..b08461c59c 100644 --- a/src/storage/meta/entry/segment_index_entry.cpp +++ b/src/storage/meta/entry/segment_index_entry.cpp @@ -819,6 +819,31 @@ ChunkIndexEntry *SegmentIndexEntry::RebuildChunkIndexEntries(TxnTableStore *txn_ txn_table_store->AddChunkIndexStore(table_index_entry_, merged_chunk_index_entry.get()); return merged_chunk_index_entry.get(); } + case IndexType::kSecondary: { + BufferManager *buffer_mgr = txn->buffer_mgr(); + Vector old_chunks; + u32 row_count = 0; + { + std::shared_lock lock(rw_locker_); + if (chunk_index_entries_.size() <= 1) { + return nullptr; + } + for (const auto &chunk_index_entry : chunk_index_entries_) { + if (chunk_index_entry->CheckVisible(begin_ts)) { + row_count += chunk_index_entry->GetRowCount(); + old_chunks.push_back(chunk_index_entry.get()); + } + } + } + RowID base_rowid(segment_id_, 0); + SharedPtr merged_chunk_index_entry = CreateSecondaryIndexChunkIndexEntry(base_rowid, row_count, buffer_mgr); + BufferHandle handle = merged_chunk_index_entry->GetIndex(); + auto data_ptr = static_cast(handle.GetDataMut()); + data_ptr->InsertMergeData(old_chunks, merged_chunk_index_entry); + ReplaceChunkIndexEntries(txn_table_store, merged_chunk_index_entry, std::move(old_chunks)); + txn_table_store->AddChunkIndexStore(table_index_entry_, merged_chunk_index_entry.get()); + return merged_chunk_index_entry.get(); + } default: { UnrecoverableError("RebuildChunkIndexEntries is not supported for this index type."); } diff --git a/src/storage/meta/entry/table_entry.cpp b/src/storage/meta/entry/table_entry.cpp index a42a3099a3..3e374abbd0 100644 --- a/src/storage/meta/entry/table_entry.cpp +++ b/src/storage/meta/entry/table_entry.cpp @@ -839,7 +839,8 @@ void TableEntry::OptimizeIndex(Txn *txn) { } break; } - case IndexType::kHnsw: { + case IndexType::kHnsw: + case IndexType::kSecondary: { TxnTimeStamp begin_ts = txn->BeginTS(); for (auto &[segment_id, segment_index_entry] : table_index_entry->index_by_segment()) { SegmentEntry *segment_entry = GetSegmentByID(segment_id, begin_ts).get(); diff --git a/src/storage/secondary_index/secondary_index_data.cpp b/src/storage/secondary_index/secondary_index_data.cpp index dfa0d3603e..a4c3308236 100644 --- a/src/storage/secondary_index/secondary_index_data.cpp +++ b/src/storage/secondary_index/secondary_index_data.cpp @@ -34,6 +34,74 @@ import buffer_handle; namespace infinity { +template +struct SecondaryIndexChunkDataReader { + using OrderedKeyType = ConvertToOrderedType; + static constexpr u32 PairSize = sizeof(OrderedKeyType) + sizeof(SegmentOffset); + ChunkIndexEntry *chunk_index_; + BufferHandle current_handle_; + u32 part_count_ = 0; + u32 current_offset_ = 0; + u32 current_part_id_ = 0; + u32 current_part_size_ = 0; + SecondaryIndexChunkDataReader(ChunkIndexEntry *chunk_index) : chunk_index_(chunk_index) { + part_count_ = chunk_index_->GetPartNum(); + current_part_size_ = chunk_index_->GetPartRowCount(current_part_id_); + } + bool GetNextDataPair(OrderedKeyType &key, u32 &offset) { + if (current_offset_ == 0) { + if (current_part_id_ >= part_count_) { + return false; + } + current_handle_ = chunk_index_->GetIndexPartAt(current_part_id_); + } + const auto *data_ptr = static_cast(current_handle_.GetData()); + std::memcpy(&key, data_ptr + current_offset_ * PairSize, sizeof(OrderedKeyType)); + std::memcpy(&offset, data_ptr + current_offset_ * PairSize + sizeof(OrderedKeyType), sizeof(SegmentOffset)); + if (++current_offset_ == current_part_size_) { + current_offset_ = 0; + if (++current_part_id_ < part_count_) { + current_part_size_ = chunk_index_->GetPartRowCount(current_part_id_); + } + } + return true; + } +}; + +template +struct SecondaryIndexChunkMerger { + using OrderedKeyType = ConvertToOrderedType; + Vector> readers_; + std::priority_queue, Vector>, std::greater>> pq_; + explicit SecondaryIndexChunkMerger(const Vector &old_chunks) { + for (ChunkIndexEntry *chunk : old_chunks) { + readers_.emplace_back(chunk); + } + OrderedKeyType key = {}; + u32 offset = 0; + for (u32 i = 0; i < readers_.size(); ++i) { + if (readers_[i].GetNextDataPair(key, offset)) { + pq_.emplace(key, offset, i); + } + } + } + bool GetNextDataPair(OrderedKeyType &out_key, u32 &out_offset) { + if (pq_.empty()) { + return false; + } + const auto [key, offset, reader_id] = pq_.top(); + out_key = key; + out_offset = offset; + pq_.pop(); + OrderedKeyType next_key = {}; + u32 next_offset = 0; + if (readers_[reader_id].GetNextDataPair(next_key, next_offset)) { + pq_.emplace(next_key, next_offset, reader_id); + } + return true; + } +}; + template class SecondaryIndexDataT final : public SecondaryIndexData { using OrderedKeyType = ConvertToOrderedType; @@ -86,6 +154,29 @@ class SecondaryIndexDataT final : public SecondaryIndexData { if (i != chunk_row_count_) { UnrecoverableError(fmt::format("InsertData(): error: i: {} != chunk_row_count_: {}", i, chunk_row_count_)); } + OutputAndBuild(chunk_index); + } + + void InsertMergeData(Vector &old_chunks, SharedPtr &merged_chunk_index_entry) override { + if (!need_save_) { + UnrecoverableError("InsertMergeData(): error: SecondaryIndexDataT is not allocated."); + } + SecondaryIndexChunkMerger merger(old_chunks); + OrderedKeyType key = {}; + u32 offset = 0; + u32 i = 0; + while (merger.GetNextDataPair(key, offset)) { + key_[i] = key; + offset_[i] = offset; + ++i; + } + if (i != chunk_row_count_) { + UnrecoverableError(fmt::format("InsertMergeData(): error: i: {} != chunk_row_count_: {}", i, chunk_row_count_)); + } + OutputAndBuild(merged_chunk_index_entry); + } + + void OutputAndBuild(SharedPtr &chunk_index) { const u32 part_num = chunk_index->GetPartNum(); for (u32 part_id = 0; part_id < part_num; ++part_id) { const u32 part_row_count = chunk_index->GetPartRowCount(part_id); @@ -94,10 +185,8 @@ class SecondaryIndexDataT final : public SecondaryIndexData { auto data_ptr = static_cast(handle.GetDataMut()); for (u32 j = 0; j < part_row_count; ++j) { const u32 index = part_offset + j; - const OrderedKeyType key = key_[index]; - const u32 offset = offset_[index]; - std::memcpy(data_ptr + j * PairSize, &key, sizeof(OrderedKeyType)); - std::memcpy(data_ptr + j * PairSize + sizeof(OrderedKeyType), &offset, sizeof(SegmentOffset)); + std::memcpy(data_ptr + j * PairSize, key_.get() + index, sizeof(OrderedKeyType)); + std::memcpy(data_ptr + j * PairSize + sizeof(OrderedKeyType), offset_.get() + index, sizeof(SegmentOffset)); } } pgm_index_->BuildIndex(chunk_row_count_, key_.get()); diff --git a/src/storage/secondary_index/secondary_index_data.cppm b/src/storage/secondary_index/secondary_index_data.cppm index 2580f2f715..253cbb48b4 100644 --- a/src/storage/secondary_index/secondary_index_data.cppm +++ b/src/storage/secondary_index/secondary_index_data.cppm @@ -28,6 +28,7 @@ import logical_type; import internal_types; import data_type; import segment_entry; +import buffer_handle; namespace infinity { class ChunkIndexEntry; @@ -134,6 +135,8 @@ public: virtual void ReadIndexInner(FileHandler &file_handler) = 0; virtual void InsertData(void *ptr, SharedPtr &chunk_index) = 0; + + virtual void InsertMergeData(Vector &old_chunks, SharedPtr &merged_chunk_index_entry) = 0; }; export SecondaryIndexData *GetSecondaryIndexData(const SharedPtr &data_type, u32 chunk_row_count, bool allocate); From 440669e35ff7bba3c7d07e3938e1fdce41756069 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Tue, 7 May 2024 10:09:50 +0800 Subject: [PATCH 7/9] unittest --- .../secondary_index/secondary_index_data.cpp | 1 + .../knnindex/merge_optimize/test_optimize.cpp | 95 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/src/storage/secondary_index/secondary_index_data.cpp b/src/storage/secondary_index/secondary_index_data.cpp index a4c3308236..c55d0939fb 100644 --- a/src/storage/secondary_index/secondary_index_data.cpp +++ b/src/storage/secondary_index/secondary_index_data.cpp @@ -74,6 +74,7 @@ struct SecondaryIndexChunkMerger { Vector> readers_; std::priority_queue, Vector>, std::greater>> pq_; explicit SecondaryIndexChunkMerger(const Vector &old_chunks) { + readers_.reserve(old_chunks.size()); for (ChunkIndexEntry *chunk : old_chunks) { readers_.emplace_back(chunk); } diff --git a/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp b/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp index 6b13ed46ff..ec3ffd9213 100644 --- a/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp +++ b/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp @@ -31,6 +31,7 @@ import logical_type; import column_vector; import data_block; import index_hnsw; +import index_secondary; import statement_common; import embedding_info; import knn_expr; @@ -218,3 +219,97 @@ TEST_F(OptimizeKnnTest, test1) { txn_mgr->CommitTxn(txn); } } + +TEST_F(OptimizeKnnTest, test_secondary_index_optimize) { + Storage *storage = InfinityContext::instance().storage(); + Catalog *catalog = storage->catalog(); + TxnManager *txn_mgr = storage->txn_manager(); + + auto db_name = std::make_shared("default_db"); + auto column_def1 = + std::make_shared(0, std::make_shared(LogicalType::kInteger), "col1", std::unordered_set{}); + auto table_name = std::make_shared("tb1"); + auto table_def = TableDef::Make(db_name, table_name, {column_def1}); + auto index_name = std::make_shared("idx1"); + + { + auto *txn = txn_mgr->BeginTxn(MakeUnique("create table")); + txn->CreateTable(*db_name, table_def, ConflictType::kError); + txn_mgr->CommitTxn(txn); + } + + { + Vector column_names{"col1"}; + const String &file_name = "idx_file.idx"; + auto *txn = txn_mgr->BeginTxn(MakeUnique("create index")); + auto [table_entry, status] = txn->GetTableByName(*db_name, *table_name); + ASSERT_TRUE(status.ok()); + auto index_secondary = IndexSecondary::Make(index_name, file_name, column_names); + auto [table_index_entry, status2] = txn->CreateIndexDef(table_entry, index_secondary, ConflictType::kError); + ASSERT_TRUE(status2.ok()); + txn_mgr->CommitTxn(txn); + } + + auto DoAppend = [&]() { + auto *txn = txn_mgr->BeginTxn(MakeUnique("insert table")); + Vector> column_vectors; + for (SizeT i = 0; i < table_def->columns().size(); ++i) { + SharedPtr data_type = table_def->columns()[i]->type(); + column_vectors.push_back(MakeShared(data_type)); + column_vectors.back()->Initialize(); + } + Vector col1{2, 4, 6, 8}; + SizeT row_cnt = 4; + for (SizeT i = 0; i < row_cnt; ++i) { + column_vectors[0]->AppendByPtr(reinterpret_cast(&col1[i])); + } + auto data_block = DataBlock::Make(); + data_block->Init(column_vectors); + + auto status = txn->Append(*db_name, *table_name, data_block); + ASSERT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn); + }; + + for (int j = 0; j < 3; ++j) { + for (int i = 0; i < 2; ++i) { + DoAppend(); + } + { + auto *txn = txn_mgr->BeginTxn(MakeUnique("insert table")); + auto [table_entry, status1] = txn->GetTableByName(*db_name, *table_name); + ASSERT_TRUE(status1.ok()); + auto [table_index_entry, status] = txn->GetIndexByName(*db_name, *table_name, *index_name); + ASSERT_TRUE(status.ok()); + TxnTableStore *txn_table_store = txn->GetTxnTableStore(table_entry); + TxnIndexStore *txn_index_store = txn_table_store->GetIndexStore(table_index_entry); + table_index_entry->MemIndexDump(txn_index_store, true); + txn_mgr->CommitTxn(txn); + } + } + TxnTimeStamp last_commit_ts = 0; + { + Txn *txn = txn_mgr->BeginTxn(MakeUnique("optimize index")); + auto [table_entry, status] = txn->GetTableByName(*db_name, *table_name); + ASSERT_TRUE(status.ok()); + table_entry->OptimizeIndex(txn); + last_commit_ts = txn_mgr->CommitTxn(txn); + } + WaitCleanup(catalog, txn_mgr, last_commit_ts); + { + auto *txn = txn_mgr->BeginTxn(MakeUnique("check index")); + auto [table_entry, status1] = txn->GetTableByName(*db_name, *table_name); + ASSERT_TRUE(status1.ok()); + auto [table_index_entry, status] = txn->GetIndexByName(*db_name, *table_name, *index_name); + ASSERT_TRUE(status.ok()); + auto &segment_index_entries = table_index_entry->index_by_segment(); + ASSERT_EQ(segment_index_entries.size(), 1ul); + auto &segment_index_entry = segment_index_entries.begin()->second; + auto [chunk_index_entries, memory_index_entry] = segment_index_entry->GetSecondaryIndexSnapshot(); + ASSERT_EQ(chunk_index_entries.size(), 1ul); + auto &chunk_index_entry = chunk_index_entries[0]; + ASSERT_EQ(chunk_index_entry->row_count_, 24u); + ASSERT_EQ(memory_index_entry.get(), nullptr); + txn_mgr->CommitTxn(txn); + } +} \ No newline at end of file From c144c416f320da8ac3d728f78e6d42a7be06bfd7 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Wed, 8 May 2024 11:50:56 +0800 Subject: [PATCH 8/9] fix conflict --- .../storage/knnindex/merge_optimize/test_optimize.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp b/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp index ec3ffd9213..31e2e508b5 100644 --- a/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp +++ b/src/unit_test/storage/knnindex/merge_optimize/test_optimize.cpp @@ -265,8 +265,9 @@ TEST_F(OptimizeKnnTest, test_secondary_index_optimize) { } auto data_block = DataBlock::Make(); data_block->Init(column_vectors); - - auto status = txn->Append(*db_name, *table_name, data_block); + auto [table_entry, status] = txn->GetTableByName(*db_name, *table_name); + ASSERT_TRUE(status.ok()); + status = txn->Append(table_entry, data_block); ASSERT_TRUE(status.ok()); txn_mgr->CommitTxn(txn); }; From da778a36d2ff8957b7ef08756a90ffac50962e27 Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Wed, 8 May 2024 13:41:05 +0800 Subject: [PATCH 9/9] fix double type --- .../secondary_index/secondary_index_pgm.cppm | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/storage/secondary_index/secondary_index_pgm.cppm b/src/storage/secondary_index/secondary_index_pgm.cppm index 28cc7d4f23..94f46eba85 100644 --- a/src/storage/secondary_index/secondary_index_pgm.cppm +++ b/src/storage/secondary_index/secondary_index_pgm.cppm @@ -30,8 +30,23 @@ struct SecondaryIndexApproxPos { SizeT upper_bound_{}; ///< The upper bound of the range. }; +template +struct PGMT; + +template + requires std::is_integral_v || std::is_same_v +struct PGMT { + using type = PGMIndex; +}; + +template + requires std::is_same_v +struct PGMT { + using type = PGMIndex; +}; + template -using PGMIndexType = PGMIndex; +using PGMIndexType = PGMT::type; // PGMIndex member objects: // size_t n; ///< The number of elements this index was built on.