Skip to content

Commit dc042db

Browse files
Ma-catyuzhichang
Ma-cat
and
yuzhichang
authored
Added unittest for BufferedSkipListWriter (infiniflow#839)
Added unittest for BufferedSkipListWriter Issue link:[infiniflow#813](infiniflow#813) ### Type of change - [x] Test cases --------- Co-authored-by: Zhichang Yu <yuzhichang@gmail.com>
1 parent 80d04e1 commit dc042db

File tree

3 files changed

+279
-1
lines changed

3 files changed

+279
-1
lines changed

‎src/storage/invertedindex/format/buffered_skiplist_writer.cpp

+8-1
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@ import short_list_optimize_util;
1515
namespace infinity {
1616

1717
BufferedSkipListWriter::BufferedSkipListWriter(MemoryPool *byte_slice_pool, MemoryPool *buffer_pool)
18-
: BufferedByteSlice(byte_slice_pool, buffer_pool), last_key_(0), last_value1_(0) {}
18+
: BufferedByteSlice(byte_slice_pool, buffer_pool), last_key_(-1), last_value1_(-1) {}
1919

2020
void BufferedSkipListWriter::AddItem(u32 key, u32 value1, u32 value2) {
21+
assert(static_cast<u32>(-1) == last_key_ || key > last_key_);
22+
assert(static_cast<u32>(-1) == last_value1_ || value1 > last_value1_);
23+
last_key_ = static_cast<u32>(-1) == last_key_ ? 0 : last_key_;
24+
last_value1_ = static_cast<u32>(-1) == last_value1_ ? 0 : last_value1_;
2125
PushBack(0, key - last_key_);
2226
PushBack(1, value1 - last_value1_);
2327
last_key_ = key;
@@ -30,6 +34,8 @@ void BufferedSkipListWriter::AddItem(u32 key, u32 value1, u32 value2) {
3034
}
3135

3236
void BufferedSkipListWriter::AddItem(u32 key, u32 value1) {
37+
assert(static_cast<u32>(-1) == last_key_ || key > last_key_);
38+
last_key_ = static_cast<u32>(-1) == last_key_ ? 0 : last_key_;
3339
PushBack(0, key - last_key_);
3440
PushBack(1, value1);
3541
EndPushBack();
@@ -42,6 +48,7 @@ void BufferedSkipListWriter::AddItem(u32 key, u32 value1) {
4248
}
4349

4450
void BufferedSkipListWriter::AddItem(u32 value_delta) {
51+
last_value1_ = static_cast<u32>(-1) == last_value1_ ? 0 : last_value1_;
4552
last_value1_ += value_delta;
4653
PushBack(0, last_value1_);
4754
EndPushBack();

‎src/unit_test/storage/invertedindex/format/buffered_skiplist_writer.cpp

+239
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
#include "unit_test/base_test.h"
2+
3+
import stl;
4+
import memory_pool;
5+
import buffered_skiplist_writer;
6+
import buffered_byte_slice_reader;
7+
import buffered_byte_slice;
8+
import index_defines;
9+
import posting_field;
10+
import file_writer;
11+
import file_system;
12+
import local_file_system;
13+
14+
using namespace infinity;
15+
16+
class BufferedSkipListWriterTest : public BaseTest {
17+
public:
18+
BufferedSkipListWriterTest() {
19+
byte_slice_pool_ = new MemoryPool(BUFFER_SIZE_);
20+
buffer_pool_ = new RecyclePool(BUFFER_SIZE_);
21+
}
22+
~BufferedSkipListWriterTest() {
23+
delete byte_slice_pool_;
24+
delete buffer_pool_;
25+
}
26+
void SetUp() override {
27+
buffered_skiplist_writer_ = MakeShared<BufferedSkipListWriter>(byte_slice_pool_, buffer_pool_);
28+
}
29+
30+
void TearDown() override { buffered_skiplist_writer_.reset(); }
31+
32+
protected:
33+
SharedPtr<FileWriter> CreateFileWriter(String file_name);
34+
35+
protected:
36+
MemoryPool *byte_slice_pool_;
37+
RecyclePool *buffer_pool_;
38+
SharedPtr<BufferedSkipListWriter> buffered_skiplist_writer_ = nullptr;
39+
const SizeT BUFFER_SIZE_ = 1024;
40+
};
41+
42+
SharedPtr<FileWriter> BufferedSkipListWriterTest::CreateFileWriter(infinity::String file_path) {
43+
auto fs = MakeShared<LocalFileSystem>();
44+
return MakeShared<FileWriter>(*fs, file_path, BUFFER_SIZE_);
45+
}
46+
47+
TEST_F(BufferedSkipListWriterTest, test1) {
48+
using namespace infinity;
49+
50+
ASSERT_NE(nullptr, buffered_skiplist_writer_);
51+
PostingFields posting_fields;
52+
auto posting_field = new TypedPostingField<u32>();
53+
posting_fields.AddValue(posting_field);
54+
buffered_skiplist_writer_->Init(&posting_fields);
55+
const u32 delta = 10;
56+
57+
String file_path = "buffered_skiplist_writer_test1.txt";
58+
auto file_writer = CreateFileWriter(file_path);
59+
60+
Vector<u32> expected_values;
61+
u32 now_value = 0;
62+
for (int i = 0; i < SKIP_LIST_BUFFER_SIZE + 2; ++i) {
63+
buffered_skiplist_writer_->AddItem(delta);
64+
now_value += delta;
65+
expected_values.emplace_back(now_value);
66+
}
67+
68+
buffered_skiplist_writer_->Dump(file_writer);
69+
70+
auto reader = MakeShared<BufferedByteSliceReader>();
71+
reader->Open(buffered_skiplist_writer_.get());
72+
73+
u32 val_buffer[BUFFER_SIZE_];
74+
SizeT decode_len;
75+
ASSERT_TRUE(reader->Decode<u32>(val_buffer, SKIP_LIST_BUFFER_SIZE, decode_len));
76+
ASSERT_TRUE(reader->Decode<u32>(val_buffer + SKIP_LIST_BUFFER_SIZE, 2, decode_len));
77+
78+
for (size_t i = 0; i < SKIP_LIST_BUFFER_SIZE + 2; ++i) {
79+
auto value = val_buffer[i];
80+
ASSERT_EQ(value, expected_values[i]);
81+
}
82+
}
83+
84+
TEST_F(BufferedSkipListWriterTest, test2) {
85+
using namespace infinity;
86+
87+
ASSERT_NE(nullptr, buffered_skiplist_writer_);
88+
PostingFields posting_fields;
89+
90+
u8 row_count = 0;
91+
u32 offset = 0;
92+
93+
auto posting_field_key = new TypedPostingField<u32>;
94+
posting_field_key->location_ = row_count++;
95+
posting_field_key->offset_ = offset;
96+
offset += sizeof(u32);
97+
98+
auto posting_field_value = new TypedPostingField<u32>;
99+
posting_field_value->location_ = row_count++;
100+
posting_field_value->offset_ = offset;
101+
offset += sizeof(u32);
102+
103+
posting_fields.AddValue(posting_field_key);
104+
posting_fields.AddValue(posting_field_value);
105+
106+
buffered_skiplist_writer_->Init(&posting_fields);
107+
const u32 delta = 10;
108+
109+
String file_path = "buffered_skiplist_writer_test2.txt";
110+
auto file_writer = CreateFileWriter(file_path);
111+
112+
Vector<u32> expected_keys;
113+
Vector<u32> expected_values;
114+
115+
u32 now_key = 0;
116+
u32 now_value = 0;
117+
118+
buffered_skiplist_writer_->AddItem(now_key, now_value);
119+
expected_keys.emplace_back(now_key);
120+
expected_values.emplace_back(now_value);
121+
122+
for (int i = 0; i < SKIP_LIST_BUFFER_SIZE + 2; ++i) {
123+
now_key += delta;
124+
now_value += delta - 1;
125+
buffered_skiplist_writer_->AddItem(now_key, now_value);
126+
expected_keys.emplace_back(now_key);
127+
expected_values.emplace_back(now_value);
128+
}
129+
130+
buffered_skiplist_writer_->Dump(file_writer);
131+
132+
auto reader = MakeShared<BufferedByteSliceReader>();
133+
reader->Open(buffered_skiplist_writer_.get());
134+
135+
u32 key_buffer[BUFFER_SIZE_];
136+
u32 val_buffer[BUFFER_SIZE_];
137+
SizeT decode_len;
138+
139+
ASSERT_TRUE(reader->Decode(key_buffer, SKIP_LIST_BUFFER_SIZE, decode_len));
140+
ASSERT_TRUE(reader->Decode(val_buffer, SKIP_LIST_BUFFER_SIZE, decode_len));
141+
ASSERT_TRUE(reader->Decode(key_buffer + SKIP_LIST_BUFFER_SIZE, 3, decode_len));
142+
ASSERT_TRUE(reader->Decode(val_buffer + SKIP_LIST_BUFFER_SIZE, 3, decode_len));
143+
144+
u32 read_key = 0;
145+
u32 read_value = 0;
146+
for (size_t i = 0; i < SKIP_LIST_BUFFER_SIZE + 2; ++i) {
147+
read_key += key_buffer[i];
148+
read_value = val_buffer[i];
149+
ASSERT_EQ(read_key, expected_keys[i]);
150+
ASSERT_EQ(read_value, expected_values[i]);
151+
}
152+
153+
}
154+
155+
TEST_F(BufferedSkipListWriterTest, test3) {
156+
using namespace infinity;
157+
158+
ASSERT_NE(nullptr, buffered_skiplist_writer_);
159+
PostingFields posting_fields;
160+
161+
u8 row_count = 0;
162+
u32 offset = 0;
163+
164+
auto posting_field_key = new TypedPostingField<u32>;
165+
posting_field_key->location_ = row_count++;
166+
posting_field_key->offset_ = offset;
167+
offset += sizeof(u32);
168+
169+
auto posting_field_value1 = new TypedPostingField<u32>;
170+
posting_field_value1->location_ = row_count++;
171+
posting_field_value1->offset_ = offset;
172+
offset += sizeof(u32);
173+
174+
auto posting_field_value2 = new TypedPostingField<u32>;
175+
posting_field_value2->location_ = row_count++;
176+
posting_field_value2->offset_ = offset;
177+
offset += sizeof(u32);
178+
179+
posting_fields.AddValue(posting_field_key);
180+
posting_fields.AddValue(posting_field_value1);
181+
posting_fields.AddValue(posting_field_value2);
182+
183+
buffered_skiplist_writer_->Init(&posting_fields);
184+
const u32 delta = 10;
185+
186+
String file_path = "buffered_skiplist_writer_test3.txt";
187+
auto file_writer = CreateFileWriter(file_path);
188+
189+
Vector<u32> expected_keys;
190+
Vector<u32> expected_values1;
191+
Vector<u32> expected_values2;
192+
193+
u32 now_key = 0;
194+
u32 now_value1 = 0;
195+
u32 now_value2 = 0;
196+
buffered_skiplist_writer_->AddItem(now_key, now_value1, now_value2);
197+
expected_keys.emplace_back(now_key);
198+
expected_values1.emplace_back(now_value1);
199+
expected_values2.emplace_back(now_value2);
200+
201+
for (int i = 0; i < SKIP_LIST_BUFFER_SIZE + 2; ++i) {
202+
now_key += delta;
203+
now_value1 += delta - 1;
204+
now_value2 += delta - 2;
205+
buffered_skiplist_writer_->AddItem(now_key, now_value1, now_value2);
206+
expected_keys.emplace_back(now_key);
207+
expected_values1.emplace_back(now_value1);
208+
expected_values2.emplace_back(now_value2);
209+
}
210+
211+
buffered_skiplist_writer_->Dump(file_writer);
212+
213+
auto reader = MakeShared<BufferedByteSliceReader>();
214+
reader->Open(buffered_skiplist_writer_.get());
215+
216+
u32 key_buffer[BUFFER_SIZE_];
217+
u32 val1_buffer[BUFFER_SIZE_];
218+
u32 val2_buffer[BUFFER_SIZE_];
219+
SizeT decode_len;
220+
221+
ASSERT_TRUE(reader->Decode(key_buffer, SKIP_LIST_BUFFER_SIZE, decode_len));
222+
ASSERT_TRUE(reader->Decode(val1_buffer, SKIP_LIST_BUFFER_SIZE, decode_len));
223+
ASSERT_TRUE(reader->Decode(val2_buffer, SKIP_LIST_BUFFER_SIZE, decode_len));
224+
ASSERT_TRUE(reader->Decode(key_buffer + SKIP_LIST_BUFFER_SIZE, 3, decode_len));
225+
ASSERT_TRUE(reader->Decode(val1_buffer + SKIP_LIST_BUFFER_SIZE, 3, decode_len));
226+
ASSERT_TRUE(reader->Decode(val2_buffer + SKIP_LIST_BUFFER_SIZE, 3, decode_len));
227+
228+
u32 read_key = 0;
229+
u32 read_value1 = 0;
230+
u32 read_value2 = 0;
231+
for (size_t i = 0; i < SKIP_LIST_BUFFER_SIZE + 3; ++i) {
232+
read_key += key_buffer[i];
233+
read_value1 += val1_buffer[i];
234+
read_value2 = val2_buffer[i];
235+
ASSERT_EQ(read_key, expected_keys[i]);
236+
ASSERT_EQ(read_value1, expected_values1[i]);
237+
ASSERT_EQ(read_value2, expected_values2[i]);
238+
}
239+
}

‎src/unit_test/storage/invertedindex/format/doc_list_encoder.cpp

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#include "unit_test/base_test.h"
2+
3+
import stl;
4+
import memory_pool;
5+
import buffered_byte_slice;
6+
import buffered_byte_slice_reader;
7+
import doc_list_format_option;
8+
import index_defines;
9+
import flush_info;
10+
11+
using namespace infinity;
12+
13+
class DocListEncoderTest : public BaseTest {
14+
public:
15+
DocListEncoderTest() {}
16+
~DocListEncoderTest() {}
17+
void SetUp() override {}
18+
19+
void TearDown() override {}
20+
21+
protected:
22+
};
23+
24+
TEST_F(DocListEncoderTest, test1) {
25+
using namespace infinity;
26+
27+
// TODO yzc:
28+
// invoke AddPosition(), EndDocument() multiple times
29+
// invoke GetInMemDocListDecoder(), check the values
30+
// invode Dump()
31+
// read back via Load() (currently absent), and check the values
32+
}

0 commit comments

Comments
 (0)