Skip to content

Commit 14a258d

Browse files
k0ushalk0ushal
authored
Merge pull request #9 from arangodb/bugfix/iresearch-address-table-tests
IResearch address_table tests
2 parents 61a8a3c + 872d553 commit 14a258d

File tree

5 files changed

+273
-26
lines changed

5 files changed

+273
-26
lines changed

‎core/formats/columnstore2.hpp‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
////////////////////////////////////////////////////////////////////////////////
2222

2323
#pragma once
24-
2524
#include "formats/formats.hpp"
2625
#include "formats/sparse_bitmap.hpp"
2726
#include "resource_manager.hpp"
@@ -80,9 +79,10 @@ class column final : public irs::column_output {
8079
private:
8180
friend class writer;
8281

82+
public:
8383
class address_table {
8484
public:
85-
address_table(ManagedTypedAllocator<uint64_t> alloc) : alloc_{alloc} {
85+
address_table(ManagedTypedAllocator<uint64_t> alloc) : alloc_(alloc) {
8686
offsets_ = alloc_.allocate(kBlockSize);
8787
offset_ = offsets_;
8888
}
@@ -94,14 +94,22 @@ class column final : public irs::column_output {
9494
return offset_[-1];
9595
}
9696

97-
void push_back(uint64_t offset) noexcept {
97+
bool push_back(uint64_t offset) noexcept {
9898
IRS_ASSERT(offset_ < offsets_ + kBlockSize);
99+
if (!(offset_ < offsets_ + kBlockSize))
100+
return false;
101+
99102
*offset_++ = offset;
103+
return true;
100104
}
101105

102-
void pop_back() noexcept {
106+
bool pop_back() noexcept {
103107
IRS_ASSERT(offsets_ < offset_);
108+
if (!(offsets_ < offset_))
109+
return false;
110+
104111
--offset_;
112+
return true;
105113
}
106114

107115
uint32_t size() const noexcept {
@@ -124,6 +132,7 @@ class column final : public irs::column_output {
124132
uint64_t* offset_{nullptr};
125133
};
126134

135+
private:
127136
void Prepare(doc_id_t key) final;
128137

129138
bool empty() const noexcept { return addr_table_.empty() && !docs_count_; }

‎core/resource_manager.hpp‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,18 +83,18 @@ struct IResearchMemoryManager : public IResourceManager {
8383
public:
8484
virtual ~IResearchMemoryManager() = default;
8585

86-
virtual void Increase([[maybe_unused]] uint64_t value) override {
86+
virtual void Increase([[maybe_unused]] size_t value) override {
8787

8888
IRS_ASSERT(this != &kForbidden);
8989
IRS_ASSERT(value >= 0);
9090

91-
if (0 == _memoryLimit) {
91+
if (_memoryLimit == 0) {
9292
// since we have no limit, we can simply use fetch-add for the increment
9393
_current.fetch_add(value, std::memory_order_relaxed);
9494
} else {
9595
// we only want to perform the update if we don't exceed the limit!
96-
std::uint64_t cur = _current.load(std::memory_order_relaxed);
97-
std::uint64_t next;
96+
size_t cur = _current.load(std::memory_order_relaxed);
97+
size_t next;
9898
do {
9999
next = cur + value;
100100
if (IRS_UNLIKELY(next > _memoryLimit.load(std::memory_order_relaxed))) {
@@ -105,25 +105,29 @@ struct IResearchMemoryManager : public IResourceManager {
105105
}
106106
}
107107

108-
virtual void Decrease([[maybe_unused]] uint64_t value) noexcept override {
108+
virtual void Decrease([[maybe_unused]] size_t value) noexcept override {
109109
IRS_ASSERT(this != &kForbidden);
110110
IRS_ASSERT(value >= 0);
111111
_current.fetch_sub(value, std::memory_order_relaxed);
112112
}
113113

114114
// NOTE: IResearchFeature owns and manages this memory limit.
115115
// That is why this method should only be used by IResearchFeature.
116-
virtual void SetMemoryLimit(uint64_t memoryLimit) {
116+
virtual void SetMemoryLimit(size_t memoryLimit) {
117117
_memoryLimit.store(memoryLimit);
118118
}
119119

120+
size_t getCurrentUsage() {
121+
return _current.load(std::memory_order_relaxed);
122+
}
123+
120124
private:
121125
// This limit should be set exclusively by IResearchFeature.
122126
// During IResearchFeature::validateOptions() this limit is set to a
123127
// percentage of either the total available physical memory or the value
124128
// of ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY envvar if specified.
125-
std::atomic<std::uint64_t> _memoryLimit = { 0 };
126-
std::atomic<std::uint64_t> _current = { 0 };
129+
std::atomic<size_t> _memoryLimit = { 0 };
130+
std::atomic<size_t> _current = { 0 };
127131

128132
// Singleton
129133
static inline std::shared_ptr<IResearchMemoryManager> _instance;

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ set(IReSearch_tests_sources
4040
./formats/formats_tests.cpp
4141
./formats/formats_test_case_base.cpp
4242
./formats/skip_list_test.cpp
43+
./formats/address_table_tests.cpp
4344
./store/directory_test_case.cpp
4445
./store/caching_directory_test.cpp
4546
./store/directory_cleaner_tests.cpp

‎tests/formats/address_table_tests.cpp‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
2+
#include <iostream>
3+
#include "formats/columnstore2.hpp"
4+
#include "resource_manager.hpp"
5+
#include <gtest/gtest.h>
6+
7+
using irs::columnstore2::column;
8+
9+
namespace {
10+
11+
class address_table_tests : public ::testing::Test {
12+
public:
13+
address_table_tests() { }
14+
15+
void SetUp() override {
16+
}
17+
18+
void TearDown() override {}
19+
20+
public:
21+
irs::ManagedTypedAllocator<uint64_t> alloc;
22+
};
23+
}
24+
25+
TEST_F(address_table_tests, smoke_test) {
26+
27+
column::address_table addr_table(alloc);
28+
29+
ASSERT_EQ(0, addr_table.size());
30+
31+
// push
32+
ASSERT_TRUE(addr_table.push_back(1));
33+
ASSERT_EQ(1, addr_table.size());
34+
35+
// pop
36+
ASSERT_TRUE(addr_table.pop_back());
37+
ASSERT_EQ(0, addr_table.size());
38+
39+
// full
40+
for (uint64_t i = 0; i < column::kBlockSize; i++) {
41+
ASSERT_TRUE(addr_table.push_back(i));
42+
}
43+
ASSERT_EQ(addr_table.size(), column::kBlockSize);
44+
ASSERT_TRUE(addr_table.full());
45+
46+
// reset
47+
addr_table.reset();
48+
ASSERT_TRUE(addr_table.empty());
49+
}
50+
51+
// pop when empty
52+
TEST_F(address_table_tests, pop_when_empty) {
53+
54+
column::address_table addr_table(alloc);
55+
ASSERT_TRUE(addr_table.empty());
56+
ASSERT_FALSE(addr_table.pop_back());
57+
}
58+
59+
// push when full
60+
TEST_F(address_table_tests, push_when_full) {
61+
62+
column::address_table addr_table(alloc);
63+
64+
ASSERT_TRUE(addr_table.empty());
65+
66+
for (uint64_t i = 0; i < column::kBlockSize; i++) {
67+
ASSERT_TRUE(addr_table.push_back(i));
68+
}
69+
70+
ASSERT_EQ(addr_table.size(), column::kBlockSize);
71+
ASSERT_FALSE(addr_table.push_back(2));
72+
73+
ASSERT_TRUE(addr_table.pop_back());
74+
ASSERT_TRUE(addr_table.push_back(2));
75+
}
76+
77+
// push and verify data
78+
TEST_F(address_table_tests, verify_data) {
79+
80+
column::address_table addr_table(alloc);
81+
82+
for (uint64_t i = 0; i < 200; i++) {
83+
ASSERT_TRUE(addr_table.push_back(i));
84+
}
85+
86+
ASSERT_EQ(addr_table.size(), 200);
87+
88+
auto curr = addr_table.current();
89+
auto begin = addr_table.begin();
90+
91+
uint64_t data { 199 };
92+
while (curr != begin) {
93+
--curr;
94+
ASSERT_EQ(*curr, data--);
95+
}
96+
}
97+
98+
// write using current ptr
99+
TEST_F(address_table_tests, write_using_current_ptr) {
100+
101+
column::address_table addr_table(alloc);
102+
103+
// add data via push_back.
104+
auto data = 0;
105+
for (uint64_t i = 0; i < 60; i++) {
106+
ASSERT_TRUE(addr_table.push_back(data++));
107+
}
108+
109+
// add data via current() ptr.
110+
// column::flush_block() uses address_table
111+
// in this way.
112+
auto curr = addr_table.current();
113+
uint64_t addr_table_size = 40900;
114+
auto end = curr + addr_table_size;
115+
116+
while (curr != end) {
117+
*curr++ = data++;
118+
}
119+
120+
// size is updated only when adding data
121+
// via push_back().
122+
ASSERT_EQ(addr_table.size(), 60);
123+
124+
auto begin = addr_table.begin();
125+
for (uint64_t i = 0; i < 40960; i++) {
126+
ASSERT_EQ(*begin++, i);
127+
}
128+
}
129+
130+
// size arithmetic
131+
TEST_F(address_table_tests, begin_current_end) {
132+
133+
column::address_table addr_table(alloc);
134+
135+
ASSERT_EQ(addr_table.size(), 0);
136+
ASSERT_TRUE(addr_table.push_back(45));
137+
ASSERT_EQ(addr_table.size(), 1);
138+
139+
ASSERT_EQ(addr_table.current(), addr_table.begin() + addr_table.size());
140+
}
141+
142+
// max size allowed
143+
TEST_F(address_table_tests, max_size_allowed) {
144+
145+
column::address_table addr_table(alloc);
146+
147+
// end() always points to begin() + kBlockSize
148+
// coz address_table pre-allocates memory to
149+
// accommodate kBlockSize elements.
150+
ASSERT_EQ(addr_table.size(), 0);
151+
ASSERT_EQ(addr_table.end(), addr_table.begin() + column::kBlockSize);
152+
ASSERT_EQ(addr_table.current(), addr_table.begin() + 0);
153+
154+
for (uint64_t i = 0; i < 1234; i++) {
155+
ASSERT_TRUE(addr_table.push_back(i));
156+
}
157+
158+
ASSERT_EQ(addr_table.end(), addr_table.begin() + column::kBlockSize);
159+
ASSERT_EQ(addr_table.current(), addr_table.begin() + 1234);
160+
}
161+
162+
// full
163+
TEST_F(address_table_tests, empty_full_and_reset) {
164+
165+
column::address_table addr_table(alloc);
166+
167+
ASSERT_FALSE(addr_table.full());
168+
for (uint64_t i = 0; i < column::kBlockSize; i++) {
169+
ASSERT_TRUE(addr_table.push_back(i));
170+
}
171+
ASSERT_TRUE(addr_table.full());
172+
173+
addr_table.pop_back();
174+
ASSERT_FALSE(addr_table.full());
175+
ASSERT_EQ(addr_table.current() + 1, addr_table.end());
176+
177+
addr_table.reset();
178+
ASSERT_FALSE(addr_table.full());
179+
ASSERT_EQ(addr_table.current(), addr_table.begin());
180+
ASSERT_EQ(addr_table.size(), 0);
181+
}

0 commit comments

Comments
 (0)