Skip to content

Commit 2fcf371

Browse files
author
Abigale Kim
authored
Integrate std::prm memory tracking for class ArraySchemaEvolution. (#4736)
All member vector variables of the ArraySchemaEvolution class have PMR tracking. --- TYPE: NO_HISTORY DESC: Integrate std::prm memory tracking for class `ArraySchemaEvolution`.
1 parent 827be1c commit 2fcf371

File tree

9 files changed

+122
-49
lines changed

9 files changed

+122
-49
lines changed

‎test/src/unit-cppapi-schema-evolution.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -797,7 +797,8 @@ TEST_CASE(
797797
TEST_CASE(
798798
"SchemaEvolution Error Handling Tests",
799799
"[cppapi][schema][evolution][errors]") {
800-
auto ase = make_shared<tiledb::sm::ArraySchemaEvolution>(HERE());
800+
auto ase = make_shared<tiledb::sm::ArraySchemaEvolution>(
801+
HERE(), tiledb::test::create_test_memory_tracker());
801802
REQUIRE_THROWS(ase->evolve_schema(nullptr));
802803
REQUIRE_THROWS(ase->add_attribute(nullptr));
803804

‎test/src/unit-enumerations.cc‎

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1576,7 +1576,7 @@ TEST_CASE_METHOD(
15761576
array->load_all_enumerations();
15771577

15781578
auto orig_schema = array->array_schema_latest_ptr();
1579-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1579+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
15801580
auto attr3 = make_shared<Attribute>(HERE(), "attr3", Datatype::UINT32);
15811581
ase->add_attribute(attr3);
15821582
CHECK_NOTHROW(ase->evolve_schema(orig_schema));
@@ -1588,7 +1588,7 @@ TEST_CASE_METHOD(
15881588
"[enumeration][array-schema-evolution][simple]") {
15891589
create_array();
15901590
auto orig_schema = get_array_schema_latest();
1591-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1591+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
15921592

15931593
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
15941594
auto enmr = create_enumeration(values);
@@ -1609,7 +1609,7 @@ TEST_CASE_METHOD(
16091609
"[enumeration][array-schema-evolution][drop-add]") {
16101610
create_array();
16111611
auto orig_schema = get_array_schema_latest();
1612-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1612+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
16131613

16141614
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
16151615
auto enmr = create_enumeration(values);
@@ -1630,7 +1630,7 @@ TEST_CASE_METHOD(
16301630
create_array();
16311631
auto orig_schema = get_array_schema_latest();
16321632

1633-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1633+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
16341634

16351635
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
16361636
auto enmr = create_enumeration(values);
@@ -1649,7 +1649,7 @@ TEST_CASE_METHOD(
16491649
"[enumeration][array-schema-evolution][enmr-to-add]") {
16501650
create_array();
16511651
auto orig_schema = get_array_schema_latest();
1652-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1652+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
16531653

16541654
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
16551655
auto enmr1 = create_enumeration(values);
@@ -1675,7 +1675,7 @@ TEST_CASE_METHOD(
16751675
REQUIRE(old_enmr != nullptr);
16761676
auto new_enmr = extend_enumeration(old_enmr, values_to_add);
16771677

1678-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1678+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
16791679
ase->extend_enumeration(new_enmr);
16801680
CHECK_NOTHROW(ase->evolve_schema(orig_schema));
16811681
}
@@ -1684,7 +1684,7 @@ TEST_CASE_METHOD(
16841684
EnumerationFx,
16851685
"ArraySchemaEvolution - Drop Enumeration",
16861686
"[enumeration][array-schema-evolution][enmr-to-drop]") {
1687-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1687+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
16881688
CHECK_NOTHROW(ase->drop_enumeration("test_enmr"));
16891689
}
16901690

@@ -1694,15 +1694,15 @@ TEST_CASE_METHOD(
16941694
"[enumeration][array-schema-evolution][enmr-to-drop]") {
16951695
create_array();
16961696
auto orig_schema = get_array_schema_latest();
1697-
auto ase1 = make_shared<ArraySchemaEvolution>(HERE());
1697+
auto ase1 = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
16981698

16991699
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
17001700
auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr");
17011701
ase1->add_enumeration(enmr1);
17021702

17031703
auto new_schema = ase1->evolve_schema(orig_schema);
17041704

1705-
auto ase2 = make_shared<ArraySchemaEvolution>(HERE());
1705+
auto ase2 = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17061706
ase2->drop_enumeration("enmr");
17071707

17081708
CHECK_NOTHROW(ase2->evolve_schema(new_schema));
@@ -1712,7 +1712,7 @@ TEST_CASE_METHOD(
17121712
EnumerationFx,
17131713
"ArraySchemaEvolution - Drop Enumeration Repeated",
17141714
"[enumeration][array-schema-evolution][enmr-to-drop-repeated]") {
1715-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1715+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17161716
CHECK_NOTHROW(ase->drop_enumeration("test_enmr"));
17171717
CHECK_NOTHROW(ase->drop_enumeration("test_enmr"));
17181718
}
@@ -1721,7 +1721,7 @@ TEST_CASE_METHOD(
17211721
EnumerationFx,
17221722
"ArraySchemaEvolution - Drop Enumeration After Add",
17231723
"[enumeration][array-schema-evolution][enmr-add-drop]") {
1724-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1724+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17251725

17261726
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
17271727
auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr");
@@ -1736,7 +1736,7 @@ TEST_CASE_METHOD(
17361736
"ArraySchemaEvolution - Enumeration to Add - nullptr",
17371737
"[enumeration][array-schema-evolution][enmr-nullptr]") {
17381738
create_array();
1739-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1739+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17401740
REQUIRE_THROWS(ase->add_enumeration(nullptr));
17411741
}
17421742

@@ -1745,7 +1745,7 @@ TEST_CASE_METHOD(
17451745
"ArraySchemaEvolution - Enumeration to Add - Already Added",
17461746
"[enumeration][array-schema-evolution][enmr-already-added]") {
17471747
create_array();
1748-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1748+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17491749

17501750
std::vector<uint64_t> values{0, 1, 2, 3, 4, 1000};
17511751
auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr");
@@ -1759,7 +1759,7 @@ TEST_CASE_METHOD(
17591759
"ArraySchemaEvolution - Enumeration to Add - Missing Name",
17601760
"[enumeration][array-schema-evolution][missing-name]") {
17611761
create_array();
1762-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1762+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17631763
REQUIRE(ase->enumeration_to_add("foo") == nullptr);
17641764
}
17651765

@@ -1769,7 +1769,7 @@ TEST_CASE_METHOD(
17691769
"[enumeration][array-schema-evolution][enmr-still-in-use]") {
17701770
create_array();
17711771
auto orig_schema = get_array_schema_latest();
1772-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1772+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17731773
ase->drop_enumeration("test_enmr");
17741774

17751775
REQUIRE_THROWS(ase->evolve_schema(orig_schema));
@@ -1784,7 +1784,7 @@ TEST_CASE_METHOD(
17841784
auto attr3 = make_shared<Attribute>(HERE(), "attr3", Datatype::UINT32);
17851785
attr3->set_enumeration_name("test_enmr");
17861786

1787-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1787+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
17881788
ase->add_attribute(attr3);
17891789

17901790
auto orig_schema = get_array_schema_latest();
@@ -1805,7 +1805,7 @@ TEST_CASE_METHOD(
18051805
auto attr3 = make_shared<Attribute>(HERE(), "attr3", Datatype::INT8);
18061806
attr3->set_enumeration_name("big_enmr");
18071807

1808-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1808+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
18091809
ase->add_enumeration(enmr);
18101810
ase->add_attribute(attr3);
18111811

@@ -1827,7 +1827,7 @@ TEST_CASE_METHOD(
18271827
auto attr3 = make_shared<Attribute>(HERE(), "attr3", Datatype::UINT8);
18281828
attr3->set_enumeration_name("big_enmr");
18291829

1830-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1830+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
18311831
ase->add_enumeration(enmr);
18321832
ase->add_attribute(attr3);
18331833

@@ -1839,7 +1839,7 @@ TEST_CASE_METHOD(
18391839
EnumerationFx,
18401840
"ArraySchemaEvolution - Extend Enumeration nullptr",
18411841
"[enumeration][array-schema-evolution][extend][error]") {
1842-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1842+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
18431843
auto matcher = Catch::Matchers::ContainsSubstring(
18441844
"Cannot extend enumeration; Input enumeration is null");
18451845
REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher);
@@ -1849,7 +1849,7 @@ TEST_CASE_METHOD(
18491849
EnumerationFx,
18501850
"ArraySchemaEvolution - Extend Enumeration Already Extended",
18511851
"[enumeration][array-schema-evolution][extend][error]") {
1852-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1852+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
18531853
std::vector<int> values = {1, 2, 3, 4, 5};
18541854
auto enmr = create_enumeration(values);
18551855
auto matcher = Catch::Matchers::ContainsSubstring(
@@ -1928,7 +1928,7 @@ TEST_CASE_METHOD(
19281928
auto old_enmr = schema->get_enumeration("test_enmr");
19291929
auto new_enmr = extend_enumeration(old_enmr, values_to_add);
19301930

1931-
auto ase = make_shared<ArraySchemaEvolution>(HERE());
1931+
auto ase = make_shared<ArraySchemaEvolution>(HERE(), memory_tracker_);
19321932
ase->extend_enumeration(new_enmr);
19331933
auto st = ctx_.storage_manager()->array_evolve_schema(
19341934
array->array_uri(), ase.get(), array->get_encryption_key());
@@ -2182,7 +2182,7 @@ TEST_CASE_METHOD(
21822182
auto attr = make_shared<Attribute>(HERE(), "ohai", Datatype::INT64);
21832183
attr->set_enumeration_name("enmr2");
21842184

2185-
ArraySchemaEvolution ase1;
2185+
ArraySchemaEvolution ase1(memory_tracker_);
21862186
ase1.add_attribute(attr);
21872187
ase1.add_enumeration(enmr1);
21882188
ase1.add_enumeration(enmr2);
@@ -2216,7 +2216,7 @@ TEST_CASE_METHOD(
22162216
std::vector<double> values2 = {1.0, 2.0, 3.0, 4.0, 5.0};
22172217
auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2");
22182218

2219-
ArraySchemaEvolution ase1;
2219+
ArraySchemaEvolution ase1(memory_tracker_);
22202220
ase1.extend_enumeration(enmr1);
22212221
ase1.extend_enumeration(enmr2);
22222222

@@ -2416,6 +2416,7 @@ EnumerationFx::EnumerationFx()
24162416
, memory_tracker_(tiledb::test::create_test_memory_tracker()) {
24172417
rm_array();
24182418
throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0));
2419+
memory_tracker_ = tiledb::test::create_test_memory_tracker();
24192420
}
24202421

24212422
EnumerationFx::~EnumerationFx() {
@@ -2762,8 +2763,8 @@ shared_ptr<ArraySchemaEvolution> EnumerationFx::ser_des_array_schema_evolution(
27622763
ase, stype, &buf, client_side));
27632764

27642765
ArraySchemaEvolution* ret;
2765-
throw_if_not_ok(
2766-
serialization::array_schema_evolution_deserialize(&ret, stype, buf));
2766+
throw_if_not_ok(serialization::array_schema_evolution_deserialize(
2767+
&ret, stype, buf, memory_tracker_));
27672768

27682769
return shared_ptr<ArraySchemaEvolution>(ret);
27692770
}

‎tiledb/common/memory_tracker.cc‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@ std::string memory_type_to_str(MemoryType type) {
6868
return "TileMinVals";
6969
case MemoryType::TILE_NULL_COUNTS:
7070
return "TileNullCounts";
71+
case MemoryType::ATTRIBUTES:
72+
return "Attributes";
73+
case MemoryType::DIMENSION_LABELS:
74+
return "DimensionLabels";
75+
case MemoryType::DIMENSIONS:
76+
return "Dimensions";
7177
case MemoryType::TILE_SUMS:
7278
return "TileSums";
7379
case MemoryType::TILE_WRITER_DATA:
@@ -82,16 +88,28 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) {
8288
switch (type) {
8389
case MemoryTrackerType::ANONYMOUS:
8490
return "Anonymous";
91+
case MemoryTrackerType::ARRAY_CREATE:
92+
return "ArrayCreate";
93+
case MemoryTrackerType::ARRAY_LOAD:
94+
return "ArrayLoad";
8595
case MemoryTrackerType::ARRAY_READ:
8696
return "ArrayRead";
8797
case MemoryTrackerType::ARRAY_WRITE:
8898
return "ArrayWrite";
99+
case MemoryTrackerType::FRAGMENT_INFO_LOAD:
100+
return "FragmentInfoLoad";
89101
case MemoryTrackerType::QUERY_READ:
90102
return "QueryRead";
91103
case MemoryTrackerType::QUERY_WRITE:
92104
return "QueryWrite";
93105
case MemoryTrackerType::CONSOLIDATOR:
94106
return "Consolidator";
107+
case MemoryTrackerType::REST_CLIENT:
108+
return "RestClient";
109+
case MemoryTrackerType::EPHEMERAL:
110+
return "Ephemeral";
111+
case MemoryTrackerType::SCHEMA_EVOLUTION:
112+
return "SchemaEvolution";
95113
default:
96114
auto val = std::to_string(static_cast<uint32_t>(type));
97115
throw std::logic_error("Invalid memory tracker type: " + val);

‎tiledb/common/memory_tracker.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,8 @@ enum class MemoryTrackerType {
131131
QUERY_WRITE,
132132
CONSOLIDATOR,
133133
REST_CLIENT,
134-
EPHEMERAL
134+
EPHEMERAL,
135+
SCHEMA_EVOLUTION
135136
};
136137

137138
class MemoryTrackerResource : public tdb::pmr::memory_resource {

‎tiledb/sm/array_schema/array_schema_evolution.cc‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include "tiledb/common/common.h"
3636
#include "tiledb/common/heap_memory.h"
3737
#include "tiledb/common/logger.h"
38+
#include "tiledb/common/memory_tracker.h"
3839
#include "tiledb/common/status.h"
3940
#include "tiledb/sm/array_schema/array_schema.h"
4041
#include "tiledb/sm/array_schema/attribute.h"
@@ -71,7 +72,15 @@ class ArraySchemaEvolutionException : public StatusException {
7172
/* CONSTRUCTORS & DESTRUCTORS */
7273
/* ****************************** */
7374

74-
ArraySchemaEvolution::ArraySchemaEvolution() {
75+
ArraySchemaEvolution::ArraySchemaEvolution(
76+
shared_ptr<MemoryTracker> memory_tracker)
77+
: memory_tracker_(memory_tracker)
78+
, attributes_to_add_map_(
79+
memory_tracker->get_resource(MemoryType::ATTRIBUTES))
80+
, enumerations_to_add_map_(
81+
memory_tracker_->get_resource(MemoryType::ENUMERATION))
82+
, enumerations_to_extend_map_(
83+
memory_tracker_->get_resource(MemoryType::ENUMERATION)) {
7584
}
7685

7786
ArraySchemaEvolution::ArraySchemaEvolution(
@@ -81,13 +90,29 @@ ArraySchemaEvolution::ArraySchemaEvolution(
8190
std::unordered_map<std::string, shared_ptr<const Enumeration>>
8291
enmrs_to_extend,
8392
std::unordered_set<std::string> enmrs_to_drop,
84-
std::pair<uint64_t, uint64_t> timestamp_range)
85-
: attributes_to_add_map_(attrs_to_add)
93+
std::pair<uint64_t, uint64_t> timestamp_range,
94+
shared_ptr<MemoryTracker> memory_tracker)
95+
: memory_tracker_(memory_tracker)
96+
, attributes_to_add_map_(
97+
memory_tracker->get_resource(MemoryType::ATTRIBUTES))
8698
, attributes_to_drop_(attrs_to_drop)
87-
, enumerations_to_add_map_(enmrs_to_add)
88-
, enumerations_to_extend_map_(enmrs_to_extend)
99+
, enumerations_to_add_map_(
100+
memory_tracker_->get_resource(MemoryType::ENUMERATION))
101+
, enumerations_to_extend_map_(
102+
memory_tracker_->get_resource(MemoryType::ENUMERATION))
89103
, enumerations_to_drop_(enmrs_to_drop)
90104
, timestamp_range_(timestamp_range) {
105+
for (auto& elem : attrs_to_add) {
106+
attributes_to_add_map_.insert(elem);
107+
}
108+
109+
for (auto& elem : enmrs_to_add) {
110+
enumerations_to_add_map_.insert(elem);
111+
}
112+
113+
for (auto& elem : enmrs_to_extend) {
114+
enumerations_to_extend_map_.insert(elem);
115+
}
91116
}
92117

93118
ArraySchemaEvolution::~ArraySchemaEvolution() {

0 commit comments

Comments
 (0)