Skip to content

Commit 5faad77

Browse files
euvreeuvre
committed
Remove all code relevant to the hnsw memory index
Signed-off-by: noob <[email protected]>
1 parent 4a9b204 commit 5faad77

27 files changed

+1384
-1407
lines changed

‎python/restart_test/test_memidx.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -556,9 +556,11 @@ def part1(infinity_obj):
556556
time.sleep(1)
557557

558558
idx1_dirs = list(pathlib.Path(db1_dir).rglob("*chunk*"))
559-
assert len(idx1_dirs) == 2
559+
# assert len(idx1_dirs) == 2
560+
assert len(idx1_dirs) == 1
560561
idx2_dirs = list(pathlib.Path(db2_dir).rglob("*chunk*"))
561-
assert len(idx2_dirs) == 2
562+
# assert len(idx2_dirs) == 2
563+
assert len(idx2_dirs) == 1
562564

563565
part1()
564566

@@ -581,9 +583,11 @@ def part2(infinity_obj):
581583

582584
idx1_dirs = list(pathlib.Path(db1_dir).rglob("*chunk*"))
583585
print(idx1_dirs)
584-
assert len(idx1_dirs) == 3
586+
# assert len(idx1_dirs) == 3
587+
assert len(idx1_dirs) == 1
585588
idx2_dirs = list(pathlib.Path(db2_dir).rglob("*chunk*"))
586-
assert len(idx2_dirs) == 3
589+
# assert len(idx2_dirs) == 3
590+
assert len(idx2_dirs) == 1
587591

588592
# wait for optimize commit
589593
# The optimization transaction creates the file before committing.

‎src/executor/operator/physical_scan/physical_knn_scan_impl.cpp‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -958,17 +958,11 @@ void ExecuteHnswSearch(QueryContext *query_context,
958958
if (!status.ok()) {
959959
UnrecoverableError(status.message());
960960
}
961+
// HnswHandler *hnsw_handler{};
961962
std::shared_ptr<HnswHandler> hnsw_handler;
962963
index_file_worker->Read(hnsw_handler);
963964
hnsw_search(hnsw_handler.get(), false);
964965
}
965-
if (mem_index) {
966-
auto memory_hnsw_index = mem_index->GetHnswIndex();
967-
if (memory_hnsw_index) {
968-
const HnswHandlerPtr hnsw_handler = memory_hnsw_index->get();
969-
hnsw_search(hnsw_handler, true);
970-
}
971-
}
972966
}
973967

974968
} // namespace infinity

‎src/storage/buffer/file_worker/hnsw_file_worker_impl.cpp‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,16 +47,16 @@ HnswFileWorker::HnswFileWorker(std::shared_ptr<std::string> file_path,
4747
std::shared_ptr<ColumnDef> column_def,
4848
size_t index_size)
4949
: IndexFileWorker(std::move(file_path), std::move(index_base), std::move(column_def)) {
50-
if (index_size == 0) {
51-
52-
std::string index_path = GetFilePath();
53-
auto [file_handle, status] = VirtualStore::Open(index_path, FileAccessMode::kReadWrite);
54-
if (status.ok()) {
55-
// When replay by checkpoint, the data is deleted, but catalog is recovered. Do not read file in recovery.
56-
index_size = file_handle->FileSize();
57-
}
58-
}
59-
index_size_ = index_size;
50+
// if (index_size == 0) {
51+
//
52+
// std::string index_path = GetFilePath();
53+
// auto [file_handle, status] = VirtualStore::Open(index_path, FileAccessMode::kReadWrite);
54+
// if (status.ok()) {
55+
// // When replay by checkpoint, the data is deleted, but catalog is recovered. Do not read file in recovery.
56+
// index_size = file_handle->FileSize();
57+
// }
58+
// }
59+
// index_size_ = index_size;
6060
}
6161

6262
HnswFileWorker::~HnswFileWorker() {
@@ -89,6 +89,7 @@ bool HnswFileWorker::Write(std::shared_ptr<HnswHandler> &data,
8989
void HnswFileWorker::Read(std::shared_ptr<HnswHandler> &data, std::unique_ptr<LocalFileHandle> &file_handle, size_t file_size) {
9090
std::unique_lock l(mutex_);
9191
if (!file_handle) {
92+
data = HnswHandler::Make(index_base_.get(), column_def_);
9293
return;
9394
}
9495
auto &path = *rel_file_path_;

‎src/storage/catalog/mem_index.cppm‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import row_id;
2525
namespace infinity {
2626

2727
class BaseMemIndex;
28-
class HnswIndexInMem;
28+
// class HnswIndexInMem;
2929
class IVFIndexInMem;
3030
class MemoryIndexer;
3131
class SecondaryIndexInMem;
@@ -52,8 +52,8 @@ public:
5252

5353
const BaseMemIndex *GetBaseMemIndex() const;
5454

55-
std::shared_ptr<HnswIndexInMem> GetHnswIndex();
56-
void SetHnswIndex(std::shared_ptr<HnswIndexInMem> hnsw_index);
55+
// std::shared_ptr<HnswIndexInMem> GetHnswIndex();
56+
// void SetHnswIndex(std::shared_ptr<HnswIndexInMem> hnsw_index);
5757

5858
std::shared_ptr<IVFIndexInMem> GetIVFIndex();
5959
void SetIVFIndex(std::shared_ptr<IVFIndexInMem> ivf_index);
@@ -85,7 +85,7 @@ private:
8585
bool is_dumping_{};
8686
bool is_updating_{};
8787

88-
std::shared_ptr<HnswIndexInMem> memory_hnsw_index_;
88+
// std::shared_ptr<HnswIndexInMem> memory_hnsw_index_;
8989
std::shared_ptr<IVFIndexInMem> memory_ivf_index_;
9090
std::shared_ptr<MemoryIndexer> memory_indexer_;
9191
std::shared_ptr<SecondaryIndexInMem> memory_secondary_index_;

‎src/storage/catalog/mem_index_impl.cpp‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -64,15 +64,15 @@ size_t MemIndex::GetRowCount() {
6464
}
6565

6666
bool MemIndex::IsNull() const {
67-
std::unique_lock<std::mutex> lock(mtx_);
68-
return memory_hnsw_index_ == nullptr && memory_ivf_index_ == nullptr && memory_indexer_ == nullptr && memory_secondary_index_ == nullptr &&
67+
std::unique_lock lock(mtx_);
68+
return /* memory_hnsw_index_ == nullptr && */ memory_ivf_index_ == nullptr && memory_indexer_ == nullptr && memory_secondary_index_ == nullptr &&
6969
memory_emvb_index_ == nullptr && memory_bmp_index_ == nullptr && memory_dummy_index_ == nullptr;
7070
}
7171

7272
void MemIndex::ClearMemIndex() {
73-
std::unique_lock<std::mutex> lock(mtx_);
73+
std::unique_lock lock(mtx_);
7474

75-
memory_hnsw_index_.reset();
75+
// memory_hnsw_index_.reset();
7676
memory_ivf_index_.reset();
7777
memory_indexer_.reset();
7878
memory_secondary_index_.reset();
@@ -83,11 +83,12 @@ void MemIndex::ClearMemIndex() {
8383
}
8484

8585
const BaseMemIndex *MemIndex::GetBaseMemIndex() const {
86-
std::unique_lock<std::mutex> lock(mtx_);
87-
BaseMemIndex *res = nullptr;
88-
if (memory_hnsw_index_.get() != nullptr) {
89-
res = static_cast<BaseMemIndex *>(memory_hnsw_index_.get());
90-
} else if (memory_ivf_index_.get() != nullptr) {
86+
std::unique_lock lock(mtx_);
87+
BaseMemIndex *res{};
88+
// if (memory_hnsw_index_.get() != nullptr) {
89+
// res = static_cast<BaseMemIndex *>(memory_hnsw_index_.get());
90+
// } else
91+
if (memory_ivf_index_.get() != nullptr) {
9192
res = static_cast<BaseMemIndex *>(memory_ivf_index_.get());
9293
} else if (memory_indexer_.get() != nullptr) {
9394
res = static_cast<BaseMemIndex *>(memory_indexer_.get());
@@ -104,15 +105,15 @@ const BaseMemIndex *MemIndex::GetBaseMemIndex() const {
104105
return res;
105106
}
106107

107-
std::shared_ptr<HnswIndexInMem> MemIndex::GetHnswIndex() {
108-
std::unique_lock<std::mutex> lock(mtx_);
109-
return memory_hnsw_index_;
110-
}
108+
// std::shared_ptr<HnswIndexInMem> MemIndex::GetHnswIndex() {
109+
// std::unique_lock<std::mutex> lock(mtx_);
110+
// return memory_hnsw_index_;
111+
// }
111112

112-
void MemIndex::SetHnswIndex(std::shared_ptr<HnswIndexInMem> hnsw_index) {
113-
std::unique_lock<std::mutex> lock(mtx_);
114-
memory_hnsw_index_ = hnsw_index;
115-
}
113+
// void MemIndex::SetHnswIndex(std::shared_ptr<HnswIndexInMem> hnsw_index) {
114+
// std::unique_lock<std::mutex> lock(mtx_);
115+
// memory_hnsw_index_ = hnsw_index;
116+
// }
116117

117118
std::shared_ptr<IVFIndexInMem> MemIndex::GetIVFIndex() {
118119
std::unique_lock<std::mutex> lock(mtx_);

‎src/storage/catalog/meta/block_version.cppm‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@ struct CreateField {
3434
static CreateField LoadFromFile(LocalFileHandle *file_handle);
3535
};
3636

37-
std::atomic_int cnt{};
38-
3937
export struct BlockVersion {
4038
using segment_manager = boost::interprocess::managed_mapped_file::segment_manager;
4139
using TxnTimeStamp_allocator = boost::interprocess::allocator<TxnTimeStamp, segment_manager>;

‎src/storage/catalog/meta/segment_index_meta.cppm‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public:
4747

4848
std::tuple<std::vector<ChunkID> *, Status> GetChunkIDs1();
4949

50-
Status SetChunkIDs(const std::vector<ChunkID> &chunk_ids);
50+
// Status SetChunkIDs(const std::vector<ChunkID> &chunk_ids);
5151

5252
Status RemoveChunkIDs(const std::vector<ChunkID> &chunk_ids);
5353

@@ -75,7 +75,7 @@ private:
7575

7676
Status LoadNextChunkID();
7777

78-
Status LoadFtInfo();
78+
// Status LoadFtInfo();
7979

8080
std::string GetSegmentIndexTag(const std::string &tag);
8181

‎src/storage/catalog/meta/segment_index_meta_impl.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ SegmentIndexMeta::SegmentIndexMeta(SegmentID segment_id, TableIndexMeta &table_i
4848
SegmentIndexMeta::~SegmentIndexMeta() = default;
4949

5050
Status SegmentIndexMeta::GetNextChunkID(ChunkID &chunk_id) {
51-
std::lock_guard<std::mutex> lock(mtx_);
51+
std::lock_guard lock(mtx_);
5252
if (!next_chunk_id_) {
5353
Status status = LoadNextChunkID();
5454
if (!status.ok()) {
@@ -126,7 +126,7 @@ Status SegmentIndexMeta::RemoveChunkIDs(const std::vector<ChunkID> &chunk_ids) {
126126
Status SegmentIndexMeta::AddChunkIndexID1(ChunkID chunk_id, NewTxn *new_txn) {
127127

128128
TableMeta &table_meta = table_index_meta_.table_meta();
129-
std::string chunk_id_key =
129+
auto chunk_id_key =
130130
KeyEncode::CatalogIdxChunkKey(table_meta.db_id_str(), table_meta.table_id_str(), table_index_meta_.index_id_str(), segment_id_, chunk_id);
131131
std::string commit_ts_str;
132132
switch (new_txn->GetTxnState()) {

‎src/storage/catalog/new_catalog.cppm‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,8 @@ public:
297297
size_t index_size,
298298
std::optional<ChunkIndexMeta> &chunk_index_meta);
299299

300+
static Status InitHnswChunkIndex(SegmentIndexMeta &segment_index_meta, NewTxn *new_txn, std::optional<ChunkIndexMeta> &chunk_index_meta);
301+
300302
static Status RestoreNewChunkIndex1(SegmentIndexMeta &segment_index_meta,
301303
NewTxn *new_txn,
302304
ChunkID chunk_id,

‎src/storage/catalog/new_catalog_impl.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,10 +124,10 @@ std::vector<std::pair<std::string, std::string>> NewCatalog::GetAllMemIndexInfo(
124124
{
125125
std::unique_lock lock(mem_index_mtx_);
126126
for (const auto &mem_index_pair : mem_index_map_) {
127-
if (mem_index_pair.second->GetHnswIndex() != nullptr) {
128-
result.push_back({mem_index_pair.first, "hnsw"});
129-
continue;
130-
}
127+
// if (mem_index_pair.second->GetHnswIndex() != nullptr) {
128+
// result.push_back({mem_index_pair.first, "hnsw"});
129+
// continue;
130+
// }
131131
if (mem_index_pair.second->GetIVFIndex() != nullptr) {
132132
result.push_back({mem_index_pair.first, "ivf"});
133133
continue;

0 commit comments

Comments
 (0)