Skip to content

Commit 549add0

Browse files
ethanglaserethanglaser
committed
update py example license and add cpp example
1 parent 57b3982 commit 549add0

File tree

4 files changed

+271
-20
lines changed

4 files changed

+271
-20
lines changed

‎examples/cpp/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ create_simple_example(saveload test_saveload saveload.cpp)
3838
create_simple_example(types test_types types.cpp)
3939
create_simple_example(vamana_iterator test_vamana_iterator vamana_iterator.cpp)
4040
create_simple_example(custom_thread_pool test_custom_thread_pool custom_thread_pool.cpp)
41+
create_simple_example(fallback test_fallback fallback.cpp)
4142

4243
## More complicated examples involving more extensive setup.
4344

‎examples/cpp/fallback.cpp‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
/*
2+
* Copyright 2023 Intel Corporation
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
//! [Example All]
18+
19+
//! [Includes]
20+
// SVS Dependencies
21+
#include "svs/orchestrators/vamana.h" // bulk of the dependencies required.
22+
#include "svs/core/recall.h" // Convenient k-recall@n computation.
23+
#include "svs/fallback/fallback.h"
24+
25+
// Alternative main definition
26+
#include "svsmain.h"
27+
28+
// stl
29+
#include <map>
30+
#include <string>
31+
#include <string_view>
32+
#include <vector>
33+
//! [Includes]
34+
35+
//! [Helper Utilities]
36+
double run_recall(
37+
svs::Vamana& index,
38+
const svs::data::SimpleData<float>& queries,
39+
const svs::data::SimpleData<uint32_t>& groundtruth,
40+
size_t search_window_size,
41+
size_t num_neighbors,
42+
std::string_view message = ""
43+
) {
44+
index.set_search_window_size(search_window_size);
45+
auto results = index.search(queries, num_neighbors);
46+
double recall = svs::k_recall_at_n(groundtruth, results, num_neighbors, num_neighbors);
47+
if (!message.empty()) {
48+
fmt::print("[{}] ", message);
49+
}
50+
fmt::print("Windowsize = {}, Recall = {}\n", search_window_size, recall);
51+
return recall;
52+
}
53+
54+
const bool DEBUG = true;
55+
void check(double expected, double got, double eps = 0.001) {
56+
double diff = std::abs(expected - got);
57+
if constexpr (DEBUG) {
58+
fmt::print("Expected {}. Got {}\n", expected, got);
59+
} else {
60+
if (diff > eps) {
61+
throw ANNEXCEPTION("Expected ", expected, ". Got ", got, '!');
62+
}
63+
}
64+
}
65+
//! [Helper Utilities]
66+
67+
// Alternative main definition
68+
int svs_main(std::vector<std::string> args) {
69+
//! [Argument Extraction]
70+
const size_t nargs = args.size();
71+
if (nargs != 4) {
72+
throw ANNEXCEPTION("Expected 3 arguments. Instead, got ", nargs, '!');
73+
}
74+
const std::string& data_vecs = args.at(1);
75+
const std::string& query_vecs = args.at(2);
76+
const std::string& groundtruth_vecs = args.at(3);
77+
//! [Argument Extraction]
78+
79+
// Building the index
80+
81+
//! [Build Parameters]
82+
auto parameters = svs::index::vamana::VamanaBuildParameters{
83+
1.2, // alpha
84+
64, // graph max degree
85+
128, // search window size
86+
1024, // max candidate pool size
87+
60, // prune to degree
88+
true, // full search history
89+
};
90+
//! [Build Parameters]
91+
92+
//! [Index Build]
93+
size_t num_threads = 4;
94+
svs::Vamana index = svs::Vamana::build<float>(
95+
parameters, svs::VectorDataLoader<float>(data_vecs), svs::DistanceL2(), num_threads
96+
);
97+
//! [Index Build]
98+
99+
// Searching the index
100+
101+
//! [Load Aux]
102+
// Load the queries and ground truth.
103+
auto queries = svs::load_data<float>(query_vecs);
104+
auto groundtruth = svs::load_data<uint32_t>(groundtruth_vecs);
105+
//! [Load Aux]
106+
107+
//! [Perform Queries]
108+
index.set_search_window_size(30);
109+
svs::QueryResult<size_t> results = index.search(queries, 10);
110+
double recall = svs::k_recall_at_n(groundtruth, results);
111+
check(0.8215, recall);
112+
//! [Perform Queries]
113+
114+
//! [Search Window Size]
115+
auto expected_recall =
116+
std::map<size_t, double>({{10, 0.5509}, {20, 0.7281}, {30, 0.8215}, {40, 0.8788}});
117+
for (auto windowsize : {10, 20, 30, 40}) {
118+
recall = run_recall(index, queries, groundtruth, windowsize, 10, "Sweep");
119+
check(expected_recall.at(windowsize), recall);
120+
}
121+
//! [Search Window Size]
122+
123+
// Saving the index
124+
125+
//! [Saving]
126+
index.save("example_config", "example_graph", "example_data");
127+
//! [Saving]
128+
129+
// Reloading a saved index
130+
131+
//! [Loading]
132+
// We can reload an index from a previously saved set of files.
133+
index = svs::Vamana::assemble<float>(
134+
"example_config",
135+
svs::GraphLoader("example_graph"),
136+
svs::VectorDataLoader<float>("example_data"),
137+
svs::DistanceType::L2,
138+
4 // num_threads
139+
);
140+
141+
recall = run_recall(index, queries, groundtruth, 30, 10, "Reload");
142+
check(0.8215, recall);
143+
//! [Loading]
144+
145+
// Search using vector compression
146+
147+
//! [Compressed Loader]
148+
// Quantization
149+
size_t padding = 32;
150+
namespace lvq = svs::quantization::lvq;
151+
namespace leanvec = svs::leanvec;
152+
namespace fallback = svs::fallback;
153+
154+
// Wrap the compressor object in a lazy functor.
155+
// This will defer loading and compression of the LVQ dataset until the threadpool
156+
// used in the index has been created.
157+
auto compressor = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) {
158+
auto data = svs::VectorDataLoader<float, 128>("example_data").load();
159+
return lvq::LVQDataset<8, 0, 128>::compress(data, threadpool, padding);
160+
});
161+
index = svs::Vamana::assemble<float>(
162+
"example_config",
163+
svs::GraphLoader("example_graph"),
164+
compressor,
165+
svs::DistanceL2(),
166+
4
167+
);
168+
169+
//! [Compressed Loader]
170+
171+
//! [Search Compressed]
172+
recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Load");
173+
check(0.8215, recall);
174+
//! [Search Compressed]
175+
176+
//! [Build Index Compressed]
177+
// Compressed building
178+
index =
179+
svs::Vamana::build<float>(parameters, compressor, svs::DistanceL2(), num_threads);
180+
recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Build");
181+
check(0.8212, recall);
182+
//! [Build Index Compressed]
183+
184+
// ! [Only Loading]
185+
// We can reload an index from a previously saved set of files.
186+
index = svs::Vamana::assemble<float>(
187+
"example_config",
188+
svs::GraphLoader("example_graph"),
189+
svs::VectorDataLoader<float>("example_data"),
190+
svs::DistanceType::L2,
191+
4 // num_threads
192+
);
193+
//! [Only Loading]
194+
195+
//! [Set n-threads]
196+
index.set_threadpool(svs::threads::DefaultThreadPool(4));
197+
//! [Set n-threads]
198+
199+
auto compressor_lean = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) {
200+
auto data = svs::VectorDataLoader<float, 128>("example_data").load();
201+
return leanvec::LeanDataset<leanvec::UsingLVQ<4>, leanvec::UsingLVQ<8>, 64, 128>::reduce(
202+
data, std::nullopt, threadpool, padding
203+
);
204+
});
205+
index = svs::Vamana::assemble<float>(
206+
"example_config",
207+
svs::GraphLoader("example_graph"),
208+
compressor_lean,
209+
svs::DistanceL2(),
210+
4
211+
);
212+
213+
//! [Compressed Loader]
214+
215+
//! [Search Compressed]
216+
recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Lean Load");
217+
check(0.8215, recall);
218+
//! [Search Compressed]
219+
220+
//! [Build Index Compressed]
221+
// Compressed building
222+
index =
223+
svs::Vamana::build<float>(parameters, compressor, svs::DistanceL2(), num_threads);
224+
recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Build");
225+
check(0.8212, recall);
226+
//! [Build Index Compressed]
227+
228+
//! [Only Loading]
229+
// We can reload an index from a previously saved set of files.
230+
index = svs::Vamana::assemble<float>(
231+
"example_config",
232+
svs::GraphLoader("example_graph"),
233+
svs::VectorDataLoader<float>("example_data"),
234+
svs::DistanceType::L2,
235+
4 // num_threads
236+
);
237+
//! [Only Loading]
238+
239+
//! [Set n-threads]
240+
index.set_threadpool(svs::threads::DefaultThreadPool(4));
241+
//! [Set n-threads]
242+
243+
return 0;
244+
}
245+
246+
// Special main providing some helpful utilties.
247+
SVS_DEFINE_MAIN();
248+
//! [Example All]

‎examples/python/example_fallback.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
# Copyright (C) 2023 Intel Corporation
1+
# Copyright 2023 Intel Corporation
22
#
3-
# This software and the related documents are Intel copyrighted materials,
4-
# and your use of them is governed by the express license under which they
5-
# were provided to you ("License"). Unless the License provides otherwise,
6-
# you may not use, modify, copy, publish, distribute, disclose or transmit
7-
# this software or the related documents without Intel's prior written
8-
# permission.
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
96
#
10-
# This software and the related documents are provided as is, with no
11-
# express or implied warranties, other than those that are expressly stated
12-
# in the License.
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
1314

1415
# Import `unittest` to allow for automated testing.
1516
import unittest

‎examples/python/example_fallback_leanvec.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
# Copyright (C) 2023 Intel Corporation
1+
# Copyright 2023 Intel Corporation
22
#
3-
# This software and the related documents are Intel copyrighted materials,
4-
# and your use of them is governed by the express license under which they
5-
# were provided to you ("License"). Unless the License provides otherwise,
6-
# you may not use, modify, copy, publish, distribute, disclose or transmit
7-
# this software or the related documents without Intel's prior written
8-
# permission.
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
96
#
10-
# This software and the related documents are provided as is, with no
11-
# express or implied warranties, other than those that are expressly stated
12-
# in the License.
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
1314

1415
# Import `unittest` to allow for auotmated testing.
1516
import unittest

0 commit comments

Comments
 (0)