From 5e7455bb9b1fe947b64512be7775991c7ac7058d Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 26 Mar 2025 18:07:45 -0700 Subject: [PATCH 01/23] API fallback and python bindings refactoring --- bindings/python/include/svs/python/core.h | 18 + bindings/python/include/svs/python/dispatch.h | 82 +++ .../include/svs/python/dynamic_vamana.h | 42 ++ bindings/python/include/svs/python/vamana.h | 103 +++ bindings/python/src/core.cpp | 514 ++++++++++++++ bindings/python/src/dynamic_vamana.cpp | 69 +- bindings/python/src/vamana.cpp | 121 +++- include/svs/fallback/fallback.h | 33 + include/svs/fallback/fallback_mode.h | 40 ++ include/svs/fallback/leanvec_fallback.h | 634 ++++++++++++++++++ include/svs/fallback/lvq_fallback.h | 624 +++++++++++++++++ include/svs/index/vamana/extensions.h | 15 + 12 files changed, 2286 insertions(+), 9 deletions(-) create mode 100644 include/svs/fallback/fallback.h create mode 100644 include/svs/fallback/fallback_mode.h create mode 100644 include/svs/fallback/leanvec_fallback.h create mode 100644 include/svs/fallback/lvq_fallback.h diff --git a/bindings/python/include/svs/python/core.h b/bindings/python/include/svs/python/core.h index ed378375..446983a9 100644 --- a/bindings/python/include/svs/python/core.h +++ b/bindings/python/include/svs/python/core.h @@ -27,6 +27,8 @@ #include "svs/lib/meta.h" #include "svs/lib/misc.h" +#include "svs/fallback/fallback.h" + // pybind #include @@ -118,6 +120,22 @@ class UnspecializedGraphLoader { using DistanceL2 = svs::distance::DistanceL2; using DistanceIP = svs::distance::DistanceIP; +///// +///// LVQ +///// + +// Compressors - online compression of existing data +using LVQReloader = svs::quantization::lvq::Reload; +using LVQ = svs::quantization::lvq::ProtoLVQLoader; + +///// +///// LeanVec +///// + +// Dimensionality reduction using LeanVec +using LeanVecReloader = svs::leanvec::Reload; +using LeanVec = svs::leanvec::ProtoLeanVecLoader; + namespace core { void wrap(pybind11::module& m); } // namespace core diff --git a/bindings/python/include/svs/python/dispatch.h b/bindings/python/include/svs/python/dispatch.h index 221d4183..e90b5a8d 100644 --- a/bindings/python/include/svs/python/dispatch.h +++ b/bindings/python/include/svs/python/dispatch.h @@ -24,6 +24,8 @@ #include "svs/lib/dispatcher.h" #include "svs/lib/saveload.h" +#include "svs/fallback/fallback.h" + // Dispatch rule for serialized objects to a VectorDataLoader. template struct svs::lib::DispatchConverter< @@ -49,3 +51,83 @@ struct svs::lib::DispatchConverter< return To{object.context().get_directory()}; } }; + +template < + size_t Primary, + size_t Residual, + size_t Extent, + svs::quantization::lvq::LVQPackingStrategy Strategy> +struct svs::lib::DispatchConverter< + svs::lib::SerializedObject, + svs::quantization::lvq::LVQLoader< + Primary, + Residual, + Extent, + Strategy, + svs::python::RebindAllocator>> { + using To = svs::quantization::lvq::LVQLoader< + Primary, + Residual, + Extent, + Strategy, + svs::python::RebindAllocator>; + + using LVQStrategyDispatch = svs::quantization::lvq::LVQStrategyDispatch; + + static int64_t match(const svs::lib::SerializedObject& object) { + // TODO: Use a LoadTable directly instead of forcing reparsing every time. + auto ex = svs::lib::try_load(object); + if (!ex) { + return svs::lib::invalid_match; + } + + return svs::quantization::lvq::overload_score( + ex.value(), LVQStrategyDispatch::Auto + ); + } + + static To convert(const svs::lib::SerializedObject& object) { + return To{ + svs::quantization::lvq::Reload{std::move(object.context().get_directory())}, + 0, + svs::python::RebindAllocator()}; + } +}; + +template +struct svs::lib::DispatchConverter< + svs::lib::SerializedObject, + svs::leanvec::LeanVecLoader< + PrimaryKind, + SecondaryKind, + LeanVecDims, + Extent, + svs::python::RebindAllocator>> { + using To = leanvec::LeanVecLoader< + PrimaryKind, + SecondaryKind, + LeanVecDims, + Extent, + svs::python::RebindAllocator>; + + static int64_t match(const svs::lib::SerializedObject& object) { + // TODO: Use a LoadTable directly instead of forcing reparsing every time. + auto ex = svs::lib::try_load(object); + if (!ex) { + return svs::lib::invalid_match; + } + + return svs::leanvec:: + overload_score(ex.value()); + } + + static To convert(const svs::lib::SerializedObject& object) { + return To{ + leanvec::Reload{object.context().get_directory()}, + LeanVecDims, // TODO: This is a hack for now. Since we're reloading, it doesn't + // matter. + std::nullopt, + 0, + svs::python::RebindAllocator()}; + } +}; diff --git a/bindings/python/include/svs/python/dynamic_vamana.h b/bindings/python/include/svs/python/dynamic_vamana.h index 26c42b68..53d95550 100644 --- a/bindings/python/include/svs/python/dynamic_vamana.h +++ b/bindings/python/include/svs/python/dynamic_vamana.h @@ -33,5 +33,47 @@ template void for_standard_specializations(F&& f) { #undef X } +template void for_compressed_specializations(F&& f) { + using Sequential = svs::quantization::lvq::Sequential; +#define X(Dist, Primary, Residual, Strategy, N) \ + f.template operator()() + // Sequential + X(DistanceL2, 4, 0, Sequential, Dynamic); + X(DistanceIP, 4, 0, Sequential, Dynamic); + X(DistanceL2, 4, 4, Sequential, Dynamic); + X(DistanceIP, 4, 4, Sequential, Dynamic); + X(DistanceL2, 4, 8, Sequential, Dynamic); + X(DistanceIP, 4, 8, Sequential, Dynamic); + X(DistanceL2, 8, 0, Sequential, Dynamic); + X(DistanceIP, 8, 0, Sequential, Dynamic); + + // Turbo + using Turbo16x8 = svs::quantization::lvq::Turbo<16, 8>; + X(DistanceL2, 4, 0, Turbo16x8, Dynamic); + X(DistanceIP, 4, 0, Turbo16x8, Dynamic); + X(DistanceL2, 4, 4, Turbo16x8, Dynamic); + X(DistanceIP, 4, 4, Turbo16x8, Dynamic); + X(DistanceL2, 4, 8, Turbo16x8, Dynamic); + X(DistanceIP, 4, 8, Turbo16x8, Dynamic); +#undef X +} + +template void for_leanvec_specializations(F&& f) { +#define X(Dist, Primary, Secondary, L, N) \ + f.template operator()() + X(DistanceL2, svs::Float16, svs::Float16, Dynamic, Dynamic); + X(DistanceIP, svs::Float16, svs::Float16, Dynamic, Dynamic); + + X(DistanceL2, svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic); + X(DistanceIP, svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic); + + X(DistanceL2, svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic); + X(DistanceIP, svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic); + + X(DistanceL2, svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic); + X(DistanceIP, svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic); +#undef X +} + void wrap(pybind11::module& m); } // namespace svs::python::dynamic_vamana diff --git a/bindings/python/include/svs/python/vamana.h b/bindings/python/include/svs/python/vamana.h index 3484b443..301dd411 100644 --- a/bindings/python/include/svs/python/vamana.h +++ b/bindings/python/include/svs/python/vamana.h @@ -89,6 +89,109 @@ template void for_standard_specializations(F&& f) { #undef XN #undef X } + +// Compressed search specializations. +// Pattern: +// DistanceType, Primary, Residual, Dimensionality, Strategy, EnableBuild +#define X(Dist, P, R, N, S, B) f.template operator()() +template void lvq_specialize_4x0(const F& f) { + using Sequential = svs::quantization::lvq::Sequential; + using Turbo = svs::quantization::lvq::Turbo<16, 8>; + + // Sequential + X(DistanceL2, 4, 0, Dynamic, Sequential, true); + X(DistanceIP, 4, 0, Dynamic, Sequential, true); + // Turbo + X(DistanceL2, 4, 0, Dynamic, Turbo, true); + X(DistanceIP, 4, 0, Dynamic, Turbo, true); +} + +template void lvq_specialize_4x4(const F& f) { + using Sequential = svs::quantization::lvq::Sequential; + using Turbo = svs::quantization::lvq::Turbo<16, 8>; + + // Sequential + X(DistanceL2, 4, 4, Dynamic, Sequential, true); + X(DistanceIP, 4, 4, Dynamic, Sequential, true); + // Turbo + X(DistanceL2, 4, 4, Dynamic, Turbo, true); + X(DistanceIP, 4, 4, Dynamic, Turbo, true); +} + +template void lvq_specialize_4x8(const F& f) { + using Sequential = svs::quantization::lvq::Sequential; + using Turbo = svs::quantization::lvq::Turbo<16, 8>; + + // Sequential + X(DistanceL2, 4, 8, Dynamic, Sequential, true); + X(DistanceIP, 4, 8, Dynamic, Sequential, true); + // Turbo + X(DistanceL2, 4, 8, Dynamic, Turbo, true); + X(DistanceIP, 4, 8, Dynamic, Turbo, true); +} + +template void lvq_specialize_8x0(const F& f) { + using Sequential = svs::quantization::lvq::Sequential; + using Turbo = svs::quantization::lvq::Turbo<16, 4>; + + // Sequential + X(DistanceL2, 8, 0, Dynamic, Sequential, true); + X(DistanceIP, 8, 0, Dynamic, Sequential, true); + // Turbo + X(DistanceL2, 8, 0, Dynamic, Turbo, true); + X(DistanceIP, 8, 0, Dynamic, Turbo, true); +} + +template void lvq_specialize_8x8(const F& f) { + using Sequential = svs::quantization::lvq::Sequential; + X(DistanceL2, 8, 8, Dynamic, Sequential, false); + X(DistanceIP, 8, 8, Dynamic, Sequential, false); +} + +template void compressed_specializations(F&& f) { + lvq_specialize_4x0(f); + lvq_specialize_4x4(f); + lvq_specialize_4x8(f); + lvq_specialize_8x0(f); + lvq_specialize_8x8(f); +} +#undef X + +// LeanVec specializations. +// Pattern: +// Primary, Secondary, LeanVec Dimensionality, Dimensionality, DistanceType +#define X(P, S, L, N, D) f.template operator()() +template void leanvec_specialize_unc_unc(const F& f) { + X(float, float, Dynamic, Dynamic, DistanceL2); + X(float, float, Dynamic, Dynamic, DistanceIP); + + X(svs::Float16, svs::Float16, Dynamic, Dynamic, DistanceL2); + X(svs::Float16, svs::Float16, Dynamic, Dynamic, DistanceIP); +} + +template void leanvec_specialize_lvq_unc(const F& f) { + X(svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic, DistanceL2); + X(svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic, DistanceIP); +} + +template void leanvec_specialize_lvq_lvq(const F& f) { + X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic, DistanceL2); + X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic, DistanceIP); + + X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceL2); + X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceIP); + + X(svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceL2); + X(svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceIP); +} + +template void leanvec_specializations(F&& f) { + leanvec_specialize_unc_unc(f); + leanvec_specialize_lvq_unc(f); + leanvec_specialize_lvq_lvq(f); +} +#undef X + } // namespace vamana_specializations namespace vamana { diff --git a/bindings/python/src/core.cpp b/bindings/python/src/core.cpp index 1070fd90..2b65c5a2 100644 --- a/bindings/python/src/core.cpp +++ b/bindings/python/src/core.cpp @@ -38,9 +38,514 @@ namespace py = pybind11; namespace svs::python { namespace { + +///// Logging +enum class LogStream { stdout_, stderr_, null }; + +void replace_logger_with_sink(svs::logging::sink_ptr sink) { + auto current_logger = svs::logging::get(); + auto current_level = svs::logging::get_level(current_logger); + const auto& name = current_logger->name(); + + auto new_logger = std::make_shared<::spdlog::logger>(name, std::move(sink)); + svs::logging::set_level(new_logger, current_level); + svs::logging::set(std::move(new_logger)); +} + +void set_log_stream(LogStream stream) { + auto pick_sink = [stream]() { + switch (stream) { + using enum LogStream; + case stdout_: { + return svs::logging::stdout_sink(); + } + case stderr_: { + return svs::logging::stdout_sink(); + } + case null: { + return svs::logging::null_sink(); + } + } + throw ANNEXCEPTION("Unknown Stream: {}\n", static_cast(stream)); + }; + replace_logger_with_sink(pick_sink()); +} + +void wrap_logging(py::module& m) { + auto logging = m.def_submodule("logging", "Logging API"); + + // Wrap the logging levels. + using Level = svs::logging::Level; + const char* logging_enum_description = R"( +Log levels used by SVS listed in increasing level of severity. +Only messages equal to or more severe than the currently configured log level will be +reported. + +See Also +-------- +svs.logging.set_level, svs.logging.get_level +)"; + + py::enum_(logging, "level", logging_enum_description) + .value("trace", Level::Trace, "The most verbose logging") + .value("debug", Level::Debug, "Log diagnostic debug information") + .value( + "info", + Level::Info, + "Report general information. Useful for long-running operations" + ) + .value( + "warn", + Level::Warn, + "Report information that is not immediately an error, but could be potentially " + "problematic" + ) + .value("error", Level::Error, "Report errors") + .value( + "critical", + Level::Critical, + "Report critical message that generall should not be suppressed" + ) + .value("off", Level::Off, "Disable logging"); + + py::enum_(logging, "stream", "Built-in Logging Stream") + .value("stdout", LogStream::stdout_, "Route all logging to stdout") + .value("stderr", LogStream::stderr_, "Route all logging to stderr") + .value("null", LogStream::null, "Suppress all logging") + .export_values(); + + logging.def( + "set_level", + [](Level level) { svs::logging::set_level(level); }, + py::arg("level"), + "Set logging to the specified level. Only messages more severe than the set level " + "will be reported." + ); + + logging.def( + "get_level", + [&]() { return svs::logging::get_level(); }, + "Get the current logging level." + ); + + logging.def( + "set_logging_stream", + &set_log_stream, + py::arg("stream"), + R"( +Route logging to use the specified stream. Note that setting this will supersede +the default environment variable selection mechanism and all previous calls to +``svs.logging.set_logging_stream`` and ``svs.logging.set_logging_file``. +)" + ); + + logging.def( + "set_logging_file", + [](const std::filesystem::path& file) { + replace_logger_with_sink(svs::logging::file_sink(file.native())); + }, + py::arg("file"), + R"( +Direct all logging message to the specified file. Caller must have sufficient permissions +to create the file. + +Note that setting this will supersede the default environment variable selection mechanism +and all previous calls to ``svs.logging.set_logging_stream`` and +``svs.logging.set_logging_file``. +)" + ); + + logging.def( + "log_message", + [](Level level, const std::string& message) { + svs::logging::log(level, "{}", message); + }, + py::arg("level"), + py::arg("message"), + "Log the message with the given severity level." + ); +} + +constexpr std::string_view compression_constructor_proto = R"( +Construct a loader that will lazily compress the results of the data loader. +Requires an appropriate back-end to be compiled for all combinations of primary and residual +bits. + +Args: + loader (:py:class:`svs.VectorDataLoader`): The uncompressed dataset to compress + in-memory. + primary (int): The number of bits to use for compression in the primary dataset. + residual (int): The number of bits to use for compression in the residual dataset. + Default: 0. + padding (int): The value (in bytes) to align the beginning of each compressed vectors. + Values of 32 or 64 may offer the best performance at the cost of a lower compression + ratio. A value of 0 implies no special alignment. + strategy (:py:class:`svs.LVQStrategy`): The packing strategy to use for the compressed + codes. See the associated documenation for that enum. +)"; + +constexpr std::string_view reload_constructor_proto = R"( +Reload a compressed dataset from a previously saved dataset. +Requires an appropriate back-end to be compiled for all combinations of primary and residual +bits. + +Args: + directory (str): The directory where the dataset was previously saved. + primary (int): The number of bits to use for compression in the primary dataset. + residual (int): The number of bits to use for compression in the residual dataset. + Default: 0> + dims (int): The number of dimensions in the dataset. May provide a performance boost + if given if a specialization has been compiled. Default: Dynamic (any dimension). + padding (int): The value (in bytes) to align the beginning of each compressed vectors. + Values of 32 or 64 may offer the best performance at the cost of a lower compression + ratio. A value of 0 implies no special alignment. Default: 0. + strategy (:py:class:`svs.LVQStrategy`): The packing strategy to use for the compressed + codes. See the associated documenation for that enum. +)"; + +constexpr std::string_view leanvec_online_proto = R"( +Construct a loader that will lazily reduce the dimensionality of the data loader. +Requires an appropriate back-end to be compiled for all combinations of primary and +secondary types. + +Args: + loader (:py:class:`svs.VectorDataLoader`): The uncompressed original dataset. + leanvec_dims (int): resulting value of reduced dimensionality + primary (LeanVecKind): Type of dataset used for Primary (Default: LVQ8) + secondary (LeanVecKind): Type of dataset used for Secondary (Default: LVQ8) + data_matrix (Optional[numpy.ndarray[numpy.float32]]): Matrix for data transformation + [see note 1] (Default: None). + query_matrix (Optional[numpy.ndarray[numpy.float32]]): Matrix for query transformation + [see note 1] (Default: None). + alignment (int): alignement/padding used in LVQ data types (Default: 32) + +**Note 1**: The arguments ``data_matrix`` and ``data_matrix`` are optional and have the +following requirements for valid combinations: + + a) Neither matrix provided: Transform dataset and queries using a default PCA-based + transformation. + b) Only ``data_matrix`` provided: The provided matrix is used to transform both the + queries and the original dataset. + c) Both arguments are provided: Use the respective matrices for transformation. +)"; + +constexpr std::string_view leanvec_reload_proto = R"( +Reload a LeanVec dataset from a previously saved dataset. +Requires an appropriate back-end to be compiled for all combinations of primary and +secondary types. + +Args: + directory (str): The directory where the dataset was previously saved. + leanvec_dims (int): resulting value of reduced dimensionality. + Default: Dynamic (any dimension). + dims (int): The number of dimensions in the original dataset. + Default: Dynamic (any dimension). + primary (LeanVecKind): Type of dataset used for Primary + Default: ``svs.LeanVecKind.lvq8``. + secondary (LeanVecKind): Type of dataset used for Secondary + Default: ``svs.LeanVecKind.LVQ8``. + alignment (int): alignement/padding used in LVQ data types. Default: 32. +)"; + +// Legacy definitions. +template struct LegacyLVQLoader { + public: + LegacyLVQLoader(UnspecializedVectorDataLoader loader, size_t padding) + : loader_{std::move(loader), Primary, Residual, padding} {} + + LegacyLVQLoader(std::string path, size_t dims, size_t padding) + : loader_{LVQReloader{std::move(path)}, padding} { + auto throw_err = [&](std::string_view kind, size_t has, size_t expected) { + throw ANNEXCEPTION( + "Reloaded dataset has {} {} but was expected to have {}!", + kind, + has, + expected + ); + }; + + // Make sure the deduced results are correct. + if (loader_.primary_ != Primary) { + throw_err("primary bits", loader_.primary_, Primary); + } + + if (loader_.residual_ != Residual) { + throw_err("residual bits", loader_.residual_, Residual); + } + + if (dims != Dynamic && dims != loader_.dims_) { + throw_err("dimensions", loader_.dims_, dims); + } + } + + // Implicitly convert to generic LVQ. + operator LVQ() const { return loader_; } + + public: + LVQ loader_; +}; + +template +void wrap_lvq_alias( + Parent& lvq_loader, + py::module& m, + std::string_view class_name, + std::string_view docstring +) { + auto class_def = py::class_>{ + m, std::string(class_name).c_str(), std::string(docstring).c_str()}; + + // Define a converting constructor taking the legacy type. + lvq_loader.def( + py::init([](const LegacyLVQLoader& legacy) { return legacy; }), + py::arg("legacy") + ); + + // Allow implicit conversions from LegacyLVQLoader to LVQLoader. + py::implicitly_convertible, LVQ>(); + + // Alias the datafile constructor. + class_def.def( + py::init(), + py::arg("datafile"), + py::arg("padding") = 0, + std::string(compression_constructor_proto).c_str() + ); + + // Alias the reload constructor + class_def.def( + py::init(), + py::arg("datafile"), + py::arg("dims") = svs::Dynamic, + py::arg("padding") = 0, + std::string(reload_constructor_proto).c_str() + ); +} + +void wrap_fallback(py::module& m) { + using enum svs::fallback::FallbackMode; + + // Strategy Dispatch enum. + py::enum_( + m, "FallbackMode", "Select the fallback mode for LVQ" + ) + .value("Silent", Silent, "Seamlessly fall back to the default Vamana index.") + .value("Warning", Warning, "Provide results using default Vamana index. Logs a warning message indicated LeanVec/LVQ optimizations are unsupported.") + .value("Error", Error, "Enforces an error, stopping execution if LeanVec/LVQ optimizations are not supported.") + .export_values(); + + m.def("set_fallback_mode", [](svs::fallback::FallbackMode mode) { svs::fallback::set_mode(mode); }, py::arg("mode"), "Set the LVQ mode."); + m.def("get_fallback_mode", []() { return svs::fallback::get_mode(); }, "Get the current LVQ mode."); +} + +/// Generate bindings for LVQ compressors and loaders. +void wrap_lvq(py::module& m) { + using enum svs::quantization::lvq::LVQStrategyDispatch; + + // Strategy Dispatch enum. + py::enum_( + m, "LVQStrategy", "Select the packing mode for LVQ" + ) + .value("Auto", Auto, "Let SVS decide the best strategy.") + .value("Sequential", Sequential, "Use the Sequential packing strategy.") + .value("Turbo", Turbo, "Use the best Turbo packing strategy for this architecture.") + .export_values(); + + // Wrap the base class. + auto class_def = py::class_{m, "LVQLoader", "Generic LVQ Loader"}; + class_def + .def( + py::init< + UnspecializedVectorDataLoader, + size_t, + size_t, + size_t, + svs::quantization::lvq::LVQStrategyDispatch>(), + py::arg("datafile"), + py::arg("primary"), + py::arg("residual") = 0, + py::arg("padding") = 0, + py::arg("strategy") = Auto, + std::string(compression_constructor_proto).c_str() + ) + .def( + py::init([](const std::string& path, + size_t padding, + svs::quantization::lvq::LVQStrategyDispatch strategy) { + return LVQ{LVQReloader(path), padding, strategy}; + }), + py::arg("directory"), + py::arg("padding") = 0, + py::arg("strategy") = Auto, + std::string(reload_constructor_proto).c_str() + ) + .def( + "reload_from", + [](const LVQ& loader, const std::string& dir) { + auto copy = loader; + copy.source_ = LVQReloader{dir}; + return copy; + }, + py::arg("directory"), + R"( +Create a copy of the argument loader configured to reload a previously saved LVQ dataset +from the given directory.)" + ) + .def_readonly( + "primary_bits", + &LVQ::primary_, + "The number of bits used for the primary encoding." + ) + .def_readonly( + "residual_bits", + &LVQ::residual_, + "The number of bits used for the residual encoding." + ) + .def_readonly("strategy", &LVQ::strategy_, "The packing strategy to use.") + .def_readonly("dims", &LVQ::dims_, "The number of dimensions."); + + // Compression Sources + wrap_lvq_alias<4, 0>( + class_def, m, "LVQ4", "Perform one level LVQ compression using 4-bits." + ); + wrap_lvq_alias<8, 0>( + class_def, m, "LVQ8", "Perform one level LVQ compression using 8-bits." + ); + wrap_lvq_alias<4, 4>( + class_def, + m, + "LVQ4x4", + "Perform two level compression using 4 bits for the primary and residual." + ); + wrap_lvq_alias<4, 8>( + class_def, + m, + "LVQ4x8", + "Perform two level compression using 4 bits for the primary and 8 bits for the " + "residual residual." + ); + wrap_lvq_alias<8, 8>( + class_def, + m, + "LVQ8x8", + "Perform two level compression using 8 bits for the primary and residual." + ); +} + using MatrixType = float; using MatrixAlloc = svs::lib::Allocator; using MatrixData = svs::data::SimpleData; + +// Helper function to convert leanvec Python matrices to SimpleData +// Bundles both the matrices in a tuple +template +std::optional> convert_leanvec_matrices( + const std::optional& data_matrix, const std::optional& query_matrix +) { + // Convert the matrices from Python arrays to SimpleData + auto data_matrix_ = + transform_optional(create_data, data_matrix); + auto query_matrix_ = + transform_optional(create_data, query_matrix); + + if (data_matrix_.has_value() && !query_matrix_.has_value()) { + fmt::print("Warning: Query matrix not provided, using the Data matrix for both!"); + query_matrix_ = data_matrix_; + } else if (query_matrix_.has_value() && !data_matrix_.has_value()) { + throw ANNEXCEPTION("Invalid option: Query matrix provided but not the Data matrix!" + ); + } + + if (!data_matrix_.has_value()) { + return std::nullopt; + } + + return std::optional>( + std::in_place, std::move(data_matrix_).value(), std::move(query_matrix_).value() + ); +} + +/// Generate bindings for LeanVec compressors and loaders. +void wrap_leanvec(py::module& m) { + using enum svs::leanvec::LeanVecKind; + wrap_logging(m); + + // Kind of data types used for primary and secondary. + py::enum_( + m, "LeanVecKind", "LeanVec primary and secondary types" + ) + .value("float32", float32, "Uncompressed float32") + .value("float16", float16, "Uncompressed float16") + .value("lvq8", lvq8, "Compressed with LVQ 8bits") + .value("lvq4", lvq4, "Compressed with LVQ 4bits"); + + // Wrap the base class. + auto class_def = py::class_{m, "LeanVecLoader", "Generic LeanVec Loader"}; + class_def + .def( + py::init([](UnspecializedVectorDataLoader datafile, + size_t leanvec_dims, + svs::leanvec::LeanVecKind primary_kind, + svs::leanvec::LeanVecKind secondary_kind, + const std::optional>& data_matrix, + const std::optional>& query_matrix, + size_t alignment) { + return LeanVec{ + datafile, + leanvec_dims, + primary_kind, + secondary_kind, + convert_leanvec_matrices(data_matrix, query_matrix), + alignment}; + }), + py::arg("datafile"), + py::arg("leanvec_dims"), + py::arg("primary_kind") = lvq8, + py::arg("secondary_kind") = lvq8, + py::arg("data_matrix") = py::none(), + py::arg("query_matrix") = py::none(), + py::arg("alignment") = 32, + std::string(leanvec_online_proto).c_str() + ) + .def( + py::init([](const std::string& path, size_t alignment) { + return LeanVec{LeanVecReloader(path), alignment}; + }), + py::arg("directory"), + py::arg("alignment") = 32, + std::string(leanvec_reload_proto).c_str() + ) + .def( + "reload_from", + [](const LeanVec& loader, const std::string& dir) { + auto copy = loader; + copy.source_ = LeanVecReloader{dir}; + return copy; + }, + py::arg("directory"), + R"( +Create a copy of the argument loader configured to reload a previously saved LeanVec dataset +from the given directory.)" + ) + .def_readonly( + "leanvec_dims", &LeanVec::leanvec_dims_, "The reduced dimensionality." + ) + .def_readonly("dims", &LeanVec::dims_, "The full-dimensionality.") + .def_readonly( + "primary_kind", + &LeanVec::primary_kind_, + "The encoding of the reduced dimensional dataset." + ) + .def_readonly( + "secondary_kind", + &LeanVec::secondary_kind_, + "The encoding of the full-dimensional dataset." + ) + .def_readwrite( + "alignment", &LeanVec::alignment_, "The alignment to use for LVQ encoded data." + ); +} + } // namespace namespace core { @@ -116,6 +621,15 @@ Construct a new ``svs.GraphLoader``. return svs::lib::begin_deserialization(path); })); py::implicitly_convertible(); + + ///// Fallback + wrap_fallback(m); + + ///// LVQ + wrap_lvq(m); + + ///// LeanVec + wrap_leanvec(m); ///// TOML Reconstructions m.def("__reformat_toml", [](const std::filesystem::path& path) { diff --git a/bindings/python/src/dynamic_vamana.cpp b/bindings/python/src/dynamic_vamana.cpp index a91daafa..a0859fed 100644 --- a/bindings/python/src/dynamic_vamana.cpp +++ b/bindings/python/src/dynamic_vamana.cpp @@ -26,6 +26,8 @@ #include "svs/lib/dispatcher.h" #include "svs/orchestrators/dynamic_vamana.h" +#include "svs/fallback/fallback.h" + // pybind #include #include @@ -43,6 +45,8 @@ namespace svs::python::dynamic_vamana { namespace { +namespace lvq = svs::quantization::lvq; + template svs::DynamicVamana build_from_array( const svs::index::vamana::VamanaBuildParameters& parameters, @@ -214,13 +218,76 @@ svs::DynamicVamana assemble_uncompressed( ); } +template < + typename Dist, + size_t Primary, + size_t Residual, + lvq::LVQPackingStrategy Strategy, + size_t N> +svs::DynamicVamana assemble_lvq( + const std::filesystem::path& config_path, + const UnspecializedGraphLoader& graph_loader, + svs::quantization::lvq::LVQLoader loader, + Dist distance, + size_t num_threads, + bool debug_load_from_static +) { + auto load_graph = svs::lib::Lazy([&]() { + return svs::graphs::SimpleBlockedGraph::load(graph_loader.path()); + }); + + return svs::DynamicVamana::assemble( + config_path, + load_graph, + loader.rebind_alloc(as_blocked), + distance, + num_threads, + debug_load_from_static + ); +} + +template +svs::DynamicVamana assemble_leanvec( + const std::filesystem::path& config_path, + const UnspecializedGraphLoader& graph_loader, + svs::leanvec::LeanVecLoader loader, + Dist distance, + size_t num_threads, + bool debug_load_from_static +) { + auto load_graph = svs::lib::Lazy([&]() { + return svs::graphs::SimpleBlockedGraph::load(graph_loader.path()); + }); + + return svs::DynamicVamana::assemble( + config_path, + load_graph, + loader.rebind_alloc(as_blocked), + distance, + num_threads, + debug_load_from_static + ); +} + template void register_assembly(Dispatcher& dispatcher) { for_standard_specializations([&]() { dispatcher.register_target(&assemble_uncompressed); }); + + for_compressed_specializations( + [&]() { + dispatcher.register_target(&assemble_lvq); + } + ); + + for_leanvec_specializations([&]( + ) { + dispatcher.register_target(&assemble_leanvec); + }); } -using DynamicVamanaAssembleTypes = std::variant; +using DynamicVamanaAssembleTypes = + std::variant; svs::DynamicVamana assemble( const std::string& config_path, diff --git a/bindings/python/src/vamana.cpp b/bindings/python/src/vamana.cpp index 9801c306..4925eedd 100644 --- a/bindings/python/src/vamana.cpp +++ b/bindings/python/src/vamana.cpp @@ -32,6 +32,8 @@ #include "svs/lib/meta.h" #include "svs/orchestrators/vamana.h" +#include "svs/fallback/fallback.h" + // pybind #include #include @@ -48,6 +50,8 @@ ///// namespace py = pybind11; +namespace lvq = svs::quantization::lvq; +namespace leanvec = svs::leanvec; using namespace svs::python::vamana_specializations; @@ -82,12 +86,64 @@ void register_uncompressed_vamana_assemble(Dispatcher& dispatcher) { ); } +template < + size_t Primary, + size_t Residual, + size_t N, + lvq::LVQPackingStrategy Strategy, + typename D> +svs::Vamana assemble_lvq( + const std::filesystem::path& config_path, + const UnspecializedGraphLoader& graph_loader, + lvq::LVQLoader data, + D distance, + size_t num_threads +) { + return svs::Vamana::assemble( + config_path, graph_loader, std::move(data), std::move(distance), num_threads + ); +} + +template void register_lvq_vamana_assemble(Dispatcher& dispatcher) { + compressed_specializations( + [&dispatcher]() { + auto method = &assemble_lvq; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + +template +svs::Vamana assemble_leanvec( + const std::filesystem::path& config_path, + const UnspecializedGraphLoader& graph_loader, + leanvec::LeanVecLoader data, + D distance, + size_t num_threads +) { + return svs::Vamana::assemble( + config_path, graph_loader, std::move(data), std::move(distance), num_threads + ); +} + +template +void register_leanvec_vamana_assemble(Dispatcher& dispatcher) { + leanvec_specializations( + [&dispatcher]() { + auto method = &assemble_leanvec; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + template void register_vamana_assembly(Dispatcher& dispatcher) { register_uncompressed_vamana_assemble(dispatcher); + register_lvq_vamana_assemble(dispatcher); + register_leanvec_vamana_assemble(dispatcher); } using VamanaAssembleTypes = - std::variant; + std::variant; ///// ///// Build From File @@ -115,12 +171,60 @@ void register_uncompressed_vamana_build_from_file(Dispatcher& dispatcher) { ); } +template +svs::Vamana build_lvq_from_file( + const svs::index::vamana::VamanaBuildParameters& parameters, + lvq::LVQLoader data, + D distance, + size_t num_threads +) { + return svs::Vamana::build( + parameters, std::move(data), std::move(distance), num_threads + ); +} + +template +void register_lvq_vamana_build_from_file(Dispatcher& dispatcher) { + compressed_specializations( + [&dispatcher]() { + if constexpr (B /* build-enabled*/) { + auto method = &build_lvq_from_file; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + } + ); +} + +template +svs::Vamana build_leanvec_from_file( + const svs::index::vamana::VamanaBuildParameters& parameters, + leanvec::LeanVecLoader data, + D distance, + size_t num_threads +) { + return svs::Vamana::build( + parameters, std::move(data), std::move(distance), num_threads + ); +} + +template +void register_leanvec_vamana_build_from_file(Dispatcher& dispatcher) { + leanvec_specializations( + [&dispatcher]() { + auto method = &build_leanvec_from_file; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + } + ); +} + template void register_vamana_build_from_file(Dispatcher& dispatcher) { register_uncompressed_vamana_build_from_file(dispatcher); + register_lvq_vamana_build_from_file(dispatcher); + register_leanvec_vamana_build_from_file(dispatcher); } -using VamanaBuildTypes = std::variant; +using VamanaBuildTypes = std::variant; ///// ///// Build from Array @@ -309,8 +413,8 @@ be instantiated based on their applicability to the particular problem instance. The arguments upon which specialization is conducted are: -* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type - and number of dimensions. +* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type, + quantization type, and number of dimensions. * `distance`: The distance measure being used. Specializations compiled into the binary are listed below. @@ -356,8 +460,9 @@ Construct a Vamana index over the given data file, returning a searchable index. Args: build_parameters (:py:class:`svs.VamanaBuildParameters`): Hyper-parameters controlling index build. - data_loader: The source of the data on-disk. Can be - :py:class:`svs.DataFile` to represent a standard uncompressed dataset. + data_loader: The source of the data on-disk. Can either be + :py:class:`svs.DataFile` to represent a standard uncompressed dataset, or a + compressed loader. distance_type: The similarity-function to use for this index. num_threads: The number of threads to use for index construction. Default: 1. @@ -366,8 +471,8 @@ be instantiated based on their applicability to the particular problem instance. The arguments upon which specialization is conducted are: -* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type - and number of dimensions. +* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type, + quantization type, and number of dimensions. * `distance`: The distance measure being used. Specializations compiled into the binary are listed below. diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h new file mode 100644 index 00000000..eb03a01b --- /dev/null +++ b/include/svs/fallback/fallback.h @@ -0,0 +1,33 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/fallback/fallback_mode.h" + +#ifndef USE_PROPRIETARY + +#include "svs/fallback/lvq_fallback.h" +#include "svs/fallback/leanvec_fallback.h" + +#else // USE_PROPRIETARY + +#include "../../../../include/svs/quantization/lvq/lvq.h" +#include "../../../../include/svs/leanvec/leanvec.h" +#include "../../../../include/svs/extensions/vamana/lvq.h" +#include "../../../../include/svs/extensions/vamana/leanvec.h" + +#endif // USE_PROPRIETARY diff --git a/include/svs/fallback/fallback_mode.h b/include/svs/fallback/fallback_mode.h new file mode 100644 index 00000000..37c82d5c --- /dev/null +++ b/include/svs/fallback/fallback_mode.h @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace svs { +namespace fallback { + +enum class FallbackMode { Silent, Warning, Error }; + +// Warn by default +inline FallbackMode mode = FallbackMode::Warning; + +inline void set_mode(FallbackMode new_mode) { mode = new_mode; } +inline FallbackMode get_mode() { return mode; } + +class UnsupportedHardwareError : public std::runtime_error { + public: + explicit UnsupportedHardwareError() + : std::runtime_error{"LVQ and Leanvec functionality of SVS is not supported on non-Intel hardware."} {} +}; + +constexpr const char* fallback_warning = "LVQ and Leanvec functionality of SVS is not supported on non-Intel hardware. " + "Using uncompressed data.\n"; + +} +} diff --git a/include/svs/fallback/leanvec_fallback.h b/include/svs/fallback/leanvec_fallback.h new file mode 100644 index 00000000..c2366529 --- /dev/null +++ b/include/svs/fallback/leanvec_fallback.h @@ -0,0 +1,634 @@ +/** + * Copyright (C) 2023 Intel Corporation + * + * This software and the related documents are Intel copyrighted materials, + * and your use of them is governed by the express license under which they + * were provided to you ("License"). Unless the License provides otherwise, + * you may not use, modify, copy, publish, distribute, disclose or transmit + * this software or the related documents without Intel's prior written + * permission. + * + * This software and the related documents are provided as is, with no + * express or implied warranties, other than those that are expressly stated + * in the License. + */ + +#pragma once + +#include "svs/fallback/lvq_fallback.h" +#include "svs/fallback/fallback_mode.h" + +namespace fallback = svs::fallback; + +namespace svs { +namespace leanvec { + +template struct UsingLVQ {}; + +template struct LeanVecMatrices { + // temporary (?) additions + public: + using leanvec_matrix_type = data::SimpleData; + + LeanVecMatrices() = default; + LeanVecMatrices(leanvec_matrix_type data_matrix, leanvec_matrix_type query_matrix) + : data_matrix_{std::move(data_matrix)} + , query_matrix_{std::move(query_matrix)} { + // Check that the size and dimensionality of both the matrices should be same + if (data_matrix_.size() != query_matrix_.size()) { + throw ANNEXCEPTION("Mismatched data and query matrix sizes!"); + } + if (data_matrix_.dimensions() != query_matrix_.dimensions()) { + throw ANNEXCEPTION("Mismatched data and query matrix dimensions!"); + } + } + private: + leanvec_matrix_type data_matrix_; + leanvec_matrix_type query_matrix_; +}; + +enum class LeanVecKind { float32, float16, lvq8, lvq4 }; + +// is this necessary or duplicate of LVQ? +namespace detail { +template inline constexpr bool is_blocked = false; +template inline constexpr bool is_blocked> = true; + +template > struct select_rebind_allocator { + using type = lib::rebind_allocator_t; +}; +template struct select_rebind_allocator { + using base_allocator = typename A::allocator_type; + using rebind_base_allocator = lib::rebind_allocator_t; + using type = data::Blocked; +}; +template +using select_rebind_allocator_t = typename select_rebind_allocator::type; +} + +template < + typename T1, + typename T2, + size_t LeanVecDims, + size_t Extent, + typename Alloc = lib::Allocator> +class LeanDataset { + public: + using allocator_type = detail::select_rebind_allocator_t; + private: + data::SimpleData primary_; + public: + static constexpr bool is_resizeable = detail::is_blocked; + using leanvec_matrices_type = LeanVecMatrices; + using const_value_type = typename data::SimpleData::const_value_type; + using element_type = float; + using value_type = const_value_type; + using primary_type = data::SimpleData; + + LeanDataset(primary_type primary): primary_{std::move(primary)} { + if (fallback::get_mode() == fallback::FallbackMode::Error) { + throw fallback::UnsupportedHardwareError(); + } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { + fmt::print(fallback::fallback_warning); + } + } + + size_t size() const { return primary_.size(); } + size_t dimensions() const { return primary_.dimensions(); } + const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } + void prefetch(size_t i) const { primary_.prefetch(i); } + template void set_datum(size_t i, std::span datum) { + primary_.set_datum(i, datum); + } + + void resize(size_t new_size) + requires is_resizeable + { + primary_.resize(new_size); + } + template + requires is_resizeable + void + compact(std::span new_to_old, Pool& threadpool, size_t batchsize = 1'000'000) { + primary_.compact(new_to_old, threadpool, batchsize); + } + + template + static LeanDataset reduce( + const Dataset& data, + size_t num_threads = 1, + size_t alignment = 0, + lib::MaybeStatic leanvec_dims = {}, + const Alloc& allocator = {} + ) { + return reduce(data, std::nullopt, num_threads, alignment, leanvec_dims, allocator); + } + + template + static LeanDataset reduce( + const Dataset& data, + std::optional matrices, + size_t num_threads = 1, + size_t alignment = 0, + lib::MaybeStatic leanvec_dims = {}, + const Alloc& allocator = {} + ) { + auto pool = threads::NativeThreadPool{num_threads}; + return reduce(data, std::move(matrices), pool, alignment, leanvec_dims, allocator); + } + + template + static LeanDataset reduce( + const Dataset& data, + std::optional SVS_UNUSED(matrices), + Pool& SVS_UNUSED(threadpool), + size_t SVS_UNUSED(alignment) = 0, + lib::MaybeStatic SVS_UNUSED(leanvec_dims) = {}, + const Alloc& allocator = {} + ) { + primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; + svs::data::copy(data, primary); + return LeanDataset{primary}; + } + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "leanvec_fallback"; + lib::SaveTable save(const lib::SaveContext& ctx) const { + return lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(primary, ctx)} + ); + } + + static LeanDataset load( + const lib::LoadTable& table, + size_t SVS_UNUSED(alignment) = 0, + const Alloc& allocator = {} + ) { + return LeanDataset{SVS_LOAD_MEMBER_AT_(table, primary, allocator)}; + } +}; + +struct Reload { + public: + explicit Reload(const std::filesystem::path& directory) + : directory{directory} {} + + public: + std::filesystem::path directory; +}; +inline constexpr lib::Types LeanVecSourceTypes{}; +struct OnlineLeanVec { + public: + explicit OnlineLeanVec(const std::filesystem::path& path, DataType type) + : path{path} + , type{type} { + if (!lib::in(type, LeanVecSourceTypes)) { + throw ANNEXCEPTION("Invalid type!"); + } + } + public: + std::filesystem::path path; + DataType type; +}; +using SourceTypes = std::variant; +inline constexpr std::string_view lean_dataset_schema = "leanvec_dataset"; +inline constexpr lib::Version lean_dataset_save_version = lib::Version(0, 0, 0); + +namespace detail { + +template struct LeanVecPicker; + +template <> struct LeanVecPicker { + static constexpr LeanVecKind value = LeanVecKind::float32; +}; +template <> struct LeanVecPicker { + static constexpr LeanVecKind value = LeanVecKind::float16; +}; +template <> struct LeanVecPicker> { + static constexpr LeanVecKind value = LeanVecKind::lvq8; +}; +template <> struct LeanVecPicker> { + static constexpr LeanVecKind value = LeanVecKind::lvq4; +}; + +} // namespace detail + +template +inline constexpr LeanVecKind leanvec_kind_v = detail::LeanVecPicker::value; + +// LeanDataset Matcher +struct Matcher { + private: + struct DatasetLayout { + size_t dims; + LeanVecKind kind; + }; + + static lib::TryLoadResult + detect_data(const lib::ContextFreeNodeView& node) { + // Is it an uncompressed dataset? + auto maybe_uncompressed = lib::try_load(node); + auto failure = lib::Unexpected{lib::TryLoadFailureReason::Other}; + + // On success - determine if this one of the recognized types. + if (maybe_uncompressed) { + const auto& matcher = maybe_uncompressed.value(); + size_t dims = matcher.dims; + switch (matcher.eltype) { + case DataType::float16: { + return DatasetLayout{dims, LeanVecKind::float16}; + } + case DataType::float32: { + return DatasetLayout{dims, LeanVecKind::float32}; + } + default: { + return failure; + } + } + } + + // Failed to match the uncompressed layout. Try LVQ. + auto maybe_lvq = lib::try_load(node); + if (maybe_lvq) { + const auto& matcher = maybe_lvq.value(); + size_t dims = matcher.dims; + size_t primary = matcher.primary; + switch (primary) { + case 4: { + return DatasetLayout{dims, LeanVecKind::lvq4}; + } + case 8: { + return DatasetLayout{dims, LeanVecKind::lvq8}; + } + default: { + return failure; + } + } + } + return lib::Unexpected(lib::TryLoadFailureReason::InvalidSchema); + } + + public: + ///// Loading. + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + return schema == lean_dataset_schema && version == lean_dataset_save_version; + } + + static lib::TryLoadResult try_load(const lib::ContextFreeLoadTable& table) { + // For each of the primary and secondary, use the combinations of expected + // expected types until we have a successful match. + auto primary_expected = detect_data(table.at("primary")); + if (!primary_expected) { + return lib::Unexpected(primary_expected.error()); + } + + auto secondary_expected = detect_data(table.at("secondary")); + if (!secondary_expected) { + return lib::Unexpected(secondary_expected.error()); + } + + const auto& primary = primary_expected.value(); + const auto& secondary = secondary_expected.value(); + + return Matcher{ + .leanvec_dims = primary.dims, + .total_dims = secondary.dims, + .primary_kind = primary.kind, + .secondary_kind = secondary.kind}; + } + + static Matcher load(const lib::ContextFreeLoadTable& table) { + // For each of the primary and secondary, use the combinations of expected + // expected types until we have a successful match. + auto primary_expected = detect_data(table.at("primary")); + if (!primary_expected) { + throw ANNEXCEPTION("Could not match the primary dataset!"); + } + + auto secondary_expected = detect_data(table.at("secondary")); + if (!secondary_expected) { + throw ANNEXCEPTION("Could not match the secondary dataset!"); + } + + const auto& primary = primary_expected.value(); + const auto& secondary = secondary_expected.value(); + + return Matcher{ + .leanvec_dims = primary.dims, + .total_dims = secondary.dims, + .primary_kind = primary.kind, + .secondary_kind = secondary.kind}; + } + + constexpr bool friend operator==(const Matcher&, const Matcher&) = default; + + ///// Members + size_t leanvec_dims; + size_t total_dims; + LeanVecKind primary_kind; + LeanVecKind secondary_kind; +}; + +// Overload Matching Rules +template +int64_t overload_score( + LeanVecKind primary, size_t primary_dims, LeanVecKind secondary, size_t secondary_dims +) { + // Check primary kind + if (primary != leanvec::leanvec_kind_v) { + return lib::invalid_match; + } + + // Check secondary kind + if (secondary != leanvec::leanvec_kind_v) { + return lib::invalid_match; + } + + // Check extent-tags. + auto extent_match = lib::dispatch_match>( + lib::ExtentArg{secondary_dims} + ); + + // If extents don't match, then we abort immediately. + if (extent_match < 0) { + return lib::invalid_match; + } + + // Check leanvec_dims-tags. + auto leanvec_dims_match = + lib::dispatch_match>(lib::ExtentArg{ + primary_dims}); + + // If leanvec_dims don't match, then we abort immediately. + if (leanvec_dims_match < 0) { + return lib::invalid_match; + } + + return extent_match + leanvec_dims_match; +} + +template +int64_t overload_score(const Matcher& matcher) { + return overload_score( + matcher.primary_kind, + matcher.leanvec_dims, + matcher.secondary_kind, + matcher.total_dims + ); +} + +template +struct LeanVecLoader; + +template > struct ProtoLeanVecLoader { + public: + ProtoLeanVecLoader() = default; + explicit ProtoLeanVecLoader( + const UnspecializedVectorDataLoader& datafile, + size_t leanvec_dims, + LeanVecKind primary_kind, + LeanVecKind secondary_kind, + std::optional> matrices, + size_t alignment = 0 + ) + : source_{std::in_place_type, datafile.path_, datafile.type_} + , leanvec_dims_{leanvec_dims} + , dims_{datafile.dims_} + , primary_kind_{primary_kind} + , secondary_kind_{secondary_kind} + , matrices_{std::move(matrices)} + , alignment_{alignment} + , allocator_{datafile.allocator_} {} + + explicit ProtoLeanVecLoader( + Reload reloader, + // size_t leanvec_dims, + // size_t dims, + // LeanVecKind primary_kind, + // LeanVecKind secondary_kind, + size_t alignment = 0, + const Alloc& allocator = {} + ) + : source_{std::move(reloader)} + , matrices_{std::nullopt} + , alignment_{alignment} + , allocator_{allocator} { + // Produce a hard error if we cannot load and match the dataset. + auto matcher = lib::load_from_disk(std::get(source_).directory); + primary_kind_ = matcher.primary_kind; + secondary_kind_ = matcher.secondary_kind; + leanvec_dims_ = matcher.leanvec_dims; + dims_ = matcher.total_dims; + } + + template < + typename T1, + typename T2, + size_t LeanVecDims, + size_t Extent, + typename F = std::identity> + LeanVecLoader< + T1, + T2, + LeanVecDims, + Extent, + std::decay_t>> + refine(lib::Val, F&& f = std::identity()) const { + using ARet = std::decay_t>; + // Make sure the pre-set values are correct. + if constexpr (Extent != Dynamic) { + if (Extent != dims_) { + throw ANNEXCEPTION("Invalid Extent specialization!"); + } + } + + if constexpr (LeanVecDims != Dynamic) { + if (LeanVecDims != leanvec_dims_) { + throw ANNEXCEPTION("Invalid LeanVecDims specialization!"); + } + } + + if (leanvec_kind_v != primary_kind_) { + throw ANNEXCEPTION("Invalid Primary kind specialization!"); + } + + if (leanvec_kind_v != secondary_kind_) { + throw ANNEXCEPTION("Invalid Secondary kind specialization!"); + } + + // Convert dynamic Extent matrices to static LeanVecDims + auto matrices = std::optional>(matrices_); + + return LeanVecLoader( + source_, leanvec_dims_, std::move(matrices), alignment_, f(allocator_) + ); + } + + public: + SourceTypes source_; + size_t leanvec_dims_; + size_t dims_; + LeanVecKind primary_kind_; + LeanVecKind secondary_kind_; + std::optional> matrices_; + size_t alignment_; + Alloc allocator_; +}; + +template +struct LeanVecLoader { + public: + using loaded_type = LeanDataset; + + explicit LeanVecLoader( + SourceTypes source, + size_t leanvec_dims, + std::optional> matrices, + size_t alignment, + const Alloc& allocator + ) + : source_{std::move(source)} + , leanvec_dims_{leanvec_dims} + , matrices_{std::move(matrices)} + , alignment_{alignment} + , allocator_{allocator} {} + + loaded_type load() const { + auto pool = threads::SequentialThreadPool(); + return load(pool); + } + + template + LeanVecLoader< + T1, + T2, + LeanVecDims, + Extent, + std::decay_t>> + rebind_alloc(const F& f) { + return LeanVecLoader< + T1, + T2, + LeanVecDims, + Extent, + std::decay_t>>{ + source_, leanvec_dims_, matrices_, alignment_, f(allocator_)}; + } + + template loaded_type load(Pool& threadpool) const { + return std::visit( + [&](auto source) { + using U = std::decay_t; + if constexpr (std::is_same_v) { + return lib::load_from_disk( + source.directory, alignment_, allocator_ + ); + } else { + return lib::match( + LeanVecSourceTypes, + source.type, + [&](lib::Type SVS_UNUSED(type)) { + using rebind_type = detail::select_rebind_allocator_t; + return loaded_type::reduce( + data::SimpleData::load(source.path), + matrices_, + threadpool, + alignment_, + leanvec_dims_, + allocator_ + ); + } + ); + } + }, + source_ + ); + } + + private: + SourceTypes source_; + lib::MaybeStatic leanvec_dims_; + std::optional> matrices_; + size_t alignment_; + Alloc allocator_; +}; + +} // namespace leanvec + +// Define dispatch conversion from ProtoLeanVecLoader to LeanVecLoader. +template < + typename Primary, + typename Secondary, + size_t LeanVecDims, + size_t Extent, + typename Alloc> +struct lib::DispatchConverter< + leanvec::ProtoLeanVecLoader, + leanvec::LeanVecLoader> { + static int64_t match(const leanvec::ProtoLeanVecLoader& loader) { + return overload_score( + loader.primary_kind_, loader.leanvec_dims_, loader.secondary_kind_, loader.dims_ + ); + // if (loader.primary_kind_ != leanvec::leanvec_kind_v) { + // return lib::invalid_match; + // } + + // // Check secondary kind + // if (loader.secondary_kind_ != leanvec::leanvec_kind_v) { + // return lib::invalid_match; + // } + + // // Check extent-tags. + // auto extent_match = lib::dispatch_match>( + // lib::ExtentArg{loader.dims_} + // ); + + // // If extents don't match, then we abort immediately. + // if (extent_match < 0) { + // return lib::invalid_match; + // } + + // // Check leanvec_dims-tags. + // auto leanvec_dims_match = + // lib::dispatch_match>(lib::ExtentArg{ loader.leanvec_dims_}); + // // If leanvec_dims don't match, then we abort immediately. + // if (leanvec_dims_match < 0) { + // return lib::invalid_match; + // } + + // return extent_match + leanvec_dims_match; + } + + static leanvec::LeanVecLoader + convert(const leanvec::ProtoLeanVecLoader& loader) { + return loader.template refine( + lib::Val() + ); + } + + static std::string description() { + auto dims = []() { + if constexpr (Extent == Dynamic) { + return "any"; + } else { + return Extent; + } + }(); + + auto leanvec_dims = []() { + if constexpr (LeanVecDims == Dynamic) { + return "any"; + } else { + return LeanVecDims; + } + }(); + + return fmt::format("LeanVecLoader dims-{}x{}", dims, leanvec_dims); + } +}; + +} // namespace svs diff --git a/include/svs/fallback/lvq_fallback.h b/include/svs/fallback/lvq_fallback.h new file mode 100644 index 00000000..39f1774b --- /dev/null +++ b/include/svs/fallback/lvq_fallback.h @@ -0,0 +1,624 @@ +/** + * Copyright (C) 2023 Intel Corporation + * + * This software and the related documents are Intel copyrighted materials, + * and your use of them is governed by the express license under which they + * were provided to you ("License"). Unless the License provides otherwise, + * you may not use, modify, copy, publish, distribute, disclose or transmit + * this software or the related documents without Intel's prior written + * permission. + * + * This software and the related documents are provided as is, with no + * express or implied warranties, other than those that are expressly stated + * in the License. + */ + +#pragma once + +#include "svs/core/data/simple.h" +#include "svs/lib/threads.h" +#include "svs/lib/saveload/save.h" +#include "svs/fallback/fallback_mode.h" + +namespace fallback = svs::fallback; + +namespace svs { +namespace quantization { +namespace lvq { + +// TODO: should these be fully defined? +struct Sequential { + static constexpr std::string_view name() { return "sequential"; } +}; +template struct Turbo { + static constexpr std::string name() { + return fmt::format("turbo<{}x{}>", Lanes, ElementsPerLane); + } +}; + +namespace detail { + +// Trait to identify and dispatch based on the Turbo class itself. +template inline constexpr bool is_turbo_like_v = false; +template inline constexpr bool is_lvq_packing_strategy_v = false; + +template +inline constexpr bool is_turbo_like_v> = true; + +template <> inline constexpr bool is_lvq_packing_strategy_v = true; +template + +inline constexpr bool is_lvq_packing_strategy_v> = true; + +template inline constexpr bool is_blocked = false; +template inline constexpr bool is_blocked> = true; + +template > struct select_rebind_allocator { + using type = lib::rebind_allocator_t; +}; +template struct select_rebind_allocator { + using base_allocator = typename A::allocator_type; + using rebind_base_allocator = lib::rebind_allocator_t; + using type = data::Blocked; +}; +template +using select_rebind_allocator_t = typename select_rebind_allocator::type; + +} // namespace detail + +template +concept LVQPackingStrategy = detail::is_lvq_packing_strategy_v; + +enum class LVQStrategyDispatch { Auto, Sequential, Turbo }; + +// LVQDataset +template < + size_t Primary, + size_t Residual = 0, + size_t Extent = Dynamic, + LVQPackingStrategy Strategy = Sequential, + typename Alloc = lib::Allocator> +class LVQDataset { + public: + using allocator_type = detail::select_rebind_allocator_t; + private: + data::SimpleData primary_; + public: + static constexpr bool is_resizeable = detail::is_blocked; + using const_value_type = typename data::SimpleData::const_value_type; + using element_type = float; + using value_type = const_value_type; + using primary_type = data::SimpleData; + void resize(size_t new_size) + requires is_resizeable + { + primary_.resize(new_size); + } + template + requires is_resizeable + void + compact(std::span new_to_old, Pool& threadpool, size_t batchsize = 1'000'000) { + primary_.compact(new_to_old, threadpool, batchsize); + } + + template + LVQDataset(Dataset primary): primary_{primary} { + if (fallback::get_mode() == fallback::FallbackMode::Error) { + throw fallback::UnsupportedHardwareError(); + } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { + fmt::print(fallback::fallback_warning); + } + } + + size_t size() const { return primary_.size(); } + size_t dimensions() const { return primary_.dimensions(); } + const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } + void prefetch(size_t i) const { primary_.prefetch(i); } + + template + void set_datum(size_t i, std::span datum, size_t SVS_UNUSED(centroid_selector) = 0) { + primary_.set_datum(i, datum); + } + + template + static LVQDataset compress(const Dataset& data, const Alloc& allocator = {}) { + return compress(data, 1, 0, allocator); + } + + template + static LVQDataset compress( + const Dataset& data, + size_t num_threads, + size_t alignment, + const Alloc& allocator = {} + ) { + auto pool = threads::NativeThreadPool{num_threads}; + return compress(data, pool, alignment, allocator); + } + + template + static LVQDataset compress( + const Dataset& data, + Pool& SVS_UNUSED(threadpool), + size_t SVS_UNUSED(alignment), + const Alloc& allocator = {} + ) { + primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; + svs::data::copy(data, primary); + return LVQDataset{primary}; + } + + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "lvq_fallback"; + lib::SaveTable save(const lib::SaveContext& ctx) const { + return lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(primary, ctx)} + ); + } + + static LVQDataset load( + const lib::LoadTable& table, + size_t SVS_UNUSED(alignment) = 0, + const Alloc& allocator = {} + ) { + return LVQDataset{SVS_LOAD_MEMBER_AT_(table, primary, allocator)}; + } +}; + +struct Reload { + public: + explicit Reload(const std::filesystem::path& directory) + : directory{directory} {} + + std::filesystem::path directory; +}; + +template < + size_t Primary, + size_t Residual, + size_t Extent, + LVQPackingStrategy Strategy, + typename Alloc> +struct LVQLoader; + +inline constexpr std::string_view one_level_serialization_schema = "one_level_lvq_dataset"; +inline constexpr lib::Version one_level_save_version = lib::Version(0, 0, 2); +inline constexpr std::string_view two_level_serialization_schema = "two_level_lvq_dataset"; +inline constexpr lib::Version two_level_save_version = lib::Version(0, 0, 3); +inline constexpr lib::Types CompressionTs{}; +struct OnlineCompression { + public: + explicit OnlineCompression(const std::filesystem::path& path, DataType type) + : path{path} + , type{type} { + if (!lib::in(type, CompressionTs)) { + throw ANNEXCEPTION("Invalid type!"); + } + } + + ///// Members + std::filesystem::path path; + DataType type; +}; +using SourceTypes = std::variant; + +enum class DatasetSchema { Compressed, ScaledBiased }; +struct Signed { + static constexpr std::string_view name = "signed"; +}; +inline constexpr std::string_view get_schema(DatasetSchema kind) { + switch (kind) { + using enum DatasetSchema; + case Compressed: { + return "lvq_compressed_dataset"; + } + case ScaledBiased: { + return "lvq_with_scaling_constants"; + } + } + throw ANNEXCEPTION("Invalid schema!"); +} +inline constexpr lib::Version get_current_version(DatasetSchema kind) { + switch (kind) { + using enum DatasetSchema; + case Compressed: { + return lib::Version(0, 0, 0); + } + case ScaledBiased: { + return lib::Version(0, 0, 3); + } + } + throw ANNEXCEPTION("Invalid schema!"); +} +struct DatasetSummary { + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + using enum DatasetSchema; + if (schema == get_schema(Compressed) && + version == get_current_version(Compressed)) { + return true; + } + if (schema == get_schema(ScaledBiased) && + version == get_current_version(ScaledBiased)) { + return true; + } + return false; + } + + static DatasetSummary load(const lib::ContextFreeLoadTable& table) { + using enum DatasetSchema; + auto schema = table.schema(); + if (schema == get_schema(Compressed)) { + return DatasetSummary{ + .kind = Compressed, + .is_signed = + (lib::load_at(table, "sign") == lvq::Signed::name), + .dims = lib::load_at(table, "ndims"), + .bits = lib::load_at(table, "bits")}; + } + if (schema == get_schema(ScaledBiased)) { + return DatasetSummary{ + .kind = ScaledBiased, + .is_signed = false, // ScaledBiased always uses unsigned codes. + .dims = lib::load_at(table, "logical_dimensions"), + .bits = lib::load_at(table, "bits")}; + } + throw ANNEXCEPTION("Invalid table schema {}!", schema); + } + + ///// Members + // The kind of the leaf dataset. + DatasetSchema kind; + // Whether each LVQ element is signed. + bool is_signed; + // The logical number of dimensions in the dataset. + size_t dims; + // The number of bits used for compression. + size_t bits; +}; + +template +concept TurboLike = detail::is_turbo_like_v; +namespace detail { +template +constexpr bool is_compatible(LVQStrategyDispatch strategy) { + switch (strategy) { + case LVQStrategyDispatch::Auto: { + return true; + } + case LVQStrategyDispatch::Sequential: { + return std::is_same_v; + } + case LVQStrategyDispatch::Turbo: { + return TurboLike; + } + } + throw ANNEXCEPTION("Could not match strategy!"); +} +} +template +int64_t overload_match_strategy(LVQStrategyDispatch strategy) { + constexpr bool is_sequential = std::is_same_v; + constexpr bool is_turbo = lvq::TurboLike; + + switch (strategy) { + // If sequential is requested - we can only match sequential. + case LVQStrategyDispatch::Sequential: { + return is_sequential ? lib::perfect_match : lib::invalid_match; + } + // If turbo is requested - we can only match turbo. + case LVQStrategyDispatch::Turbo: { + return is_turbo ? lib::perfect_match : lib::invalid_match; + } + case LVQStrategyDispatch::Auto: { + // Preference: + // (1) Turbo + // (2) Sequential + return is_turbo ? 0 : 1; + } + } + throw ANNEXCEPTION("Unreachable!"); +} + +struct Matcher { + // Load a matcher for either one or two level datasets. + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + if (schema == one_level_serialization_schema && version == one_level_save_version) { + return true; + } + if (schema == two_level_serialization_schema && version == two_level_save_version) { + return true; + } + return false; + } + + static Matcher load(const lib::ContextFreeLoadTable& table) { + auto schema = table.schema(); + auto primary_summary = lib::load_at(table, "primary"); + if (schema == one_level_serialization_schema) { + return Matcher{ + .primary = primary_summary.bits, + .residual = 0, + .dims = primary_summary.dims}; + } + if (schema == two_level_serialization_schema) { + auto residual_summary = lib::load_at(table, "residual"); + return Matcher{ + .primary = primary_summary.bits, + .residual = residual_summary.bits, + .dims = primary_summary.dims}; + } + throw ANNEXCEPTION( + "Unreachable reached with schema and version ({}, {})!", + table.schema(), + table.version() + ); + } + + static lib::TryLoadResult try_load(const lib::ContextFreeLoadTable& table) { + // The saving and loading framework will check schema compatibility before + // calling try-load. + // + // In that case, the logic behind `try_load` and `load` are the same. + // Note that `load` will throw if sub-keys do not match, but that is okay + // because mismatching sub-keys means we have an invalid schema. + return load(table); + } + + constexpr bool friend operator==(const Matcher&, const Matcher&) = default; + + ///// Members + size_t primary; + size_t residual; + size_t dims; +}; + +// Compatibility ranking for LVQ +template +int64_t overload_score(size_t p, size_t r, size_t e, LVQStrategyDispatch strategy) { + // Reject easy matches. + if (p != Primary || r != Residual) { + return lib::invalid_match; + } + + // Check static dimensionality. + auto extent_match = + lib::dispatch_match>(lib::ExtentArg{e}); + + // If the extent match fails - abort immediately. + if (extent_match < 0) { + return lib::invalid_match; + } + + // We know dimensionality matches, now we have to try to match strategy. + auto strategy_match = overload_match_strategy(strategy); + if (strategy_match < 0) { + return lib::invalid_match; + } + + // Prioritize matching dimensionality over better strategies. + // Dispatch matching prefers lower return values over larger return values. + // + // By multiplying the `extent_match`, we enter a regime where better extent + // matches always have precedence over strategy matches. + constexpr size_t extent_multiplier = 1000; + return strategy_match + extent_multiplier * extent_match; +} + +template +int64_t overload_score(Matcher matcher, LVQStrategyDispatch strategy) { + return overload_score( + matcher.primary, matcher.residual, matcher.dims, strategy + ); +} + + +template > struct ProtoLVQLoader { + public: + // Constructors + ProtoLVQLoader() = default; + + // TODO: Propagate allocator request. + explicit ProtoLVQLoader( + const UnspecializedVectorDataLoader& datafile, + size_t primary, + size_t residual, + size_t alignment = 0, + LVQStrategyDispatch strategy = LVQStrategyDispatch::Auto + ) + : source_{std::in_place_type_t(), datafile.path_, datafile.type_} + , primary_{primary} + , residual_{residual} + , dims_{datafile.dims_} + , alignment_{alignment} + , strategy_{strategy} + , allocator_{datafile.allocator_} {} + + explicit ProtoLVQLoader( + Reload reloader, + size_t alignment, + LVQStrategyDispatch strategy = LVQStrategyDispatch::Auto, + const Alloc& allocator = {} + ) + : source_{std::move(reloader)} + , primary_{0} + , residual_{0} + , dims_{0} + , alignment_{alignment} + , strategy_{strategy} + , allocator_{allocator} { + const auto& directory = std::get(source_).directory; + auto result = lib::try_load_from_disk(directory); + if (!result) { + throw ANNEXCEPTION( + "Cannot determine primary, residual, and dimensions " + "from data source {}. " + "Code {}!", + directory, + static_cast(result.error()) + ); + } + const auto& match = result.value(); + primary_ = match.primary; + residual_ = match.residual; + dims_ = match.dims; + } + + template < + size_t Primary, + size_t Residual, + size_t Extent, + LVQPackingStrategy Strategy, + typename F = std::identity> + LVQLoader< + Primary, + Residual, + Extent, + Strategy, + std::decay_t>> + refine(lib::Val, F&& f = std::identity()) const { + using ARet = std::decay_t>; + // Make sure the pre-set values are correct. + if constexpr (Extent != Dynamic) { + if (Extent != dims_) { + throw ANNEXCEPTION("Invalid specialization!"); + } + } + if (Primary != primary_ || Residual != residual_) { + throw ANNEXCEPTION("Encoding bits mismatched!"); + } + if (!detail::is_compatible(strategy_)) { + throw ANNEXCEPTION("Trying to dispatch to an inappropriate strategy!"); + } + + return LVQLoader( + source_, alignment_, f(allocator_) + ); + } + + public: + SourceTypes source_; + size_t primary_; + size_t residual_; + size_t dims_; + size_t alignment_; + LVQStrategyDispatch strategy_; + Alloc allocator_; +}; + +template < + size_t Primary, + size_t Residual, + size_t Extent, + LVQPackingStrategy Strategy, + typename Alloc> +struct LVQLoader { + public: + using loaded_type = LVQDataset; + + explicit LVQLoader(SourceTypes source, size_t alignment, const Alloc& allocator) + : source_{std::move(source)} + , alignment_{alignment} + , allocator_{allocator} {} + + loaded_type load() const { + auto pool = threads::SequentialThreadPool(); + return load(pool); + } + + template + LVQLoader< + Primary, + Residual, + Extent, + Strategy, + std::decay_t>> + rebind_alloc(const F& f) { + return LVQLoader< + Primary, + Residual, + Extent, + Strategy, + std::decay_t>>{ + source_, alignment_, f(allocator_)}; + } + + template loaded_type load(Pool& threadpool) const { + return std::visit( + [&](auto source) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return lib::load_from_disk( + source.directory, alignment_, allocator_ + ); + } else { + return lib::match( + CompressionTs, + source.type, + [&](lib::Type SVS_UNUSED(type)) { + return loaded_type::compress( + data::SimpleData::load(source.path), + threadpool, + alignment_, + allocator_ + ); + } + ); + } + }, + source_ + ); + } + + private: + SourceTypes source_; + size_t alignment_; + Alloc allocator_; +}; + +} // namespace lvq +} // namespace quantization + +// Define dispatch conversion from ProtoLVQLoader to LVQLoader. +template < + size_t Primary, + size_t Residual, + size_t Extent, + quantization::lvq::LVQPackingStrategy Strategy, + typename Alloc> +struct lib::DispatchConverter< + quantization::lvq::ProtoLVQLoader, + quantization::lvq::LVQLoader> { + static int64_t match(const quantization::lvq::ProtoLVQLoader& loader) { + return quantization::lvq::overload_score( + loader.primary_, loader.residual_, loader.dims_, loader.strategy_ + ); + } + + static quantization::lvq::LVQLoader + convert(const quantization::lvq::ProtoLVQLoader& loader) { + return loader.template refine(lib::Val( + )); + } + + static std::string description() { + auto dims = []() { + if constexpr (Extent == Dynamic) { + return "any"; + } else { + return Extent; + } + }(); + + return fmt::format( + "LVQLoader {}x{} ({}) with {} dimensions", + Primary, + Residual, + Strategy::name(), + dims + ); + } +}; +} diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index d600ba97..9bf5e2e9 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -583,6 +583,9 @@ struct Reconstruct { // Customization point for reconstructing vectors. inline constexpr Reconstruct reconstruct_accessor{}; +// TOOD: unify these +#ifdef USE_PROPRIETARY + template SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( svs::tag_t SVS_UNUSED(cpo), @@ -591,4 +594,16 @@ SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( return data::GetDatumAccessor(); } +#else // USE_PROPRIETARY + +template +SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( + svs::tag_t SVS_UNUSED(cpo), + const Data& SVS_UNUSED(dataset) +) { + return data::GetDatumAccessor(); +} + +#endif // USE_PROPRIETARY + } // namespace svs::index::vamana::extensions From 65161bb94bb074c37cc5ee779bd1bd0234043c22 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 26 Mar 2025 18:15:04 -0700 Subject: [PATCH 02/23] add proper licenses --- include/svs/fallback/leanvec_fallback.h | 23 ++++++++++++----------- include/svs/fallback/lvq_fallback.h | 23 ++++++++++++----------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/include/svs/fallback/leanvec_fallback.h b/include/svs/fallback/leanvec_fallback.h index c2366529..1b0941eb 100644 --- a/include/svs/fallback/leanvec_fallback.h +++ b/include/svs/fallback/leanvec_fallback.h @@ -1,16 +1,17 @@ -/** - * Copyright (C) 2023 Intel Corporation +/* + * Copyright 2023 Intel Corporation * - * This software and the related documents are Intel copyrighted materials, - * and your use of them is governed by the express license under which they - * were provided to you ("License"). Unless the License provides otherwise, - * you may not use, modify, copy, publish, distribute, disclose or transmit - * this software or the related documents without Intel's prior written - * permission. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * This software and the related documents are provided as is, with no - * express or implied warranties, other than those that are expressly stated - * in the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once diff --git a/include/svs/fallback/lvq_fallback.h b/include/svs/fallback/lvq_fallback.h index 39f1774b..c4d2fe1e 100644 --- a/include/svs/fallback/lvq_fallback.h +++ b/include/svs/fallback/lvq_fallback.h @@ -1,16 +1,17 @@ -/** - * Copyright (C) 2023 Intel Corporation +/* + * Copyright 2023 Intel Corporation * - * This software and the related documents are Intel copyrighted materials, - * and your use of them is governed by the express license under which they - * were provided to you ("License"). Unless the License provides otherwise, - * you may not use, modify, copy, publish, distribute, disclose or transmit - * this software or the related documents without Intel's prior written - * permission. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * This software and the related documents are provided as is, with no - * express or implied warranties, other than those that are expressly stated - * in the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once From b066f6c49ca703d1ce22587869736ad5c293ea54 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 26 Mar 2025 22:46:11 -0700 Subject: [PATCH 03/23] apply rebase from new changes --- .../python/include/svs/python/dynamic_vamana.h | 3 +++ bindings/python/include/svs/python/vamana.h | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/bindings/python/include/svs/python/dynamic_vamana.h b/bindings/python/include/svs/python/dynamic_vamana.h index 53d95550..7d75a2af 100644 --- a/bindings/python/include/svs/python/dynamic_vamana.h +++ b/bindings/python/include/svs/python/dynamic_vamana.h @@ -72,6 +72,9 @@ template void for_leanvec_specializations(F&& f) { X(DistanceL2, svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic); X(DistanceIP, svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic); + + X(DistanceL2, svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic); + X(DistanceIP, svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic); #undef X } diff --git a/bindings/python/include/svs/python/vamana.h b/bindings/python/include/svs/python/vamana.h index 301dd411..4f7d3071 100644 --- a/bindings/python/include/svs/python/vamana.h +++ b/bindings/python/include/svs/python/vamana.h @@ -101,9 +101,11 @@ template void lvq_specialize_4x0(const F& f) { // Sequential X(DistanceL2, 4, 0, Dynamic, Sequential, true); X(DistanceIP, 4, 0, Dynamic, Sequential, true); + X(DistanceCosineSimilarity, 4, 0, Dynamic, Sequential, true); // Turbo X(DistanceL2, 4, 0, Dynamic, Turbo, true); X(DistanceIP, 4, 0, Dynamic, Turbo, true); + X(DistanceCosineSimilarity, 4, 0, Dynamic, Turbo, true); } template void lvq_specialize_4x4(const F& f) { @@ -113,9 +115,11 @@ template void lvq_specialize_4x4(const F& f) { // Sequential X(DistanceL2, 4, 4, Dynamic, Sequential, true); X(DistanceIP, 4, 4, Dynamic, Sequential, true); + X(DistanceCosineSimilarity, 4, 4, Dynamic, Sequential, true); // Turbo X(DistanceL2, 4, 4, Dynamic, Turbo, true); X(DistanceIP, 4, 4, Dynamic, Turbo, true); + X(DistanceCosineSimilarity, 4, 4, Dynamic, Turbo, true); } template void lvq_specialize_4x8(const F& f) { @@ -125,9 +129,11 @@ template void lvq_specialize_4x8(const F& f) { // Sequential X(DistanceL2, 4, 8, Dynamic, Sequential, true); X(DistanceIP, 4, 8, Dynamic, Sequential, true); + X(DistanceCosineSimilarity, 4, 8, Dynamic, Sequential, true); // Turbo X(DistanceL2, 4, 8, Dynamic, Turbo, true); X(DistanceIP, 4, 8, Dynamic, Turbo, true); + X(DistanceCosineSimilarity, 4, 8, Dynamic, Turbo, true); } template void lvq_specialize_8x0(const F& f) { @@ -137,15 +143,18 @@ template void lvq_specialize_8x0(const F& f) { // Sequential X(DistanceL2, 8, 0, Dynamic, Sequential, true); X(DistanceIP, 8, 0, Dynamic, Sequential, true); + X(DistanceCosineSimilarity, 8, 0, Dynamic, Sequential, true); // Turbo X(DistanceL2, 8, 0, Dynamic, Turbo, true); X(DistanceIP, 8, 0, Dynamic, Turbo, true); + X(DistanceCosineSimilarity, 8, 0, Dynamic, Turbo, true); } template void lvq_specialize_8x8(const F& f) { using Sequential = svs::quantization::lvq::Sequential; X(DistanceL2, 8, 8, Dynamic, Sequential, false); X(DistanceIP, 8, 8, Dynamic, Sequential, false); + X(DistanceCosineSimilarity, 8, 8, Dynamic, Sequential, false); } template void compressed_specializations(F&& f) { @@ -164,25 +173,33 @@ template void compressed_specializations(F&& f) { template void leanvec_specialize_unc_unc(const F& f) { X(float, float, Dynamic, Dynamic, DistanceL2); X(float, float, Dynamic, Dynamic, DistanceIP); + X(float, float, Dynamic, Dynamic, DistanceCosineSimilarity); X(svs::Float16, svs::Float16, Dynamic, Dynamic, DistanceL2); X(svs::Float16, svs::Float16, Dynamic, Dynamic, DistanceIP); + X(svs::Float16, svs::Float16, Dynamic, Dynamic, DistanceCosineSimilarity); } template void leanvec_specialize_lvq_unc(const F& f) { X(svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic, DistanceL2); X(svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic, DistanceIP); + X(svs::leanvec::UsingLVQ<8>, svs::Float16, Dynamic, Dynamic, DistanceCosineSimilarity); } template void leanvec_specialize_lvq_lvq(const F& f) { + // clang-format off X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic, DistanceL2); X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic, DistanceIP); + X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<4>, Dynamic, Dynamic, DistanceCosineSimilarity); X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceL2); X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceIP); + X(svs::leanvec::UsingLVQ<4>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceCosineSimilarity); X(svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceL2); X(svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceIP); + X(svs::leanvec::UsingLVQ<8>, svs::leanvec::UsingLVQ<8>, Dynamic, Dynamic, DistanceCosineSimilarity); + // clang-format on } template void leanvec_specializations(F&& f) { From a246fb7b17cd72fc68920b40252363049461f86e Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 26 Mar 2025 23:28:09 -0700 Subject: [PATCH 04/23] add all extensions --- include/svs/fallback/fallback.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index eb03a01b..8b1d6dbe 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -27,6 +27,10 @@ #include "../../../../include/svs/quantization/lvq/lvq.h" #include "../../../../include/svs/leanvec/leanvec.h" +#include "../../../../include/svs/flat/vamana/lvq.h" +#include "../../../../include/svs/flat/vamana/leanvec.h" +#include "../../../../include/svs/inverted/vamana/lvq.h" +#include "../../../../include/svs/inverted/vamana/leanvec.h" #include "../../../../include/svs/extensions/vamana/lvq.h" #include "../../../../include/svs/extensions/vamana/leanvec.h" From 520b4f855f1a2d77340c911573520bcd8cfc916a Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 27 Mar 2025 07:23:21 -0700 Subject: [PATCH 05/23] fix incorrect path --- include/svs/fallback/fallback.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index 8b1d6dbe..1b55baad 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -27,10 +27,10 @@ #include "../../../../include/svs/quantization/lvq/lvq.h" #include "../../../../include/svs/leanvec/leanvec.h" -#include "../../../../include/svs/flat/vamana/lvq.h" -#include "../../../../include/svs/flat/vamana/leanvec.h" -#include "../../../../include/svs/inverted/vamana/lvq.h" -#include "../../../../include/svs/inverted/vamana/leanvec.h" +#include "../../../../include/svs/extensions/flat/lvq.h" +#include "../../../../include/svs/extensions/flat/leanvec.h" +#include "../../../../include/svs/extensions/inverted/lvq.h" +#include "../../../../include/svs/extensions/inverted/leanvec.h" #include "../../../../include/svs/extensions/vamana/lvq.h" #include "../../../../include/svs/extensions/vamana/leanvec.h" From 703a909e4d09cb0813c0c30f27828715be79d491 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 27 Mar 2025 07:57:20 -0700 Subject: [PATCH 06/23] remove nonexistent file --- include/svs/fallback/fallback.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index 1b55baad..a9ce84e8 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -30,7 +30,6 @@ #include "../../../../include/svs/extensions/flat/lvq.h" #include "../../../../include/svs/extensions/flat/leanvec.h" #include "../../../../include/svs/extensions/inverted/lvq.h" -#include "../../../../include/svs/extensions/inverted/leanvec.h" #include "../../../../include/svs/extensions/vamana/lvq.h" #include "../../../../include/svs/extensions/vamana/leanvec.h" From 2398f13ded78e4c1fe197f164510c3e5deb66c58 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 27 Mar 2025 23:01:16 -0700 Subject: [PATCH 07/23] split into common/fallback/concept and transfer more pybinds --- .../python/include/svs/python/conversion.h | 24 ++ bindings/python/include/svs/python/dispatch.h | 2 - bindings/python/src/conversion.cpp | 168 +++++++++ bindings/python/src/dynamic_vamana.cpp | 2 - bindings/python/src/flat.cpp | 36 +- bindings/python/src/python_bindings.cpp | 6 + bindings/python/src/vamana.cpp | 2 - include/svs/fallback/fallback.h | 17 +- include/svs/leanvec/leanvec_common.h | 42 +++ .../leanvec_concept.h} | 199 ++--------- include/svs/leanvec/leanvec_fallback.h | 174 +++++++++ include/svs/quantization/lvq/lvq_common.h | 141 ++++++++ .../lvq/lvq_concept.h} | 333 ++++-------------- include/svs/quantization/lvq/lvq_fallback.h | 173 +++++++++ 14 files changed, 875 insertions(+), 444 deletions(-) create mode 100644 bindings/python/include/svs/python/conversion.h create mode 100644 bindings/python/src/conversion.cpp create mode 100644 include/svs/leanvec/leanvec_common.h rename include/svs/{fallback/leanvec_fallback.h => leanvec/leanvec_concept.h} (72%) create mode 100644 include/svs/leanvec/leanvec_fallback.h create mode 100644 include/svs/quantization/lvq/lvq_common.h rename include/svs/{fallback/lvq_fallback.h => quantization/lvq/lvq_concept.h} (60%) create mode 100644 include/svs/quantization/lvq/lvq_fallback.h diff --git a/bindings/python/include/svs/python/conversion.h b/bindings/python/include/svs/python/conversion.h new file mode 100644 index 00000000..ecaf5cec --- /dev/null +++ b/bindings/python/include/svs/python/conversion.h @@ -0,0 +1,24 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// pybind +#include + +namespace svs::python::conversion { +void wrap(pybind11::module& m); +} // namespace svs::python::conversion diff --git a/bindings/python/include/svs/python/dispatch.h b/bindings/python/include/svs/python/dispatch.h index e90b5a8d..6dff43ea 100644 --- a/bindings/python/include/svs/python/dispatch.h +++ b/bindings/python/include/svs/python/dispatch.h @@ -24,8 +24,6 @@ #include "svs/lib/dispatcher.h" #include "svs/lib/saveload.h" -#include "svs/fallback/fallback.h" - // Dispatch rule for serialized objects to a VectorDataLoader. template struct svs::lib::DispatchConverter< diff --git a/bindings/python/src/conversion.cpp b/bindings/python/src/conversion.cpp new file mode 100644 index 00000000..7c641f9a --- /dev/null +++ b/bindings/python/src/conversion.cpp @@ -0,0 +1,168 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// svs python bindings +#include "svs/python/conversion.h" +#include "svs/python/common.h" +#include "svs/python/core.h" + +// svs +#include "svs/quantization/lvq/lvq_concept.h" + +// pybind +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/stl/filesystem.h" + +// stl +#include +#include + +namespace lvq = svs::quantization::lvq; +namespace py = pybind11; + +namespace svs::python { +namespace { + +template void register_specializations(F&& f) { + // Pattern: Primary, Residual, Strategy + f.template operator()<4, 0, lvq::Sequential>(); + f.template operator()<8, 0, lvq::Sequential>(); + f.template operator()<4, 4, lvq::Sequential>(); + f.template operator()<4, 8, lvq::Sequential>(); + f.template operator()<8, 8, lvq::Sequential>(); +} + +template +void compress( + lvq::LVQLoader SVS_UNUSED(dispatch + ), + const std::filesystem::path& data_path, + const std::filesystem::path& centroid_path, + const std::filesystem::path& assignment_path, + const std::filesystem::path& save_path, + size_t num_threads +) { + using dataset_t = + svs::quantization::lvq::LVQDataset; + + auto data = svs::VectorDataLoader(data_path).load(); + auto centroids = svs::VectorDataLoader(centroid_path).load(); + + auto assignments = std::vector(data.size()); + { + auto stream = svs::lib::open_read(assignment_path); + svs::lib::read_binary(stream, assignments); + } + + // Allocate the storage dataset and set copy over the centroids. + auto dst = dataset_t(data.size(), svs::lib::MaybeStatic(data.dimensions())); + dst.reproducibility_set_centroids(centroids.cview()); + + // Compress the dataset into the compressed destination. + auto pool = svs::threads::DefaultThreadPool(num_threads); + svs::threads::parallel_for( + pool, + svs::threads::StaticPartition(data.size()), + [&](auto is, auto SVS_UNUSED(tid)) { + for (auto i : is) { + dst.set_datum(i, data.get_datum(i), assignments.at(i)); + } + } + ); + + // Save the result. + svs::lib::save_to_disk(dst, save_path); +} + +struct Compress { + void operator()( + const LVQ& source, + const std::filesystem::path& data_path, + const std::filesystem::path& centroid_path, + const std::filesystem::path& assignment_path, + const std::filesystem::path& save_path, + size_t num_threads + ) { + auto dispatcher = svs::lib::Dispatcher< + void, + LVQ, + const std::filesystem::path&, + const std::filesystem::path&, + const std::filesystem::path&, + const std::filesystem::path&, + size_t>(); + + register_specializations([&]() { + dispatcher.register_target(&compress); + }); + + dispatcher.invoke( + source, data_path, centroid_path, assignment_path, save_path, num_threads + ); + } +}; + +template +void decompress( + lvq::LVQLoader loader, + const std::filesystem::path& save_path +) { + auto dataset = loader.load(); + auto dst = svs::data::SimpleData(dataset.size(), dataset.dimensions()); + + auto decompressor = dataset.decompressor(); + for (size_t i = 0, imax = dataset.size(); i < imax; ++i) { + dst.set_datum(i, decompressor(dataset.get_datum(i))); + } + svs::lib::save_to_disk(dst, save_path); +} + +struct Decompress { + void operator()(const LVQ& loader, const std::filesystem::path& save_path) { + auto dispatcher = svs::lib::Dispatcher(); + register_specializations([&]() { + dispatcher.register_target(&decompress); + }); + dispatcher.invoke(loader, save_path); + } +}; + +} // namespace + +namespace conversion { + +void wrap(py::module& m) { + auto sub = m.def_submodule( + "reproducibility", "Compatibility methods to reproduce paper results." + ); + + sub.def( + "compress", + Compress(), + py::arg("source"), + py::arg("data_path"), + py::arg("centroid_path"), + py::arg("assignment_path"), + py::arg("save_path"), + py::arg("num_threads") = 1 + ); + + sub.def("decompress", Decompress(), py::arg("source"), py::arg("save_path")); +} + +} // namespace conversion +} // namespace svs::python diff --git a/bindings/python/src/dynamic_vamana.cpp b/bindings/python/src/dynamic_vamana.cpp index a0859fed..4b7bad32 100644 --- a/bindings/python/src/dynamic_vamana.cpp +++ b/bindings/python/src/dynamic_vamana.cpp @@ -26,8 +26,6 @@ #include "svs/lib/dispatcher.h" #include "svs/orchestrators/dynamic_vamana.h" -#include "svs/fallback/fallback.h" - // pybind #include #include diff --git a/bindings/python/src/flat.cpp b/bindings/python/src/flat.cpp index 378fbb37..182f9644 100644 --- a/bindings/python/src/flat.cpp +++ b/bindings/python/src/flat.cpp @@ -21,6 +21,7 @@ #include "svs/python/manager.h" // svs +#include "svs/quantization/lvq/lvq_concept.h" #include "svs/lib/datatype.h" #include "svs/lib/dispatcher.h" #include "svs/orchestrators/exhaustive.h" @@ -39,6 +40,7 @@ ///// namespace py = pybind11; +namespace lvq = svs::quantization::lvq; namespace svs::python::flat { template void for_standard_specializations(F&& f) { @@ -54,9 +56,22 @@ template void for_standard_specializations(F&& f) { #undef X } +// Compressed search specializations. +template void for_lvq_specializations(F&& f) { +#define X(Dist, Primary, Residual, N) f.template operator()() + // Pattern: + // DistanceType, Primary, Residual, Dimensionality + X(DistanceL2, 4, 4, Dynamic); + X(DistanceL2, 8, 0, Dynamic); + + X(DistanceIP, 4, 4, Dynamic); + X(DistanceIP, 8, 0, Dynamic); +#undef X +} + namespace detail { -using FlatSourceTypes = std::variant; +using FlatSourceTypes = std::variant; template svs::Flat assemble_uncompressed( @@ -67,6 +82,15 @@ svs::Flat assemble_uncompressed( return svs::Flat::assemble(std::move(datafile), distance_type, num_threads); } +template +svs::Flat assemble_lvq( + lvq::LVQLoader loader, + D distance, + size_t num_threads +) { + return svs::Flat::assemble(std::move(loader), std::move(distance), num_threads); +} + using AssemblyDispatcher = svs::lib::Dispatcher; @@ -78,6 +102,12 @@ AssemblyDispatcher assembly_dispatcher() { dispatcher.register_target(svs::lib::dispatcher_build_docs, method); }); + // LVQ instantiations. + for_lvq_specializations([&dispatcher]() { + auto method = &assemble_lvq; + dispatcher.register_target(svs::lib::dispatcher_build_docs, method); + }); + return dispatcher; } ///// @@ -160,8 +190,8 @@ be instantiated based on their applicability to the particular problem instance. The arguments upon which specialization is conducted are: -* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type - and number of dimensions. +* `data_loader`: Both kind (type of loader) and inner aspects of the loader like data type, + quantization type, and number of dimensions. * `distance`: The distance measure being used. Specializations compiled into the binary are listed below. diff --git a/bindings/python/src/python_bindings.cpp b/bindings/python/src/python_bindings.cpp index e1ac92b6..88a414bd 100644 --- a/bindings/python/src/python_bindings.cpp +++ b/bindings/python/src/python_bindings.cpp @@ -17,6 +17,7 @@ // Dependencies within the python SVS bindings directory. #include "svs/python/allocator.h" #include "svs/python/common.h" +#include "svs/python/conversion.h" #include "svs/python/core.h" #include "svs/python/dynamic_vamana.h" #include "svs/python/flat.h" @@ -202,6 +203,11 @@ Convert the `fvecs` file on disk with 32-bit floating point entries to a `fvecs` // Core data types svs::python::core::wrap(m); +#ifdef USE_PROPRIETARY + // Dataset conversion. + svs::python::conversion::wrap(m); +#endif + // Intel(R) MKL m.def( "have_mkl", diff --git a/bindings/python/src/vamana.cpp b/bindings/python/src/vamana.cpp index 4925eedd..b09203ca 100644 --- a/bindings/python/src/vamana.cpp +++ b/bindings/python/src/vamana.cpp @@ -32,8 +32,6 @@ #include "svs/lib/meta.h" #include "svs/orchestrators/vamana.h" -#include "svs/fallback/fallback.h" - // pybind #include #include diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index a9ce84e8..35c571fa 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -17,20 +17,11 @@ #pragma once #include "svs/fallback/fallback_mode.h" +#include "svs/quantization/lvq/lvq_concept.h" +#include "svs/leanvec/leanvec_concept.h" -#ifndef USE_PROPRIETARY +#ifdef USE_PROPRIETARY -#include "svs/fallback/lvq_fallback.h" -#include "svs/fallback/leanvec_fallback.h" - -#else // USE_PROPRIETARY - -#include "../../../../include/svs/quantization/lvq/lvq.h" -#include "../../../../include/svs/leanvec/leanvec.h" -#include "../../../../include/svs/extensions/flat/lvq.h" -#include "../../../../include/svs/extensions/flat/leanvec.h" -#include "../../../../include/svs/extensions/inverted/lvq.h" -#include "../../../../include/svs/extensions/vamana/lvq.h" -#include "../../../../include/svs/extensions/vamana/leanvec.h" +#include "../../../../include/svs/fallback/fallback2.h" #endif // USE_PROPRIETARY diff --git a/include/svs/leanvec/leanvec_common.h b/include/svs/leanvec/leanvec_common.h new file mode 100644 index 00000000..3a93a034 --- /dev/null +++ b/include/svs/leanvec/leanvec_common.h @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace svs { +namespace leanvec { + +// Sentinel type to select an LVQ dataset as either the primary or secondary +// dataset for `LeanVec`. +template struct UsingLVQ {}; + +// Hoist out schemas for reuse while auto-loading. +inline constexpr std::string_view lean_dataset_schema = "leanvec_dataset"; +inline constexpr lib::Version lean_dataset_save_version = lib::Version(0, 0, 0); + +namespace detail { + +template inline constexpr bool is_using_lvq_tag_v = false; +template inline constexpr bool is_using_lvq_tag_v> = true; + +} + +// Compatible type parameters for LeanDatasets +template +concept LeanCompatible = has_datatype_v || detail::is_using_lvq_tag_v; + +} +} diff --git a/include/svs/fallback/leanvec_fallback.h b/include/svs/leanvec/leanvec_concept.h similarity index 72% rename from include/svs/fallback/leanvec_fallback.h rename to include/svs/leanvec/leanvec_concept.h index 1b0941eb..69e3eb07 100644 --- a/include/svs/fallback/leanvec_fallback.h +++ b/include/svs/leanvec/leanvec_concept.h @@ -16,170 +16,30 @@ #pragma once -#include "svs/fallback/lvq_fallback.h" -#include "svs/fallback/fallback_mode.h" +#include "svs/quantization/lvq/lvq_concept.h" -namespace fallback = svs::fallback; +#ifndef USE_PROPRIETARY -namespace svs { -namespace leanvec { - -template struct UsingLVQ {}; - -template struct LeanVecMatrices { - // temporary (?) additions - public: - using leanvec_matrix_type = data::SimpleData; - - LeanVecMatrices() = default; - LeanVecMatrices(leanvec_matrix_type data_matrix, leanvec_matrix_type query_matrix) - : data_matrix_{std::move(data_matrix)} - , query_matrix_{std::move(query_matrix)} { - // Check that the size and dimensionality of both the matrices should be same - if (data_matrix_.size() != query_matrix_.size()) { - throw ANNEXCEPTION("Mismatched data and query matrix sizes!"); - } - if (data_matrix_.dimensions() != query_matrix_.dimensions()) { - throw ANNEXCEPTION("Mismatched data and query matrix dimensions!"); - } - } - private: - leanvec_matrix_type data_matrix_; - leanvec_matrix_type query_matrix_; -}; - -enum class LeanVecKind { float32, float16, lvq8, lvq4 }; - -// is this necessary or duplicate of LVQ? -namespace detail { -template inline constexpr bool is_blocked = false; -template inline constexpr bool is_blocked> = true; - -template > struct select_rebind_allocator { - using type = lib::rebind_allocator_t; -}; -template struct select_rebind_allocator { - using base_allocator = typename A::allocator_type; - using rebind_base_allocator = lib::rebind_allocator_t; - using type = data::Blocked; -}; -template -using select_rebind_allocator_t = typename select_rebind_allocator::type; -} - -template < - typename T1, - typename T2, - size_t LeanVecDims, - size_t Extent, - typename Alloc = lib::Allocator> -class LeanDataset { - public: - using allocator_type = detail::select_rebind_allocator_t; - private: - data::SimpleData primary_; - public: - static constexpr bool is_resizeable = detail::is_blocked; - using leanvec_matrices_type = LeanVecMatrices; - using const_value_type = typename data::SimpleData::const_value_type; - using element_type = float; - using value_type = const_value_type; - using primary_type = data::SimpleData; - - LeanDataset(primary_type primary): primary_{std::move(primary)} { - if (fallback::get_mode() == fallback::FallbackMode::Error) { - throw fallback::UnsupportedHardwareError(); - } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { - fmt::print(fallback::fallback_warning); - } - } - - size_t size() const { return primary_.size(); } - size_t dimensions() const { return primary_.dimensions(); } - const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } - void prefetch(size_t i) const { primary_.prefetch(i); } - template void set_datum(size_t i, std::span datum) { - primary_.set_datum(i, datum); - } - - void resize(size_t new_size) - requires is_resizeable - { - primary_.resize(new_size); - } - template - requires is_resizeable - void - compact(std::span new_to_old, Pool& threadpool, size_t batchsize = 1'000'000) { - primary_.compact(new_to_old, threadpool, batchsize); - } - - template - static LeanDataset reduce( - const Dataset& data, - size_t num_threads = 1, - size_t alignment = 0, - lib::MaybeStatic leanvec_dims = {}, - const Alloc& allocator = {} - ) { - return reduce(data, std::nullopt, num_threads, alignment, leanvec_dims, allocator); - } +#include "svs/leanvec/leanvec_fallback.h" - template - static LeanDataset reduce( - const Dataset& data, - std::optional matrices, - size_t num_threads = 1, - size_t alignment = 0, - lib::MaybeStatic leanvec_dims = {}, - const Alloc& allocator = {} - ) { - auto pool = threads::NativeThreadPool{num_threads}; - return reduce(data, std::move(matrices), pool, alignment, leanvec_dims, allocator); - } +#else // USE_PROPRIETARY - template - static LeanDataset reduce( - const Dataset& data, - std::optional SVS_UNUSED(matrices), - Pool& SVS_UNUSED(threadpool), - size_t SVS_UNUSED(alignment) = 0, - lib::MaybeStatic SVS_UNUSED(leanvec_dims) = {}, - const Alloc& allocator = {} - ) { - primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; - svs::data::copy(data, primary); - return LeanDataset{primary}; - } +#include "../../../../include/svs/leanvec/leanvec.h" - static constexpr lib::Version save_version = lib::Version(0, 0, 0); - static constexpr std::string_view serialization_schema = "leanvec_fallback"; - lib::SaveTable save(const lib::SaveContext& ctx) const { - return lib::SaveTable( - serialization_schema, - save_version, - {SVS_LIST_SAVE_(primary, ctx)} - ); - } +#endif // USE_PROPRIETARY - static LeanDataset load( - const lib::LoadTable& table, - size_t SVS_UNUSED(alignment) = 0, - const Alloc& allocator = {} - ) { - return LeanDataset{SVS_LOAD_MEMBER_AT_(table, primary, allocator)}; - } -}; +namespace svs { +namespace leanvec { -struct Reload { - public: - explicit Reload(const std::filesystem::path& directory) - : directory{directory} {} +///// +///// Load Helpers +///// - public: - std::filesystem::path directory; -}; +// Types to use for leanvec. inline constexpr lib::Types LeanVecSourceTypes{}; + +// LeanVec based loaders can either perform LeanVec conversion online, or reload +// a previously saved LeanVec dataset. struct OnlineLeanVec { public: explicit OnlineLeanVec(const std::filesystem::path& path, DataType type) @@ -189,17 +49,33 @@ struct OnlineLeanVec { throw ANNEXCEPTION("Invalid type!"); } } + + // Members public: std::filesystem::path path; DataType type; }; + +struct Reload { + public: + explicit Reload(const std::filesystem::path& directory) + : directory{directory} {} + + // Members + public: + std::filesystem::path directory; +}; + +// The various ways we can instantiate LeanVec-based datasets.. using SourceTypes = std::variant; -inline constexpr std::string_view lean_dataset_schema = "leanvec_dataset"; -inline constexpr lib::Version lean_dataset_save_version = lib::Version(0, 0, 0); + +/// A type used to request a specific specialization of LeanVec at runtime. +/// Used for dispatching. +enum class LeanVecKind { float32, float16, lvq8, lvq4 }; namespace detail { -template struct LeanVecPicker; +template struct LeanVecPicker; template <> struct LeanVecPicker { static constexpr LeanVecKind value = LeanVecKind::float32; @@ -218,7 +94,7 @@ template <> struct LeanVecPicker> { template inline constexpr LeanVecKind leanvec_kind_v = detail::LeanVecPicker::value; - + // LeanDataset Matcher struct Matcher { private: @@ -333,7 +209,7 @@ struct Matcher { }; // Overload Matching Rules -template +template int64_t overload_score( LeanVecKind primary, size_t primary_dims, LeanVecKind secondary, size_t secondary_dims ) { @@ -370,7 +246,7 @@ int64_t overload_score( return extent_match + leanvec_dims_match; } -template +template int64_t overload_score(const Matcher& matcher) { return overload_score( matcher.primary_kind, @@ -380,6 +256,7 @@ int64_t overload_score(const Matcher& matcher) { ); } +// Forward Declaration. template struct LeanVecLoader; diff --git a/include/svs/leanvec/leanvec_fallback.h b/include/svs/leanvec/leanvec_fallback.h new file mode 100644 index 00000000..362cfe64 --- /dev/null +++ b/include/svs/leanvec/leanvec_fallback.h @@ -0,0 +1,174 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/quantization/lvq/lvq_fallback.h" +#include "svs/fallback/fallback_mode.h" +#include "svs/leanvec/leanvec_common.h" + +// #include leanvec_common.h + +namespace fallback = svs::fallback; + +namespace svs { +namespace leanvec { + +template struct LeanVecMatrices { + // temporary (?) additions + public: + using leanvec_matrix_type = data::SimpleData; + + LeanVecMatrices() = default; + LeanVecMatrices(leanvec_matrix_type data_matrix, leanvec_matrix_type query_matrix) + : data_matrix_{std::move(data_matrix)} + , query_matrix_{std::move(query_matrix)} { + // Check that the size and dimensionality of both the matrices should be same + if (data_matrix_.size() != query_matrix_.size()) { + throw ANNEXCEPTION("Mismatched data and query matrix sizes!"); + } + if (data_matrix_.dimensions() != query_matrix_.dimensions()) { + throw ANNEXCEPTION("Mismatched data and query matrix dimensions!"); + } + } + private: + leanvec_matrix_type data_matrix_; + leanvec_matrix_type query_matrix_; +}; + +// is this necessary or duplicate of LVQ? +namespace detail { +template inline constexpr bool is_blocked = false; +template inline constexpr bool is_blocked> = true; + +template > struct select_rebind_allocator { + using type = lib::rebind_allocator_t; +}; +template struct select_rebind_allocator { + using base_allocator = typename A::allocator_type; + using rebind_base_allocator = lib::rebind_allocator_t; + using type = data::Blocked; +}; +template +using select_rebind_allocator_t = typename select_rebind_allocator::type; +} + +template < + typename T1, + typename T2, + size_t LeanVecDims, + size_t Extent, + typename Alloc = lib::Allocator> +class LeanDataset { + public: + using allocator_type = detail::select_rebind_allocator_t; + private: + data::SimpleData primary_; + public: + static constexpr bool is_resizeable = detail::is_blocked; + using leanvec_matrices_type = LeanVecMatrices; + using const_value_type = typename data::SimpleData::const_value_type; + using element_type = float; + using value_type = const_value_type; + using primary_type = data::SimpleData; + + LeanDataset(primary_type primary): primary_{std::move(primary)} { + if (fallback::get_mode() == fallback::FallbackMode::Error) { + throw fallback::UnsupportedHardwareError(); + } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { + fmt::print(fallback::fallback_warning); + } + } + + size_t size() const { return primary_.size(); } + size_t dimensions() const { return primary_.dimensions(); } + const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } + void prefetch(size_t i) const { primary_.prefetch(i); } + template void set_datum(size_t i, std::span datum) { + primary_.set_datum(i, datum); + } + + void resize(size_t new_size) + requires is_resizeable + { + primary_.resize(new_size); + } + template + requires is_resizeable + void + compact(std::span new_to_old, Pool& threadpool, size_t batchsize = 1'000'000) { + primary_.compact(new_to_old, threadpool, batchsize); + } + + template + static LeanDataset reduce( + const Dataset& data, + size_t num_threads = 1, + size_t alignment = 0, + lib::MaybeStatic leanvec_dims = {}, + const Alloc& allocator = {} + ) { + return reduce(data, std::nullopt, num_threads, alignment, leanvec_dims, allocator); + } + + template + static LeanDataset reduce( + const Dataset& data, + std::optional matrices, + size_t num_threads = 1, + size_t alignment = 0, + lib::MaybeStatic leanvec_dims = {}, + const Alloc& allocator = {} + ) { + auto pool = threads::NativeThreadPool{num_threads}; + return reduce(data, std::move(matrices), pool, alignment, leanvec_dims, allocator); + } + + template + static LeanDataset reduce( + const Dataset& data, + std::optional SVS_UNUSED(matrices), + Pool& SVS_UNUSED(threadpool), + size_t SVS_UNUSED(alignment) = 0, + lib::MaybeStatic SVS_UNUSED(leanvec_dims) = {}, + const Alloc& allocator = {} + ) { + primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; + svs::data::copy(data, primary); + return LeanDataset{primary}; + } + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "leanvec_fallback"; + lib::SaveTable save(const lib::SaveContext& ctx) const { + return lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(primary, ctx)} + ); + } + + static LeanDataset load( + const lib::LoadTable& table, + size_t SVS_UNUSED(alignment) = 0, + const Alloc& allocator = {} + ) { + return LeanDataset{SVS_LOAD_MEMBER_AT_(table, primary, allocator)}; + } +}; + +} // namespace leanvec +} // namespace svs diff --git a/include/svs/quantization/lvq/lvq_common.h b/include/svs/quantization/lvq/lvq_common.h new file mode 100644 index 00000000..10d44502 --- /dev/null +++ b/include/svs/quantization/lvq/lvq_common.h @@ -0,0 +1,141 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace svs { +namespace quantization { +namespace lvq { + +namespace detail { + +// Trait to determine if an allocator is blocked or not. +// Used to SFINAE away resizing methods if the allocator is not blocked. +template inline constexpr bool is_blocked = false; +template inline constexpr bool is_blocked> = true; + +} // namespace detail + +enum class LVQStrategyDispatch { + Auto, // Choose between sequential and turbo. + Sequential, // Force Sequential + Turbo // Force Turbo +}; + +/// +/// Place-holder to indicate that a given direct compression stores its values as +/// signed integers (taking positive and negative values in accordance with a two-s +/// complement encoding). +/// +struct Signed { + static constexpr std::string_view name = "signed"; +}; + +/// +/// Place-holder to indicate that a given direct compression stores its values as +/// unsigned integers. +/// +struct Unsigned { + static constexpr std::string_view name = "unsigned"; +}; + +// Schemas are independent of most type parameters. +// Hoist them as stand-alone variables to they are accessible to the auto load +// matchers as well. +inline constexpr std::string_view one_level_serialization_schema = "one_level_lvq_dataset"; +inline constexpr lib::Version one_level_save_version = lib::Version(0, 0, 2); +inline constexpr std::string_view two_level_serialization_schema = "two_level_lvq_dataset"; +inline constexpr lib::Version two_level_save_version = lib::Version(0, 0, 3); + +enum class DatasetSchema { Compressed, ScaledBiased }; +/// +/// Support for deduction. +/// +inline constexpr std::string_view get_schema(DatasetSchema kind) { + switch (kind) { + using enum DatasetSchema; + case Compressed: { + return "lvq_compressed_dataset"; + } + case ScaledBiased: { + return "lvq_with_scaling_constants"; + } + } + throw ANNEXCEPTION("Invalid schema!"); +} + +inline constexpr lib::Version get_current_version(DatasetSchema kind) { + switch (kind) { + using enum DatasetSchema; + case Compressed: { + return lib::Version(0, 0, 0); + } + case ScaledBiased: { + return lib::Version(0, 0, 3); + } + } + throw ANNEXCEPTION("Invalid schema!"); +} + +struct DatasetSummary { + static bool check_load_compatibility(std::string_view schema, lib::Version version) { + using enum DatasetSchema; + if (schema == get_schema(Compressed) && + version == get_current_version(Compressed)) { + return true; + } + if (schema == get_schema(ScaledBiased) && + version == get_current_version(ScaledBiased)) { + return true; + } + return false; + } + + static DatasetSummary load(const lib::ContextFreeLoadTable& table) { + using enum DatasetSchema; + auto schema = table.schema(); + if (schema == get_schema(Compressed)) { + return DatasetSummary{ + .kind = Compressed, + .is_signed = + (lib::load_at(table, "sign") == lvq::Signed::name), + .dims = lib::load_at(table, "ndims"), + .bits = lib::load_at(table, "bits")}; + } + if (schema == get_schema(ScaledBiased)) { + return DatasetSummary{ + .kind = ScaledBiased, + .is_signed = false, // ScaledBiased always uses unsigned codes. + .dims = lib::load_at(table, "logical_dimensions"), + .bits = lib::load_at(table, "bits")}; + } + throw ANNEXCEPTION("Invalid table schema {}!", schema); + } + + ///// Members + // The kind of the leaf dataset. + DatasetSchema kind; + // Whether each LVQ element is signed. + bool is_signed; + // The logical number of dimensions in the dataset. + size_t dims; + // The number of bits used for compression. + size_t bits; +}; + +} // namespace lvq +} // namespace quantization +} // namespace svs diff --git a/include/svs/fallback/lvq_fallback.h b/include/svs/quantization/lvq/lvq_concept.h similarity index 60% rename from include/svs/fallback/lvq_fallback.h rename to include/svs/quantization/lvq/lvq_concept.h index c4d2fe1e..4afa0169 100644 --- a/include/svs/fallback/lvq_fallback.h +++ b/include/svs/quantization/lvq/lvq_concept.h @@ -16,167 +16,72 @@ #pragma once -#include "svs/core/data/simple.h" -#include "svs/lib/threads.h" -#include "svs/lib/saveload/save.h" -#include "svs/fallback/fallback_mode.h" +#ifndef USE_PROPRIETARY -namespace fallback = svs::fallback; +#include "svs/quantization/lvq/lvq_fallback.h" -namespace svs { -namespace quantization { -namespace lvq { - -// TODO: should these be fully defined? -struct Sequential { - static constexpr std::string_view name() { return "sequential"; } -}; -template struct Turbo { - static constexpr std::string name() { - return fmt::format("turbo<{}x{}>", Lanes, ElementsPerLane); - } -}; - -namespace detail { - -// Trait to identify and dispatch based on the Turbo class itself. -template inline constexpr bool is_turbo_like_v = false; -template inline constexpr bool is_lvq_packing_strategy_v = false; +#else // USE_PROPRIETARY -template -inline constexpr bool is_turbo_like_v> = true; +#include "../../../../../include/svs/quantization/lvq/lvq.h" -template <> inline constexpr bool is_lvq_packing_strategy_v = true; -template +#endif // USE_PROPRIETARY -inline constexpr bool is_lvq_packing_strategy_v> = true; - -template inline constexpr bool is_blocked = false; -template inline constexpr bool is_blocked> = true; - -template > struct select_rebind_allocator { - using type = lib::rebind_allocator_t; -}; -template struct select_rebind_allocator { - using base_allocator = typename A::allocator_type; - using rebind_base_allocator = lib::rebind_allocator_t; - using type = data::Blocked; -}; -template -using select_rebind_allocator_t = typename select_rebind_allocator::type; - -} // namespace detail +namespace svs { +namespace quantization { +namespace lvq { -template -concept LVQPackingStrategy = detail::is_lvq_packing_strategy_v; +///// +///// Load Helpers +///// -enum class LVQStrategyDispatch { Auto, Sequential, Turbo }; +// Types to use for lazy compression. +inline constexpr lib::Types CompressionTs{}; -// LVQDataset -template < - size_t Primary, - size_t Residual = 0, - size_t Extent = Dynamic, - LVQPackingStrategy Strategy = Sequential, - typename Alloc = lib::Allocator> -class LVQDataset { - public: - using allocator_type = detail::select_rebind_allocator_t; - private: - data::SimpleData primary_; +// How are we expecting to obtain the data. +struct OnlineCompression { public: - static constexpr bool is_resizeable = detail::is_blocked; - using const_value_type = typename data::SimpleData::const_value_type; - using element_type = float; - using value_type = const_value_type; - using primary_type = data::SimpleData; - void resize(size_t new_size) - requires is_resizeable - { - primary_.resize(new_size); - } - template - requires is_resizeable - void - compact(std::span new_to_old, Pool& threadpool, size_t batchsize = 1'000'000) { - primary_.compact(new_to_old, threadpool, batchsize); - } - - template - LVQDataset(Dataset primary): primary_{primary} { - if (fallback::get_mode() == fallback::FallbackMode::Error) { - throw fallback::UnsupportedHardwareError(); - } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { - fmt::print(fallback::fallback_warning); + explicit OnlineCompression(const std::filesystem::path& path, DataType type) + : path{path} + , type{type} { + if (!lib::in(type, CompressionTs)) { + throw ANNEXCEPTION("Invalid type!"); } } - size_t size() const { return primary_.size(); } - size_t dimensions() const { return primary_.dimensions(); } - const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } - void prefetch(size_t i) const { primary_.prefetch(i); } - - template - void set_datum(size_t i, std::span datum, size_t SVS_UNUSED(centroid_selector) = 0) { - primary_.set_datum(i, datum); - } - - template - static LVQDataset compress(const Dataset& data, const Alloc& allocator = {}) { - return compress(data, 1, 0, allocator); - } - - template - static LVQDataset compress( - const Dataset& data, - size_t num_threads, - size_t alignment, - const Alloc& allocator = {} - ) { - auto pool = threads::NativeThreadPool{num_threads}; - return compress(data, pool, alignment, allocator); - } - - template - static LVQDataset compress( - const Dataset& data, - Pool& SVS_UNUSED(threadpool), - size_t SVS_UNUSED(alignment), - const Alloc& allocator = {} - ) { - primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; - svs::data::copy(data, primary); - return LVQDataset{primary}; - } - - - static constexpr lib::Version save_version = lib::Version(0, 0, 0); - static constexpr std::string_view serialization_schema = "lvq_fallback"; - lib::SaveTable save(const lib::SaveContext& ctx) const { - return lib::SaveTable( - serialization_schema, - save_version, - {SVS_LIST_SAVE_(primary, ctx)} - ); - } - - static LVQDataset load( - const lib::LoadTable& table, - size_t SVS_UNUSED(alignment) = 0, - const Alloc& allocator = {} - ) { - return LVQDataset{SVS_LOAD_MEMBER_AT_(table, primary, allocator)}; - } + ///// Members + std::filesystem::path path; + DataType type; }; +/// +/// @brief Dispatch type indicating that a compressed dataset should be reloaded +/// directly. +/// +/// LVQ based loaders can either perform dataset compression online, or reload a +/// previously saved dataset. +/// +/// Using this type in LVQ loader constructors indicates that reloading is +/// desired. +/// struct Reload { public: + /// + /// @brief Construct a new Reloader. + /// + /// @param directory The directory where a LVQ compressed dataset was + /// previously saved. + /// explicit Reload(const std::filesystem::path& directory) : directory{directory} {} + ///// Members std::filesystem::path directory; }; +// The various ways we can instantiate LVQ-based datasets.. +using SourceTypes = std::variant; + +// Forward Declaration. template < size_t Primary, size_t Residual, @@ -185,104 +90,8 @@ template < typename Alloc> struct LVQLoader; -inline constexpr std::string_view one_level_serialization_schema = "one_level_lvq_dataset"; -inline constexpr lib::Version one_level_save_version = lib::Version(0, 0, 2); -inline constexpr std::string_view two_level_serialization_schema = "two_level_lvq_dataset"; -inline constexpr lib::Version two_level_save_version = lib::Version(0, 0, 3); -inline constexpr lib::Types CompressionTs{}; -struct OnlineCompression { - public: - explicit OnlineCompression(const std::filesystem::path& path, DataType type) - : path{path} - , type{type} { - if (!lib::in(type, CompressionTs)) { - throw ANNEXCEPTION("Invalid type!"); - } - } - - ///// Members - std::filesystem::path path; - DataType type; -}; -using SourceTypes = std::variant; - -enum class DatasetSchema { Compressed, ScaledBiased }; -struct Signed { - static constexpr std::string_view name = "signed"; -}; -inline constexpr std::string_view get_schema(DatasetSchema kind) { - switch (kind) { - using enum DatasetSchema; - case Compressed: { - return "lvq_compressed_dataset"; - } - case ScaledBiased: { - return "lvq_with_scaling_constants"; - } - } - throw ANNEXCEPTION("Invalid schema!"); -} -inline constexpr lib::Version get_current_version(DatasetSchema kind) { - switch (kind) { - using enum DatasetSchema; - case Compressed: { - return lib::Version(0, 0, 0); - } - case ScaledBiased: { - return lib::Version(0, 0, 3); - } - } - throw ANNEXCEPTION("Invalid schema!"); -} -struct DatasetSummary { - static bool check_load_compatibility(std::string_view schema, lib::Version version) { - using enum DatasetSchema; - if (schema == get_schema(Compressed) && - version == get_current_version(Compressed)) { - return true; - } - if (schema == get_schema(ScaledBiased) && - version == get_current_version(ScaledBiased)) { - return true; - } - return false; - } - - static DatasetSummary load(const lib::ContextFreeLoadTable& table) { - using enum DatasetSchema; - auto schema = table.schema(); - if (schema == get_schema(Compressed)) { - return DatasetSummary{ - .kind = Compressed, - .is_signed = - (lib::load_at(table, "sign") == lvq::Signed::name), - .dims = lib::load_at(table, "ndims"), - .bits = lib::load_at(table, "bits")}; - } - if (schema == get_schema(ScaledBiased)) { - return DatasetSummary{ - .kind = ScaledBiased, - .is_signed = false, // ScaledBiased always uses unsigned codes. - .dims = lib::load_at(table, "logical_dimensions"), - .bits = lib::load_at(table, "bits")}; - } - throw ANNEXCEPTION("Invalid table schema {}!", schema); - } - - ///// Members - // The kind of the leaf dataset. - DatasetSchema kind; - // Whether each LVQ element is signed. - bool is_signed; - // The logical number of dimensions in the dataset. - size_t dims; - // The number of bits used for compression. - size_t bits; -}; - -template -concept TurboLike = detail::is_turbo_like_v; namespace detail { + template constexpr bool is_compatible(LVQStrategyDispatch strategy) { switch (strategy) { @@ -298,30 +107,8 @@ constexpr bool is_compatible(LVQStrategyDispatch strategy) { } throw ANNEXCEPTION("Could not match strategy!"); } -} -template -int64_t overload_match_strategy(LVQStrategyDispatch strategy) { - constexpr bool is_sequential = std::is_same_v; - constexpr bool is_turbo = lvq::TurboLike; - switch (strategy) { - // If sequential is requested - we can only match sequential. - case LVQStrategyDispatch::Sequential: { - return is_sequential ? lib::perfect_match : lib::invalid_match; - } - // If turbo is requested - we can only match turbo. - case LVQStrategyDispatch::Turbo: { - return is_turbo ? lib::perfect_match : lib::invalid_match; - } - case LVQStrategyDispatch::Auto: { - // Preference: - // (1) Turbo - // (2) Sequential - return is_turbo ? 0 : 1; - } - } - throw ANNEXCEPTION("Unreachable!"); -} +} // namespace detail struct Matcher { // Load a matcher for either one or two level datasets. @@ -376,6 +163,30 @@ struct Matcher { size_t dims; }; +template +int64_t overload_match_strategy(LVQStrategyDispatch strategy) { + constexpr bool is_sequential = std::is_same_v; + constexpr bool is_turbo = lvq::TurboLike; + + switch (strategy) { + // If sequential is requested - we can only match sequential. + case LVQStrategyDispatch::Sequential: { + return is_sequential ? lib::perfect_match : lib::invalid_match; + } + // If turbo is requested - we can only match turbo. + case LVQStrategyDispatch::Turbo: { + return is_turbo ? lib::perfect_match : lib::invalid_match; + } + case LVQStrategyDispatch::Auto: { + // Preference: + // (1) Turbo + // (2) Sequential + return is_turbo ? 0 : 1; + } + } + throw ANNEXCEPTION("Unreachable!"); +} + // Compatibility ranking for LVQ template int64_t overload_score(size_t p, size_t r, size_t e, LVQStrategyDispatch strategy) { @@ -415,7 +226,6 @@ int64_t overload_score(Matcher matcher, LVQStrategyDispatch strategy) { ); } - template > struct ProtoLVQLoader { public: // Constructors @@ -622,4 +432,5 @@ struct lib::DispatchConverter< ); } }; -} + +} // namespace svs diff --git a/include/svs/quantization/lvq/lvq_fallback.h b/include/svs/quantization/lvq/lvq_fallback.h new file mode 100644 index 00000000..63afb6a6 --- /dev/null +++ b/include/svs/quantization/lvq/lvq_fallback.h @@ -0,0 +1,173 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/core/data/simple.h" +#include "svs/lib/threads.h" +#include "svs/lib/saveload/save.h" +#include "svs/fallback/fallback_mode.h" +#include "svs/quantization/lvq/lvq_common.h" + +namespace fallback = svs::fallback; + +namespace svs { +namespace quantization { +namespace lvq { + +struct Sequential { + static constexpr std::string_view name() { return "sequential"; } +}; + +template struct Turbo { + static constexpr std::string name() { + return fmt::format("turbo<{}x{}>", Lanes, ElementsPerLane); + } +}; + +namespace detail { + +// Trait to identify and dispatch based on the Turbo class itself. +template inline constexpr bool is_turbo_like_v = false; +template inline constexpr bool is_lvq_packing_strategy_v = false; + +template +inline constexpr bool is_turbo_like_v> = true; + +template <> inline constexpr bool is_lvq_packing_strategy_v = true; +template + +inline constexpr bool is_lvq_packing_strategy_v> = true; + +template > struct select_rebind_allocator { + using type = lib::rebind_allocator_t; +}; +template struct select_rebind_allocator { + using base_allocator = typename A::allocator_type; + using rebind_base_allocator = lib::rebind_allocator_t; + using type = data::Blocked; +}; +template +using select_rebind_allocator_t = typename select_rebind_allocator::type; + +} + +template +concept LVQPackingStrategy = detail::is_lvq_packing_strategy_v; + +template +concept TurboLike = detail::is_turbo_like_v; + +// LVQDataset +template < + size_t Primary, + size_t Residual = 0, + size_t Extent = Dynamic, + LVQPackingStrategy Strategy = Sequential, + typename Alloc = lib::Allocator> +class LVQDataset { + public: + using allocator_type = detail::select_rebind_allocator_t; + private: + data::SimpleData primary_; + public: + static constexpr bool is_resizeable = detail::is_blocked; + using const_value_type = typename data::SimpleData::const_value_type; + using element_type = float; + using value_type = const_value_type; + using primary_type = data::SimpleData; + void resize(size_t new_size) + requires is_resizeable + { + primary_.resize(new_size); + } + template + requires is_resizeable + void + compact(std::span new_to_old, Pool& threadpool, size_t batchsize = 1'000'000) { + primary_.compact(new_to_old, threadpool, batchsize); + } + + template + LVQDataset(Dataset primary): primary_{primary} { + if (fallback::get_mode() == fallback::FallbackMode::Error) { + throw fallback::UnsupportedHardwareError(); + } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { + fmt::print(fallback::fallback_warning); + } + } + + size_t size() const { return primary_.size(); } + size_t dimensions() const { return primary_.dimensions(); } + const_value_type get_datum(size_t i) const { return primary_.get_datum(i); } + void prefetch(size_t i) const { primary_.prefetch(i); } + + template + void set_datum(size_t i, std::span datum, size_t SVS_UNUSED(centroid_selector) = 0) { + primary_.set_datum(i, datum); + } + + template + static LVQDataset compress(const Dataset& data, const Alloc& allocator = {}) { + return compress(data, 1, 0, allocator); + } + + template + static LVQDataset compress( + const Dataset& data, + size_t num_threads, + size_t alignment, + const Alloc& allocator = {} + ) { + auto pool = threads::NativeThreadPool{num_threads}; + return compress(data, pool, alignment, allocator); + } + + template + static LVQDataset compress( + const Dataset& data, + Pool& SVS_UNUSED(threadpool), + size_t SVS_UNUSED(alignment), + const Alloc& allocator = {} + ) { + primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; + svs::data::copy(data, primary); + return LVQDataset{primary}; + } + + + static constexpr lib::Version save_version = lib::Version(0, 0, 0); + static constexpr std::string_view serialization_schema = "lvq_fallback"; + lib::SaveTable save(const lib::SaveContext& ctx) const { + return lib::SaveTable( + serialization_schema, + save_version, + {SVS_LIST_SAVE_(primary, ctx)} + ); + } + + static LVQDataset load( + const lib::LoadTable& table, + size_t SVS_UNUSED(alignment) = 0, + const Alloc& allocator = {} + ) { + return LVQDataset{SVS_LOAD_MEMBER_AT_(table, primary, allocator)}; + } +}; + +} // namespace lvq +} // namespace quantization +} // namespace svs From 256b5b3f3dfcc093fa12266f62d2e9693665893b Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 28 Mar 2025 05:50:08 -0700 Subject: [PATCH 08/23] update fallback --- bindings/python/include/svs/python/core.h | 6 ++++++ include/svs/fallback/fallback.h | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bindings/python/include/svs/python/core.h b/bindings/python/include/svs/python/core.h index 446983a9..c2921ac7 100644 --- a/bindings/python/include/svs/python/core.h +++ b/bindings/python/include/svs/python/core.h @@ -29,6 +29,12 @@ #include "svs/fallback/fallback.h" +#ifdef USE_PROPRIETARY + +#include "../../../../../../include/svs/fallback/fallback_python.h" + +#endif // USE_PROPRIETARY + // pybind #include diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index 35c571fa..f1174900 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -22,6 +22,6 @@ #ifdef USE_PROPRIETARY -#include "../../../../include/svs/fallback/fallback2.h" +#include "../../../../include/svs/fallback/fallback_cpp.h" #endif // USE_PROPRIETARY From 73b8642fa32ced57de20437e6b4b7fc842d6419b Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 28 Mar 2025 06:35:16 -0700 Subject: [PATCH 09/23] add python examples --- examples/python/example_fallback.py | 342 ++++++++++++++++++++ examples/python/example_fallback_leanvec.py | 124 +++++++ 2 files changed, 466 insertions(+) create mode 100644 examples/python/example_fallback.py create mode 100644 examples/python/example_fallback_leanvec.py diff --git a/examples/python/example_fallback.py b/examples/python/example_fallback.py new file mode 100644 index 00000000..3588d686 --- /dev/null +++ b/examples/python/example_fallback.py @@ -0,0 +1,342 @@ +# Copyright (C) 2023 Intel Corporation +# +# This software and the related documents are Intel copyrighted materials, +# and your use of them is governed by the express license under which they +# were provided to you ("License"). Unless the License provides otherwise, +# you may not use, modify, copy, publish, distribute, disclose or transmit +# this software or the related documents without Intel's prior written +# permission. +# +# This software and the related documents are provided as is, with no +# express or implied warranties, other than those that are expressly stated +# in the License. + +# Import `unittest` to allow for automated testing. +import unittest + +# [imports] +import os +import svs +# [imports] + +DEBUG_MODE = False +def assert_equal(lhs, rhs, message: str = ""): + if DEBUG_MODE: + print(f"{message}: {lhs} == {rhs}") + else: + assert lhs == rhs, message + +def run_test_float(index, queries, groundtruth): + expected = { + 10: 0.5664, + 20: 0.7397, + 30: 0.8288, + 40: 0.8837, + } + + for window_size in range(10, 50, 10): + index.search_window_size = window_size + I, D = index.search(queries, 10) + recall = svs.k_recall_at(groundtruth, I, 10, 10) + assert_equal( + recall, expected[window_size], f"Standard Search Check ({window_size})" + ) + +def run_test_two_level4_8(index, queries, groundtruth): + # expected = { + # 10: 0.5482, + # 20: 0.7294, + # 30: 0.8223, + # 40: 0.8756, + # } + expected = { + 10: 0.5664, + 20: 0.7397, + 30: 0.8288, + 40: 0.8837, + } + + for window_size in range(10, 50, 10): + index.search_window_size = window_size + I, D = index.search(queries, 10) + recall = svs.k_recall_at(groundtruth, I, 10, 10) + assert_equal( + recall, expected[window_size], f"Compressed Search Check ({window_size})" + ) + +def run_test_build_two_level4_8(index, queries, groundtruth): + # expected = { + # 10: 0.5484, + # 20: 0.7295, + # 30: 0.8221, + # 40: 0.8758, + # } + expected = { + 10: 0.5664, + 20: 0.7397, + 30: 0.8288, + 40: 0.8837, + } + + for window_size in range(10, 50, 10): + index.search_window_size = window_size + I, D = index.search(queries, 10) + recall = svs.k_recall_at(groundtruth, I, 10, 10) + assert_equal( + recall, expected[window_size], f"Compressed Search Check ({window_size})" + ) + +# Shadow this as a global to make it available to the test-case clean-up. +test_data_dir = None + +def run(): + + # ### + # Generating test data + # ### + + # [generate-dataset] + # Create a test dataset. + # This will create a directory "example_data_vamana" and populate it with three + # entries: + # - data.fvecs: The test dataset. + # - queries.fvecs: The test queries. + # - groundtruth.ivecs: The groundtruth. + test_data_dir = "./example_data_vamana" + svs.generate_test_dataset( + 10000, # Create 10000 vectors in the dataset. + 1000, # Generate 1000 query vectors. + 128, # Set the vector dimensionality to 128. + test_data_dir, # The directory where results will be generated. + data_seed = 1234, # Random number seed for reproducibility. + query_seed = 5678, # Random number seed for reproducibility. + num_threads = 4, # Number of threads to use. + distance = svs.DistanceType.L2, # The distance type to use. + ) + # [generate-dataset] + + + # ### + # Building the index + # ### + + # [build-parameters] + # Now, we can build a graph index over the data set. + parameters = svs.VamanaBuildParameters( + graph_max_degree = 64, + window_size = 128, + ) + # [build-parameters] + + # [build-index] + # Build the index. + index = svs.Vamana.build( + parameters, + svs.VectorDataLoader( + os.path.join(test_data_dir, "data.fvecs"), svs.DataType.float32 + ), + svs.DistanceType.L2, + num_threads = 4, + ) + # [build-index] + + # [build-index-fromNumpyArray] + # Build the index. + data = svs.read_vecs(os.path.join(test_data_dir, "data.fvecs")) + index = svs.Vamana.build( + parameters, + data, + svs.DistanceType.L2, + num_threads = 4, + ) + # [build-index-fromNumpyArray] + + + # ### + # Searching the index + # ### + + # [load-aux] + # Load the queries and ground truth. + queries = svs.read_vecs(os.path.join(test_data_dir, "queries.fvecs")) + groundtruth = svs.read_vecs(os.path.join(test_data_dir, "groundtruth.ivecs")) + # [load-aux] + + # [perform-queries] + # Set the search window size of the index and perform queries. + index.search_window_size = 30 + I, D = index.search(queries, 10) + + # Compare with the groundtruth. + recall = svs.k_recall_at(groundtruth, I, 10, 10) + print(f"Recall = {recall}") + assert(recall == 0.8288) + # [perform-queries] + + # [search-window-size] + # We can vary the search window size to demonstrate the trade off in accuracy. + for window_size in range(10, 50, 10): + index.search_window_size = window_size + I, D = index.search(queries, 10) + recall = svs.k_recall_at(groundtruth, I, 10, 10) + print(f"Window size = {window_size}, Recall = {recall}") + # [search-window-size] + + ##### Begin Test + run_test_float(index, queries, groundtruth) + ##### End Test + + + # ### + # Saving the index + # ### + + # [saving-results] + # Finally, we can save the results. + index.save( + os.path.join(test_data_dir, "example_config"), + os.path.join(test_data_dir, "example_graph"), + os.path.join(test_data_dir, "example_data"), + ) + # [saving-results] + + + # ### + # Reloading a saved index + # ### + + # [loading] + # We can reload an index from a previously saved set of files. + index = svs.Vamana( + os.path.join(test_data_dir, "example_config"), + svs.GraphLoader(os.path.join(test_data_dir, "example_graph")), + svs.VectorDataLoader( + os.path.join(test_data_dir, "example_data"), svs.DataType.float32 + ), + svs.DistanceType.L2, + num_threads = 4, + ) + + # We can rerun the queries to ensure everything works properly. + index.search_window_size = 30 + I, D = index.search(queries, 10) + + # Compare with the groundtruth. + recall = svs.k_recall_at(groundtruth, I, 10, 10) + print(f"Recall = {recall}") + assert(recall == 0.8288) + # [loading] + + ##### Begin Test + run_test_float(index, queries, groundtruth) + ##### End Test + + # [only-loading] + # We can reload an index from a previously saved set of files. + index = svs.Vamana( + os.path.join(test_data_dir, "example_config"), + svs.GraphLoader(os.path.join(test_data_dir, "example_graph")), + svs.VectorDataLoader( + os.path.join(test_data_dir, "example_data"), svs.DataType.float32 + ), + svs.DistanceType.L2, + num_threads = 4, + ) + # [only-loading] + + # [runtime-nthreads] + index.num_threads = 4 + # [runtime-nthreads] + + + # ### + # Search using vector compression + # ### + + # [search-compressed-loader] + data_loader = svs.VectorDataLoader( + os.path.join(test_data_dir, "example_data"), # Uncompressed data + svs.DataType.float32, + dims = 128 # Passing dimensionality is optional + ) + B1 = 4 # Number of bits for the first level LVQ quantization + B2 = 8 # Number of bits for the residuals quantization + padding = 32 + strategy = svs.LVQStrategy.Turbo + compressed_loader = svs.LVQLoader(data_loader, + primary=B1, + residual=B2, + strategy=strategy, # Passing the strategy is optional. + padding=padding # Passing padding is optional. + ) + # [search-compressed-loader] + + # [search-compressed] + index = svs.Vamana( + os.path.join(test_data_dir, "example_config"), + svs.GraphLoader(os.path.join(test_data_dir, "example_graph")), + compressed_loader, + # Optional keyword arguments + distance = svs.DistanceType.L2, + num_threads = 4 + ) + + # Compare with the groundtruth.. + index.search_window_size = 30 + I, D = index.search(queries, 10) + recall = svs.k_recall_at(groundtruth, I, 10, 10) + print(f"Compressed recall: {recall}") + assert(recall == 0.8288) #assert(recall == 0.8223) + # [search-compressed] + + ##### Begin Test + run_test_two_level4_8(index, queries, groundtruth) + ##### End Test + + # [build-index-compressed] + # Build the index. + index = svs.Vamana.build( + parameters, + compressed_loader, + svs.DistanceType.L2, + num_threads = 4 + ) + # [build-index-compressed] + + # 1. Building Uncompressed + # 2. Loading Uncompressed + # 3. Loading with a recompressor + + # We can rerun the queries to ensure everything works properly. + index.search_window_size = 30 + I, D = index.search(queries, 10) + + # Compare with the groundtruth. + recall = svs.k_recall_at(groundtruth, I, 10, 10) + print(f"Recall = {recall}") + assert(recall == 0.8288) + # [loading] + + ##### Begin Test + run_test_build_two_level4_8(index, queries, groundtruth) + ##### End Test + +##### +##### Main Executable +##### + +if __name__ == "__main__": + run() + +##### +##### As a unit test. +##### + +class VamanaExampleTestCase(unittest.TestCase): + def tearDown(self): + if test_data_dir is not None: + print(f"Removing temporary directory {test_data_dir}") + os.rmdir(test_data_dir) + + def test_all(self): + run() diff --git a/examples/python/example_fallback_leanvec.py b/examples/python/example_fallback_leanvec.py new file mode 100644 index 00000000..6bb8ed5c --- /dev/null +++ b/examples/python/example_fallback_leanvec.py @@ -0,0 +1,124 @@ +# Copyright (C) 2023 Intel Corporation +# +# This software and the related documents are Intel copyrighted materials, +# and your use of them is governed by the express license under which they +# were provided to you ("License"). Unless the License provides otherwise, +# you may not use, modify, copy, publish, distribute, disclose or transmit +# this software or the related documents without Intel's prior written +# permission. +# +# This software and the related documents are provided as is, with no +# express or implied warranties, other than those that are expressly stated +# in the License. + +# Import `unittest` to allow for auotmated testing. +import unittest + +# [imports] +import os +import svs +# [imports] + +DEBUG_MODE = False +def assert_equal(lhs, rhs, message: str = ""): + if DEBUG_MODE: + print(f"{message}: {lhs} == {rhs}") + else: + assert lhs == rhs, message + +test_data_dir = None + +def run(): + # [generate-dataset] + # Create a test dataset. + # This will create a directory "example_data_vamana" and populate it with three + # entries: + # - data.fvecs: The test dataset. + # - queries.fvecs: The test queries. + # - groundtruth.fvecs: The groundtruth. + test_data_dir = "./example_data_vamana" + svs.generate_test_dataset( + 1000, # Create 1000 vectors in the dataset. + 100, # Generate 100 query vectors. + 256, # Set the vector dimensionality to 256. + test_data_dir, # The directory where results will be generated. + data_seed = 1234, # Random number seed for reproducibility. + query_seed = 5678, # Random number seed for reproducibility. + num_threads = 4, # Number of threads to use. + distance = svs.DistanceType.MIP, # The distance type to use. + ) + # [generate-dataset] + + # [create-loader] + # We are going to construct a LeanVec dataset on-the-fly from uncompressed data. + # First, we construct a loader for the uncompressed data. + uncompressed_loader = svs.VectorDataLoader( + os.path.join(test_data_dir, "data.fvecs"), + svs.DataType.float32 + ) + + # Next - we construct a LeanVecLoader. + # This loader is configured to perform the following: + # - Reduce dimensionality of the primary dataset to 256 dimensions. + # - Use LVQ8 for the primary dataset. + # - Use Float16 for the secondary, unreduced dataset. + leanvec_loader = svs.LeanVecLoader( + uncompressed_loader, + 128, # The reduced number of dimensions. + primary_kind = svs.LeanVecKind.lvq8, # The encoding of the primary dataset. + secondary_kind = svs.LeanVecKind.float16, # The encoding of the secondary dataset. + ) + # [create-loader] + + # [build-and-search-index] + # An index can be constructed using a LeanVec dataset. + # Use an alpha less than 1 since we are using the Inner Product distance. + parameters = svs.VamanaBuildParameters( + alpha = 0.95, + graph_max_degree = 64, + prune_to = 60, + window_size = 128, + ) + + index = svs.Vamana.build( + parameters, + leanvec_loader, + svs.DistanceType.MIP, + num_threads = 4, + ) + + # Load queries and ground-truth. + queries = svs.read_vecs(os.path.join(test_data_dir, "queries.fvecs")) + groundtruth = svs.read_vecs(os.path.join(test_data_dir, "groundtruth.ivecs")) + + # Set the search window size of the index and perform queries. + p = index.search_parameters + p.buffer_config = svs.SearchBufferConfig(30, 60) + index.search_parameters = p + I, D = index.search(queries, 10) + + # Compare with the groundtruth. + recall = svs.k_recall_at(groundtruth, I, 10, 10) + print(f"Recall = {recall}") + assert_equal(recall, 0.976, "initial recall") + # [build-and-search-index] + +##### +##### Main Executable +##### + +if __name__ == "__main__": + run() + +##### +##### As a unit test. +##### + +class VamanaExampleTestCase(unittest.TestCase): + def tearDown(self): + if test_data_dir is not None: + print(f"Removing temporary directory {test_data_dir}") + os.rmdir(test_data_dir) + + def test_all(self): + run() From 57b398280b6f9339317908d9a49d8a296d527320 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 28 Mar 2025 06:43:46 -0700 Subject: [PATCH 10/23] add cpp test --- tests/CMakeLists.txt | 2 + tests/svs/fallback/fallback.cpp | 636 ++++++++++++++++++++++++++++++++ tests/svs/fallback/utils.h | 116 ++++++ 3 files changed, 754 insertions(+) create mode 100644 tests/svs/fallback/fallback.cpp create mode 100644 tests/svs/fallback/utils.h diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b8352697..acd83c7d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -140,6 +140,8 @@ set(TEST_SOURCES # Inverted ${TEST_DIR}/svs/index/inverted/clustering.cpp ${TEST_DIR}/svs/index/inverted/memory_based.cpp + # Fallback + ${TEST_DIR}/svs/fallback/fallback.cpp # # ${TEST_DIR}/svs/index/vamana/dynamic_index.cpp ) diff --git a/tests/svs/fallback/fallback.cpp b/tests/svs/fallback/fallback.cpp new file mode 100644 index 00000000..c925a0a9 --- /dev/null +++ b/tests/svs/fallback/fallback.cpp @@ -0,0 +1,636 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// SVS +#include "svs/core/recall.h" +#include "svs/lib/static.h" +#include "svs/fallback/fallback.h" +#include "svs/orchestrators/dynamic_vamana.h" +#include "svs/orchestrators/exhaustive.h" +#include "svs/orchestrators/vamana.h" + +// catch2 +#include "catch2/catch_test_macros.hpp" + +#include "utils.h" + +namespace { + +// SVS setup and parameters +const size_t num_threads = 4; +size_t search_window_size = 20; +size_t n_neighbors = 10; +std::string dfname = "data.vecs"; +std::string dfname_f16 = "data_f16.vecs"; +std::string qfname = "query.vecs"; +std::string qfname_f16 = "query_f16.vecs"; +std::string gtfname = "gt.vecs"; + +const std::filesystem::path& config_path = "./config"; +const std::filesystem::path& graph_path = "./graph"; +// const std::filesystem::path& data_path = "./data"; +const std::filesystem::path& config_path_dynamic = "./config_dynamic"; +const std::filesystem::path& graph_path_dynamic = "./graph_dynamic"; + +void svs_setup() { + // convert to fp16 + auto reader = svs::io::vecs::VecsReader{dfname}; + auto writer = svs::io::vecs::VecsWriter{dfname_f16, reader.ndims()}; + { + for (auto i : reader) { + writer << i; + } + } + writer.flush(); + + reader = svs::io::vecs::VecsReader{qfname}; + writer = svs::io::vecs::VecsWriter{qfname_f16, reader.ndims()}; + { + for (auto i : reader) { + writer << i; + } + } + writer.flush(); +} + +template auto create_lvq_data() { + namespace lvq = svs::quantization::lvq; + + auto compressor = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader(dfname).load(); + return lvq::LVQDataset::compress(data, threadpool, 32); + }); + + auto threadpool = svs::threads::as_threadpool(num_threads); + auto data = svs::detail::dispatch_load(compressor, threadpool); + fmt::print("Create LVQ data with P={}, R={}, E={}\n", P, R, E); + return data; +} + +template +auto create_blocked_lvq_data() { + namespace lvq = svs::quantization::lvq; + using blocked_type = svs::data::Blocked; + + auto compressor = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader(dfname).load(); + return lvq::LVQDataset::compress(data, threadpool, 32); + }); + + auto threadpool = svs::threads::as_threadpool(num_threads); + auto data = svs::detail::dispatch_load(compressor, threadpool); + fmt::print("Create Blocked LVQ data with P={}, R={}, E={}\n", P, R, E); + return data; +} + +template +auto create_lvq_data_with_alloc_handle(const A& alloc) { + namespace lvq = svs::quantization::lvq; + + auto compressor = + svs::lib::Lazy([=, &alloc](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader(dfname).load(); + return lvq::LVQDataset::compress(data, threadpool, 32, alloc); + }); + + auto threadpool = svs::threads::as_threadpool(num_threads); + auto data = svs::detail::dispatch_load(compressor, threadpool); + fmt::print("Create LVQ data using AllocatorHandle with P={}, R={}, E={}\n", P, R, E); + return data; +} + +template +auto create_leanvec_data() { + namespace leanvec = svs::leanvec; + assert(D >= 32); + size_t leanvec_dim = (L == svs::Dynamic) ? 32 : L; + + auto compressor = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader(dfname).load(); + return leanvec::LeanDataset::reduce( + data, std::nullopt, threadpool, 32, svs::lib::MaybeStatic(leanvec_dim) + ); + }); + + auto threadpool = svs::threads::as_threadpool(num_threads); + auto data = svs::detail::dispatch_load(compressor, threadpool); + fmt::print("Create Leanvec data with L={}, D={}\n", L, D); + return data; +} + +template +auto create_leanvec_data_with_alloc_handle(const A& alloc) { + namespace leanvec = svs::leanvec; + assert(D >= 32); + size_t leanvec_dim = (L == svs::Dynamic) ? 32 : L; + + auto compressor = svs::lib::Lazy([=, + &alloc](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader(dfname).load(); + return leanvec::LeanDataset::reduce( + data, std::nullopt, threadpool, 32, svs::lib::MaybeStatic(leanvec_dim), alloc + ); + }); + + auto threadpool = svs::threads::as_threadpool(num_threads); + auto data = svs::detail::dispatch_load(compressor, threadpool); + fmt::print("Create Leanvec data with L={}, D={}\n", L, D); + return data; +} + +template +auto create_blocked_leanvec_data() { + namespace leanvec = svs::leanvec; + using blocked_type = svs::data::Blocked; + assert(D >= 32); + size_t leanvec_dim = (L == svs::Dynamic) ? 32 : L; + + auto compressor = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader(dfname).load(); + return leanvec::LeanDataset::reduce( + data, std::nullopt, threadpool, 32, svs::lib::MaybeStatic(leanvec_dim) + ); + }); + + auto threadpool = svs::threads::as_threadpool(num_threads); + auto data = svs::detail::dispatch_load(compressor, threadpool); + fmt::print("Create Blocked Leanvec data with L={}, D={}\n", L, D); + return data; +} + +float get_alpha(svs::distance::DistanceL2 /*dist*/) { return 1.2; } + +float get_alpha(svs::distance::DistanceIP /*dist*/) { return 0.9; } + +template +void vamana_build(Data& data, Distance distance) { + auto parameters = svs::index::vamana::VamanaBuildParameters{ + get_alpha(distance), // alpha + 64, // graph max degree + 128, // search window size + 750, // max candidate pool size + 60, // prune to degree + true, // full search history + }; + + auto tic = svs::lib::now(); + svs::Vamana index = + svs::Vamana::build(parameters, data, Distance(), num_threads); + auto build_time = svs::lib::time_difference(tic); + fmt::print("Vamana index build time: {}\n", build_time); + index.save("config", "graph", "data"); +} + +template +void vamana_search(Data& data, Distance distance) { + auto index = svs::Vamana::assemble( + config_path, svs::GraphLoader(graph_path), data, distance, num_threads + ); + + index.set_search_window_size(search_window_size); + const auto query_data = svs::load_data(qfname); + const auto groundtruth = svs::load_data(gtfname); + + auto tic = svs::lib::now(); + auto query_result = index.search(query_data, n_neighbors); + auto search_time = svs::lib::time_difference(tic); + + std::vector qps; + for (int i = 0; i < 5; i++) { + tic = svs::lib::now(); + query_result = index.search(query_data, n_neighbors); + search_time = svs::lib::time_difference(tic); + qps.push_back(query_data.size() / search_time); + } + + auto recall = svs::k_recall_at_n(groundtruth, query_result, 1, 1); + fmt::print("Raw QPS: {:7.3f} \n", fmt::join(qps, ", ")); + fmt::print( + "Vamana search window size: {}, 1-Recall@1: {}, Max QPS: {:7.3f} \n", + search_window_size, + recall, + *std::max_element(qps.begin(), qps.end()) + ); +} + +template void vamana_build_search(Data& data) { + vamana_build(data, svs::distance::DistanceL2()); + vamana_search(data, svs::distance::DistanceL2()); + + vamana_build(data, svs::distance::DistanceIP()); + vamana_search(data, svs::distance::DistanceIP()); +} + +template +void dynamic_vamana_build(Data& data, Distance distance) { + auto parameters = svs::index::vamana::VamanaBuildParameters{ + get_alpha(distance), // alpha + 64, // graph max degree + 128, // search window size + 750, // max candidate pool size + 60, // prune to degree + true, // full search history + }; + + auto tic = svs::lib::now(); + std::vector ids(data.size()); + for (size_t i = 0; i < data.size(); ++i) { + ids[i] = i; + } + + svs::DynamicVamana index = svs::DynamicVamana::build( + parameters, data, svs::lib::as_span(ids), Distance(), num_threads + ); + auto build_time = svs::lib::time_difference(tic); + fmt::print("DynamicVamana index build time: {}\n", build_time); + index.save("config_dynamic", "graph_dynamic", "data_dynamic"); +} + +template +void dynamic_vamana_search(Data& data, Distance distance) { + using Idx = uint32_t; + auto index = svs::DynamicVamana::assemble( + config_path_dynamic, + SVS_LAZY(svs::graphs::SimpleBlockedGraph::load(graph_path_dynamic)), + data, + distance, + num_threads + ); + + index.set_search_window_size(search_window_size); + const auto query_data = svs::load_data(qfname); + const auto groundtruth = svs::load_data(gtfname); + + auto tic = svs::lib::now(); + auto query_result = index.search(query_data, n_neighbors); + auto search_time = svs::lib::time_difference(tic); + + std::vector qps; + for (int i = 0; i < 5; i++) { + tic = svs::lib::now(); + query_result = index.search(query_data, n_neighbors); + search_time = svs::lib::time_difference(tic); + qps.push_back(query_data.size() / search_time); + } + + auto recall = svs::k_recall_at_n(groundtruth, query_result, 1, 1); + fmt::print("Raw QPS: {:7.3f} \n", fmt::join(qps, ", ")); + fmt::print( + "Dynamic vamana search window size: {}, 1-Recall@1: {}, Max QPS: {:7.3f} \n", + search_window_size, + recall, + *std::max_element(qps.begin(), qps.end()) + ); +} + +template void dynamic_vamana_build_search(Data& data) { + dynamic_vamana_build(data, svs::distance::DistanceL2()); + dynamic_vamana_search(data, svs::distance::DistanceL2()); + + dynamic_vamana_build(data, svs::distance::DistanceIP()); + dynamic_vamana_search(data, svs::distance::DistanceIP()); +} + +template +void flat_search(Data& data, Distance distance) { + svs::Flat index = svs::Flat::assemble(data, distance, num_threads); + + const auto query_data = svs::load_data(qfname); + const auto groundtruth = svs::load_data(gtfname); + + auto tic = svs::lib::now(); + auto query_result = index.search(query_data, n_neighbors); + auto search_time = svs::lib::time_difference(tic); + + std::vector qps; + for (int i = 0; i < 5; i++) { + tic = svs::lib::now(); + query_result = index.search(query_data, n_neighbors); + search_time = svs::lib::time_difference(tic); + qps.push_back(query_data.size() / search_time); + } + + auto recall = svs::k_recall_at_n(groundtruth, query_result, 1, 1); + fmt::print("Raw QPS: {:7.3f} \n", fmt::join(qps, ", ")); + fmt::print( + "Flat search 1-Recall@1: {}, Max QPS: {:7.3f} \n", + recall, + *std::max_element(qps.begin(), qps.end()) + ); +} + +template void flat_search(Data& data) { + flat_search(data, svs::distance::DistanceL2()); + flat_search(data, svs::distance::DistanceIP()); +} + +template void dynamic_vamana_search() { + // Dynamic Index + using S = svs::quantization::lvq::Sequential; + using S1 = svs::quantization::lvq::Turbo<16, 8>; + { + using Alloc = svs::data::Blocked>; + auto data = svs::VectorDataLoader(dfname).load(); + dynamic_vamana_build_search(data); + } + + { + using Alloc = svs::data::Blocked>; + auto data = svs::VectorDataLoader(dfname_f16).load(); + dynamic_vamana_build_search(data); + } + + { + auto data = create_blocked_lvq_data<4, 8, D, S, A>(); + dynamic_vamana_build_search(data); + } + + { + auto data = create_blocked_lvq_data<4, 0, D, S1, A>(); + dynamic_vamana_build_search(data); + } + + { + auto data = create_blocked_lvq_data<4, 4, D, S1, A>(); + dynamic_vamana_build_search(data); + } + + { + auto data = create_blocked_lvq_data<4, 8, D, S1, A>(); + dynamic_vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<8>; + using S = svs::leanvec::UsingLVQ<8>; + auto data = create_blocked_leanvec_data(); + dynamic_vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<4>; + using S = svs::leanvec::UsingLVQ<8>; + auto data = create_blocked_leanvec_data(); + dynamic_vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<8>; + using S = svs::Float16; + auto data = create_blocked_leanvec_data(); + dynamic_vamana_build_search(data); + } +} + +template void flat_search() { + // using S = svs::quantization::lvq::Sequential; + using S1 = svs::quantization::lvq::Turbo<16, 8>; + { + auto data = svs::VectorDataLoader(dfname).load(); + flat_search(data); + } + + { + auto data = svs::VectorDataLoader(dfname_f16).load(); + flat_search(data); + } + + { + auto data = create_lvq_data<4, 8, D, S1, A>(); + flat_search(data); + } +} + +template void vamana_search() { + using S = svs::quantization::lvq::Sequential; + using S1 = svs::quantization::lvq::Turbo<16, 8>; + { + auto data = svs::VectorDataLoader(dfname).load(); + vamana_build_search(data); + } + + { + auto data = svs::VectorDataLoader(dfname_f16).load(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<4, 0, D, S, A>(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<4, 4, D, S, A>(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<4, 8, D, S, A>(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<4, 0, D, S1, A>(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<4, 4, D, S1, A>(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<4, 8, D, S1, A>(); + vamana_build_search(data); + } + + { + auto data = create_lvq_data<8, 0, D, S, A>(); + vamana_build_search(data); + } + + { + auto alloc = svs::make_allocator_handle(svs::HugepageAllocator()); + auto data = + create_lvq_data_with_alloc_handle<4, 4, D, S, decltype(alloc)>(std::move(alloc) + ); + vamana_build_search(data); + } + + { + auto alloc = + svs::make_blocked_allocator_handle(svs::HugepageAllocator()); + auto data = + create_lvq_data_with_alloc_handle<4, 4, D, S, decltype(alloc)>(std::move(alloc) + ); + vamana_build_search(data); + } + + { + auto alloc = + svs::make_blocked_allocator_handle(svs::HugepageAllocator()); + auto data = + create_lvq_data_with_alloc_handle<4, 8, D, S1, decltype(alloc)>(std::move(alloc) + ); + vamana_build_search(data); + } + + { + auto alloc = svs::make_allocator_handle(svs::lib::Allocator()); + auto data = + create_lvq_data_with_alloc_handle<4, 4, D, S, decltype(alloc)>(std::move(alloc) + ); + vamana_build_search(data); + } + + { + auto alloc = svs::make_blocked_allocator_handle(svs::lib::Allocator()); + auto data = + create_lvq_data_with_alloc_handle<4, 4, D, S, decltype(alloc)>(std::move(alloc) + ); + vamana_build_search(data); + } + + { + auto alloc = svs::make_blocked_allocator_handle(svs::lib::Allocator()); + auto data = + create_lvq_data_with_alloc_handle<4, 8, D, S1, decltype(alloc)>(std::move(alloc) + ); + vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<8>; + using S = svs::leanvec::UsingLVQ<8>; + auto data = create_leanvec_data(); + vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<8>; + using S = svs::leanvec::UsingLVQ<8>; + auto alloc = svs::make_allocator_handle(svs::lib::Allocator()); + auto data = create_leanvec_data_with_alloc_handle( + std::move(alloc) + ); + vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<8>; + using S = svs::leanvec::UsingLVQ<8>; + auto alloc = svs::make_blocked_allocator_handle(svs::lib::Allocator()); + auto data = create_leanvec_data_with_alloc_handle( + std::move(alloc) + ); + vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<4>; + using S = svs::leanvec::UsingLVQ<8>; + auto data = create_leanvec_data(); + vamana_build_search(data); + } + + { + using P = svs::leanvec::UsingLVQ<8>; + using S = svs::Float16; + auto data = create_leanvec_data(); + vamana_build_search(data); + } + + { + using P = svs::Float16; + using S = svs::Float16; + auto data = create_leanvec_data(); + vamana_build_search(data); + } + + { + using P = svs::Float16; + using S = svs::Float16; + auto alloc = svs::make_allocator_handle(svs::lib::Allocator()); + auto data = create_leanvec_data_with_alloc_handle( + std::move(alloc) + ); + vamana_build_search(data); + } + + { + using P = svs::Float16; + using S = svs::Float16; + auto alloc = svs::make_blocked_allocator_handle(svs::lib::Allocator()); + auto data = create_leanvec_data_with_alloc_handle( + std::move(alloc) + ); + vamana_build_search(data); + } + + { + using P = float; + using S = float; + auto data = create_leanvec_data(); + vamana_build_search(data); + } + + { + using P = float; + using S = float; + auto alloc = svs::make_allocator_handle(svs::lib::Allocator()); + auto data = create_leanvec_data_with_alloc_handle( + std::move(alloc) + ); + vamana_build_search(data); + } + + { + using P = float; + using S = float; + auto alloc = svs::make_blocked_allocator_handle(svs::lib::Allocator()); + auto data = create_leanvec_data_with_alloc_handle( + std::move(alloc) + ); + vamana_build_search(data); + } +} + +} // namespace + +CATCH_TEST_CASE("Shared library", "[shared][shared][shared_search]") { + const size_t D = 512; + size_t dataset_size = 14; + size_t query_size = 3; + using A = svs::lib::Allocator; + using A1 = svs::HugepageAllocator; + generate_random_data(D, dataset_size, query_size); + svs_setup(); + + CATCH_SECTION("Vamana Search") { + vamana_search(); + vamana_search(); + } + + CATCH_SECTION("Flat Search") { + flat_search(); + flat_search(); + } + + CATCH_SECTION("Dynamic Vamana Search") { + dynamic_vamana_search(); + dynamic_vamana_search(); + } +} diff --git a/tests/svs/fallback/utils.h b/tests/svs/fallback/utils.h new file mode 100644 index 00000000..d0767b69 --- /dev/null +++ b/tests/svs/fallback/utils.h @@ -0,0 +1,116 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/***************************************************** + * I/O functions for fvecs, ivecs and xVecs + *****************************************************/ + +#include +#include +#include +#include + +int fvec_fwrite(FILE* fo, const float* v, int d) { + int ret; + ret = fwrite(&d, sizeof(int), 1, fo); + if (ret != 1) { + perror("fvec_fwrite: write error 1"); + return -1; + } + ret = fwrite(v, sizeof(float), d, fo); + if (ret != d) { + perror("fvec_fwrite: write error 2"); + return -1; + } + return 0; +} + +int fvecs_write(const char* fname, int d, int n, const float* vf) { + FILE* fo = fopen(fname, "w"); + if (!fo) { + perror("fvecs_write: cannot open file"); + return -1; + } + + int i; + /* write down the vectors as fvecs */ + for (i = 0; i < n; i++) { + if (fvec_fwrite(fo, vf + i * d, d) < 0) + return -1; + } + fclose(fo); + return n; +} + +int ivec_iwrite(FILE* fo, const int* v, int d) { + int ret; + ret = fwrite(&d, sizeof(int), 1, fo); + if (ret != 1) { + perror("fvec_fwrite: write error 1"); + return -1; + } + ret = fwrite(v, sizeof(float), d, fo); + if (ret != d) { + perror("fvec_fwrite: write error 2"); + return -1; + } + return 0; +} + +int ivecs_write(const char* fname, int d, int n, const int* vf) { + FILE* fo = fopen(fname, "w"); + if (!fo) { + perror("fvecs_write: cannot open file"); + return -1; + } + + int i; + /* write down the vectors as fvecs */ + for (i = 0; i < n; i++) { + if (ivec_iwrite(fo, vf + i * d, d) < 0) + return -1; + } + fclose(fo); + return n; +} + +void generate_random_data(size_t data_dim, size_t dataset_size, size_t query_size) { + float dataset_std = 1.0f, query_std = 0.1f; + + srand(100); + std::default_random_engine generator; + std::normal_distribution dataset_dist(0.0f, dataset_std); + std::normal_distribution query_dist(0.0f, query_std); + std::uniform_int_distribution<> uni_dist(0, dataset_size - 1); + + std::vector dataset(dataset_size * data_dim); + for (size_t i = 0; i < dataset.size(); ++i) { + dataset[i] = dataset_dist(generator); + } + + std::vector queries(query_size * data_dim); + std::vector gt(query_size); + for (size_t i = 0; i < query_size; ++i) { + int e = uni_dist(generator); + for (size_t j = 0; j < data_dim; ++j) { + queries[i * data_dim + j] = dataset[e * data_dim + j] + query_dist(generator); + } + gt[i] = e; + } + + fvecs_write("data.vecs", data_dim, dataset_size, dataset.data()); + fvecs_write("query.vecs", data_dim, query_size, queries.data()); + ivecs_write("gt.vecs", 1, query_size, gt.data()); +} From 549add07eccf2600e680f3fc81a6bd8c225cb39c Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 28 Mar 2025 07:00:13 -0700 Subject: [PATCH 11/23] update py example license and add cpp example --- examples/cpp/CMakeLists.txt | 1 + examples/cpp/fallback.cpp | 248 ++++++++++++++++++++ examples/python/example_fallback.py | 21 +- examples/python/example_fallback_leanvec.py | 21 +- 4 files changed, 271 insertions(+), 20 deletions(-) create mode 100644 examples/cpp/fallback.cpp diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 52530f57..fe7f4658 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -38,6 +38,7 @@ create_simple_example(saveload test_saveload saveload.cpp) create_simple_example(types test_types types.cpp) create_simple_example(vamana_iterator test_vamana_iterator vamana_iterator.cpp) create_simple_example(custom_thread_pool test_custom_thread_pool custom_thread_pool.cpp) +create_simple_example(fallback test_fallback fallback.cpp) ## More complicated examples involving more extensive setup. diff --git a/examples/cpp/fallback.cpp b/examples/cpp/fallback.cpp new file mode 100644 index 00000000..c2e05674 --- /dev/null +++ b/examples/cpp/fallback.cpp @@ -0,0 +1,248 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! [Example All] + +//! [Includes] +// SVS Dependencies +#include "svs/orchestrators/vamana.h" // bulk of the dependencies required. +#include "svs/core/recall.h" // Convenient k-recall@n computation. +#include "svs/fallback/fallback.h" + +// Alternative main definition +#include "svsmain.h" + +// stl +#include +#include +#include +#include +//! [Includes] + +//! [Helper Utilities] +double run_recall( + svs::Vamana& index, + const svs::data::SimpleData& queries, + const svs::data::SimpleData& groundtruth, + size_t search_window_size, + size_t num_neighbors, + std::string_view message = "" +) { + index.set_search_window_size(search_window_size); + auto results = index.search(queries, num_neighbors); + double recall = svs::k_recall_at_n(groundtruth, results, num_neighbors, num_neighbors); + if (!message.empty()) { + fmt::print("[{}] ", message); + } + fmt::print("Windowsize = {}, Recall = {}\n", search_window_size, recall); + return recall; +} + +const bool DEBUG = true; +void check(double expected, double got, double eps = 0.001) { + double diff = std::abs(expected - got); + if constexpr (DEBUG) { + fmt::print("Expected {}. Got {}\n", expected, got); + } else { + if (diff > eps) { + throw ANNEXCEPTION("Expected ", expected, ". Got ", got, '!'); + } + } +} +//! [Helper Utilities] + +// Alternative main definition +int svs_main(std::vector args) { + //! [Argument Extraction] + const size_t nargs = args.size(); + if (nargs != 4) { + throw ANNEXCEPTION("Expected 3 arguments. Instead, got ", nargs, '!'); + } + const std::string& data_vecs = args.at(1); + const std::string& query_vecs = args.at(2); + const std::string& groundtruth_vecs = args.at(3); + //! [Argument Extraction] + + // Building the index + + //! [Build Parameters] + auto parameters = svs::index::vamana::VamanaBuildParameters{ + 1.2, // alpha + 64, // graph max degree + 128, // search window size + 1024, // max candidate pool size + 60, // prune to degree + true, // full search history + }; + //! [Build Parameters] + + //! [Index Build] + size_t num_threads = 4; + svs::Vamana index = svs::Vamana::build( + parameters, svs::VectorDataLoader(data_vecs), svs::DistanceL2(), num_threads + ); + //! [Index Build] + + // Searching the index + + //! [Load Aux] + // Load the queries and ground truth. + auto queries = svs::load_data(query_vecs); + auto groundtruth = svs::load_data(groundtruth_vecs); + //! [Load Aux] + + //! [Perform Queries] + index.set_search_window_size(30); + svs::QueryResult results = index.search(queries, 10); + double recall = svs::k_recall_at_n(groundtruth, results); + check(0.8215, recall); + //! [Perform Queries] + + //! [Search Window Size] + auto expected_recall = + std::map({{10, 0.5509}, {20, 0.7281}, {30, 0.8215}, {40, 0.8788}}); + for (auto windowsize : {10, 20, 30, 40}) { + recall = run_recall(index, queries, groundtruth, windowsize, 10, "Sweep"); + check(expected_recall.at(windowsize), recall); + } + //! [Search Window Size] + + // Saving the index + + //! [Saving] + index.save("example_config", "example_graph", "example_data"); + //! [Saving] + + // Reloading a saved index + + //! [Loading] + // We can reload an index from a previously saved set of files. + index = svs::Vamana::assemble( + "example_config", + svs::GraphLoader("example_graph"), + svs::VectorDataLoader("example_data"), + svs::DistanceType::L2, + 4 // num_threads + ); + + recall = run_recall(index, queries, groundtruth, 30, 10, "Reload"); + check(0.8215, recall); + //! [Loading] + + // Search using vector compression + + //! [Compressed Loader] + // Quantization + size_t padding = 32; + namespace lvq = svs::quantization::lvq; + namespace leanvec = svs::leanvec; + namespace fallback = svs::fallback; + + // Wrap the compressor object in a lazy functor. + // This will defer loading and compression of the LVQ dataset until the threadpool + // used in the index has been created. + auto compressor = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader("example_data").load(); + return lvq::LVQDataset<8, 0, 128>::compress(data, threadpool, padding); + }); + index = svs::Vamana::assemble( + "example_config", + svs::GraphLoader("example_graph"), + compressor, + svs::DistanceL2(), + 4 + ); + + //! [Compressed Loader] + + //! [Search Compressed] + recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Load"); + check(0.8215, recall); + //! [Search Compressed] + + //! [Build Index Compressed] + // Compressed building + index = + svs::Vamana::build(parameters, compressor, svs::DistanceL2(), num_threads); + recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Build"); + check(0.8212, recall); + //! [Build Index Compressed] + + // ! [Only Loading] + // We can reload an index from a previously saved set of files. + index = svs::Vamana::assemble( + "example_config", + svs::GraphLoader("example_graph"), + svs::VectorDataLoader("example_data"), + svs::DistanceType::L2, + 4 // num_threads + ); + //! [Only Loading] + + //! [Set n-threads] + index.set_threadpool(svs::threads::DefaultThreadPool(4)); + //! [Set n-threads] + + auto compressor_lean = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { + auto data = svs::VectorDataLoader("example_data").load(); + return leanvec::LeanDataset, leanvec::UsingLVQ<8>, 64, 128>::reduce( + data, std::nullopt, threadpool, padding + ); + }); + index = svs::Vamana::assemble( + "example_config", + svs::GraphLoader("example_graph"), + compressor_lean, + svs::DistanceL2(), + 4 + ); + + //! [Compressed Loader] + + //! [Search Compressed] + recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Lean Load"); + check(0.8215, recall); + //! [Search Compressed] + + //! [Build Index Compressed] + // Compressed building + index = + svs::Vamana::build(parameters, compressor, svs::DistanceL2(), num_threads); + recall = run_recall(index, queries, groundtruth, 30, 10, "Compressed Build"); + check(0.8212, recall); + //! [Build Index Compressed] + + //! [Only Loading] + // We can reload an index from a previously saved set of files. + index = svs::Vamana::assemble( + "example_config", + svs::GraphLoader("example_graph"), + svs::VectorDataLoader("example_data"), + svs::DistanceType::L2, + 4 // num_threads + ); + //! [Only Loading] + + //! [Set n-threads] + index.set_threadpool(svs::threads::DefaultThreadPool(4)); + //! [Set n-threads] + + return 0; +} + +// Special main providing some helpful utilties. +SVS_DEFINE_MAIN(); +//! [Example All] diff --git a/examples/python/example_fallback.py b/examples/python/example_fallback.py index 3588d686..f2653bce 100644 --- a/examples/python/example_fallback.py +++ b/examples/python/example_fallback.py @@ -1,15 +1,16 @@ -# Copyright (C) 2023 Intel Corporation +# Copyright 2023 Intel Corporation # -# This software and the related documents are Intel copyrighted materials, -# and your use of them is governed by the express license under which they -# were provided to you ("License"). Unless the License provides otherwise, -# you may not use, modify, copy, publish, distribute, disclose or transmit -# this software or the related documents without Intel's prior written -# permission. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# This software and the related documents are provided as is, with no -# express or implied warranties, other than those that are expressly stated -# in the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Import `unittest` to allow for automated testing. import unittest diff --git a/examples/python/example_fallback_leanvec.py b/examples/python/example_fallback_leanvec.py index 6bb8ed5c..7d93f964 100644 --- a/examples/python/example_fallback_leanvec.py +++ b/examples/python/example_fallback_leanvec.py @@ -1,15 +1,16 @@ -# Copyright (C) 2023 Intel Corporation +# Copyright 2023 Intel Corporation # -# This software and the related documents are Intel copyrighted materials, -# and your use of them is governed by the express license under which they -# were provided to you ("License"). Unless the License provides otherwise, -# you may not use, modify, copy, publish, distribute, disclose or transmit -# this software or the related documents without Intel's prior written -# permission. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# This software and the related documents are provided as is, with no -# express or implied warranties, other than those that are expressly stated -# in the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Import `unittest` to allow for auotmated testing. import unittest From 0d6777fce6e0f71d52db61c5deb28bae097999a3 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 31 Mar 2025 06:17:25 -0700 Subject: [PATCH 12/23] update examples from rebase --- examples/python/example_fallback.py | 23 +++++++++++++++------ examples/python/example_fallback_leanvec.py | 11 +++++++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/examples/python/example_fallback.py b/examples/python/example_fallback.py index f2653bce..5bd98f49 100644 --- a/examples/python/example_fallback.py +++ b/examples/python/example_fallback.py @@ -21,11 +21,12 @@ # [imports] DEBUG_MODE = False -def assert_equal(lhs, rhs, message: str = ""): +def assert_equal(lhs, rhs, message: str = "", expected_alpha = 0.05): if DEBUG_MODE: print(f"{message}: {lhs} == {rhs}") else: - assert lhs == rhs, message + assert lhs < rhs + expected_alpha, f"{message}" + assert lhs > rhs - expected_alpha, f"{message}" def run_test_float(index, queries, groundtruth): expected = { @@ -91,6 +92,7 @@ def run_test_build_two_level4_8(index, queries, groundtruth): test_data_dir = None def run(): + expected_delta = 0.05 # ### # Generating test data @@ -170,8 +172,11 @@ def run(): # Compare with the groundtruth. recall = svs.k_recall_at(groundtruth, I, 10, 10) + expected_recall = 0.8288 print(f"Recall = {recall}") - assert(recall == 0.8288) + assert recall < expected_recall + expected_delta + assert recall > expected_recall - expected_delta + # [perform-queries] # [search-window-size] @@ -225,7 +230,9 @@ def run(): # Compare with the groundtruth. recall = svs.k_recall_at(groundtruth, I, 10, 10) print(f"Recall = {recall}") - assert(recall == 0.8288) + expected_recall = 0.8288 + assert recall < expected_recall + expected_delta + assert recall > expected_recall - expected_delta # [loading] ##### Begin Test @@ -287,7 +294,9 @@ def run(): I, D = index.search(queries, 10) recall = svs.k_recall_at(groundtruth, I, 10, 10) print(f"Compressed recall: {recall}") - assert(recall == 0.8288) #assert(recall == 0.8223) + expected_recall = 0.8223 + assert recall < expected_recall + expected_delta + assert recall > expected_recall - expected_delta # [search-compressed] ##### Begin Test @@ -315,7 +324,9 @@ def run(): # Compare with the groundtruth. recall = svs.k_recall_at(groundtruth, I, 10, 10) print(f"Recall = {recall}") - assert(recall == 0.8288) + expected_recall = 0.8221 + assert recall < expected_recall + expected_delta + assert recall > expected_recall - expected_delta # [loading] ##### Begin Test diff --git a/examples/python/example_fallback_leanvec.py b/examples/python/example_fallback_leanvec.py index 7d93f964..48457c9e 100644 --- a/examples/python/example_fallback_leanvec.py +++ b/examples/python/example_fallback_leanvec.py @@ -21,15 +21,18 @@ # [imports] DEBUG_MODE = False -def assert_equal(lhs, rhs, message: str = ""): +def assert_equal(lhs, rhs, message: str = "", expected_alpha = 0.05): if DEBUG_MODE: print(f"{message}: {lhs} == {rhs}") else: - assert lhs == rhs, message + assert lhs < rhs + expected_alpha, f"{message}" + assert lhs > rhs - expected_alpha, f"{message}" test_data_dir = None def run(): + expected_delta = 0.05 + # [generate-dataset] # Create a test dataset. # This will create a directory "example_data_vamana" and populate it with three @@ -101,7 +104,9 @@ def run(): # Compare with the groundtruth. recall = svs.k_recall_at(groundtruth, I, 10, 10) print(f"Recall = {recall}") - assert_equal(recall, 0.976, "initial recall") + expected_recall = 0.976 + assert recall < expected_recall + expected_delta + assert recall > expected_recall - expected_delta # [build-and-search-index] ##### From edbec563955e31c089d4d75563e587c57f6e5c66 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 22 Apr 2025 13:29:26 -0700 Subject: [PATCH 13/23] Migrate python scripts to public and support fallback in tests --- bindings/python/src/svs/__init__.py | 3 + bindings/python/src/svs/common.py | 26 ++ bindings/python/src/svs/leanvec.py | 23 ++ bindings/python/tests/common.py | 36 +++ bindings/python/tests/dataset.py | 16 ++ bindings/python/tests/test_flat.py | 4 + bindings/python/tests/test_loader_api.py | 69 ++++- bindings/python/tests/test_reconstruction.py | 81 +++++- bindings/python/tests/test_vamana.py | 251 ++++++++++++++++++- data/test_dataset/leanvec_data_matrix.fvecs | Bin 0 -> 33280 bytes data/test_dataset/leanvec_query_matrix.fvecs | Bin 0 -> 33280 bytes include/svs/leanvec/leanvec_common.h | 2 + include/svs/leanvec/leanvec_concept.h | 68 +++-- include/svs/leanvec/leanvec_fallback.h | 4 +- include/svs/quantization/lvq/lvq_common.h | 21 +- include/svs/quantization/lvq/lvq_concept.h | 35 +-- include/svs/quantization/lvq/lvq_fallback.h | 23 +- 17 files changed, 605 insertions(+), 57 deletions(-) create mode 100644 bindings/python/src/svs/leanvec.py create mode 100644 data/test_dataset/leanvec_data_matrix.fvecs create mode 100644 data/test_dataset/leanvec_query_matrix.fvecs diff --git a/bindings/python/src/svs/__init__.py b/bindings/python/src/svs/__init__.py index dd9948e7..43469f8b 100644 --- a/bindings/python/src/svs/__init__.py +++ b/bindings/python/src/svs/__init__.py @@ -31,6 +31,9 @@ k_recall_at, \ generate_test_dataset +# LeanVec computation +from .leanvec import compute_leanvec_matrices + # Make the upgrader available without explicit import. from . import upgrader diff --git a/bindings/python/src/svs/common.py b/bindings/python/src/svs/common.py index a0c8e5e2..010ac7c6 100644 --- a/bindings/python/src/svs/common.py +++ b/bindings/python/src/svs/common.py @@ -280,3 +280,29 @@ def k_recall_at(gt_idx, result_idx, k: int, at: int): ls_recall = [len(intersect) for intersect in ls_intersection] return sum(ls_recall) / (len(ls_recall) * k) + +def get_lvq_range(data: np.array): + """ + For a given uncompressed dataset, get the difference between the minimum and maximum + values for each vector after LVQ-style preprocessing. + + This pre-processing involves removing the component-wise average of the dataset. + + This is not an efficient function. + + Args: + - data: A 2-D numpy array + + Returns: + - A 1-D numpy array returning the difference between each vector's maximum and + minimum component after pre-processing. + """ + + assert(data.ndim == 2) + center = np.sum(data, axis = 0, dtype = np.float64) / data.shape[0] + centered_data = data - center + + # Obtain the minimum and maximum values for each dimension. + mins = np.min(centered_data, axis = 1) + maxs = np.max(centered_data, axis = 1) + return maxs - mins diff --git a/bindings/python/src/svs/leanvec.py b/bindings/python/src/svs/leanvec.py new file mode 100644 index 00000000..d1492dee --- /dev/null +++ b/bindings/python/src/svs/leanvec.py @@ -0,0 +1,23 @@ +# Copyright (C) 2023 Intel Corporation +# +# This software and the related documents are Intel copyrighted materials, +# and your use of them is governed by the express license under which they +# were provided to you ("License"). Unless the License provides otherwise, +# you may not use, modify, copy, publish, distribute, disclose or transmit +# this software or the related documents without Intel's prior written +# permission. +# +# This software and the related documents are provided as is, with no +# express or implied warranties, other than those that are expressly stated +# in the License. + +import numpy as np +from typing import Tuple + + +def compute_leanvec_matrices(X: np.ndarray, Q: np.ndarray, n_components: int, + n_max_steps: int = 500, rel_tol:float = 1e-3) -> Tuple[np.ndarray, np.ndarray]: + A = np.zeros((Q.shape[1], n_components)) + B = np.zeros((X.shape[1], n_components)) + + return B.astype(np.float32), A.astype(np.float32) diff --git a/bindings/python/tests/common.py b/bindings/python/tests/common.py index 6459d3bb..0497c8c8 100644 --- a/bindings/python/tests/common.py +++ b/bindings/python/tests/common.py @@ -39,6 +39,8 @@ test_groundtruth_mip = str(TEST_DATASET_DIR.joinpath("groundtruth_mip.ivecs")) test_groundtruth_cosine = str(TEST_DATASET_DIR.joinpath("groundtruth_cosine.ivecs")) test_vamana_reference = str(TEST_DATASET_DIR.joinpath("reference/vamana_reference.toml")) +test_leanvec_data_matrix = str(TEST_DATASET_DIR.joinpath("leanvec_data_matrix.fvecs")) +test_leanvec_query_matrix = str(TEST_DATASET_DIR.joinpath("leanvec_query_matrix.fvecs")) test_number_of_vectors = 10000 test_dimensions = 128 @@ -123,3 +125,37 @@ def test_threading(f, *args, validate = None, iters = 4, print_times = False): # For short lived processes, we generally see closer to a 3x speedup than a 4x # speedup when using 4 threads. testcase.assertTrue(1.3 * new_time < base_time) + +def test_close_lvq(original, reconstructed, primary_bits: int, residual_bits: int = 0): + """ + Test that the reconstructed values are within the expected tolerance for LVQ compressed + data. + + Arguments: + - original: The original, uncompressed data. + - reconstucted: The reconstructed data. + + Keyword Arguments: + - primary_bits: The number of bits in the primary encoding. + - residual_bits: The number of bits in the residual encoding. + """ + + # Obtain the difference between the maximum and minimum values in the pre-processed + # dataset. + spans = svs.common.get_lvq_range(original) + + # Compute the max delta for each component of the dataset. + # NOTE: We *should* divide by another factor of two here, but there are some values in + # the LVQ quantization space that will exceed this threshold due to compression + # limitations. + # + # See the C++ tests for LVQ reconstruction for a more complete explanation. + deltas = spans / (((2 ** primary_bits) - 1) * 2) + if residual_bits != 0: + deltas = deltas / ((2 ** residual_bits) - 1) + + # Ensure that each reconstructed value is within the target threshold (plus a tiny + # fudge factor to help offset rounding imprecision. + upper_bound = np.expand_dims(deltas, axis = 1) + upper_bound = upper_bound + 0.0125 * upper_bound; + return np.all(np.abs(original - reconstructed) <= upper_bound) diff --git a/bindings/python/tests/dataset.py b/bindings/python/tests/dataset.py index bbd7c728..781b1367 100644 --- a/bindings/python/tests/dataset.py +++ b/bindings/python/tests/dataset.py @@ -27,3 +27,19 @@ def is_match(self, d: dict): return False return d["dataset"]["data_type"] == self.data_type + +# LVQ (fallback) datasets +class LVQMatcher(UncompressedMatcher): + def __init__(self, primary: int, residual: int = 0): + super().__init__("float32") + self.primary = primary + self.residual = residual + +# LeanVec (fallback) datasets +class LeanVecMatcher(UncompressedMatcher): + def __init__(self, primary_kind: str, secondary_kind: str, leanvec_dims: int, is_pca: bool = True): + super().__init__("float32") + self.primary_kind = primary_kind + self.secondary_kind = secondary_kind + self.leanvec_dims = leanvec_dims + self.is_pca = is_pca diff --git a/bindings/python/tests/test_flat.py b/bindings/python/tests/test_flat.py index 60990d00..9b629111 100644 --- a/bindings/python/tests/test_flat.py +++ b/bindings/python/tests/test_flat.py @@ -52,6 +52,10 @@ def _loaders(self, file: svs.VectorDataLoader): svs.DistanceType.L2: 1.0, svs.DistanceType.MIP: 1.0, }), + (svs.LVQ8(file, 0), { + svs.DistanceType.L2: 0.99997, + svs.DistanceType.MIP: 0.99993, + }), ] def _do_test(self, flat, queries, groundtruth, expected_recall = 1.0): diff --git a/bindings/python/tests/test_loader_api.py b/bindings/python/tests/test_loader_api.py index b77b28cd..5f4e968c 100644 --- a/bindings/python/tests/test_loader_api.py +++ b/bindings/python/tests/test_loader_api.py @@ -18,7 +18,11 @@ import svs # Local dependencies -from .common import test_data_vecs +from .common import \ + isapprox, \ + test_data_svs, \ + test_data_vecs, \ + test_data_dims DEBUG = False; @@ -31,3 +35,66 @@ def _get_basic_loader(self): self.assertEqual(loader.data_type, svs.float32) self.assertEqual(loader.dims, 128) return loader + + def test_lvq_loader(self): + loader = self._get_basic_loader() + + # One Level LVQ - 4 bits. + lvq = svs.LVQLoader(loader, primary = 4) + self.assertEqual(lvq.dims, 128) + self.assertEqual(lvq.primary_bits, 4) + self.assertEqual(lvq.residual_bits, 0) + self.assertEqual(lvq.strategy, svs.LVQStrategy.Auto) + + # One Level LVQ - 8 bits. + lvq = svs.LVQLoader( + loader, primary = 8, strategy = svs.LVQStrategy.Sequential + ) + self.assertEqual(lvq.dims, 128) + self.assertEqual(lvq.primary_bits, 8) + self.assertEqual(lvq.residual_bits, 0) + self.assertEqual(lvq.strategy, svs.LVQStrategy.Sequential) + + # Two level LVQ - 4x8 bits + lvq = svs.LVQLoader( + loader, primary = 4, residual = 8, strategy = svs.LVQStrategy.Turbo + ) + self.assertEqual(lvq.dims, 128) + self.assertEqual(lvq.primary_bits, 4) + self.assertEqual(lvq.residual_bits, 8) + self.assertEqual(lvq.strategy, svs.LVQStrategy.Turbo) + + + # Two level LVQ - 8x8 bits + lvq = svs.LVQLoader(loader, primary = 8, residual = 8) + self.assertEqual(lvq.dims, 128) + self.assertEqual(lvq.primary_bits, 8) + self.assertEqual(lvq.residual_bits, 8) + self.assertEqual(lvq.strategy, svs.LVQStrategy.Auto) + + def test_leanvec_loader(self): + loader = self._get_basic_loader() + + kinds = [ + svs.LeanVecKind.lvq4, + svs.LeanVecKind.lvq8, + svs.LeanVecKind.float16, + svs.LeanVecKind.float32, + ] + + alignments = [0, 32] + dims = [64, 96] + + for (p, s, a, d) in itertools.product(kinds, kinds, alignments, dims): + leanvec = svs.LeanVecLoader( + loader, + d, + primary_kind = p, + secondary_kind = s, + alignment = a + ) + + self.assertEqual(leanvec.dims, 128) + self.assertEqual(leanvec.primary_kind, p) + self.assertEqual(leanvec.secondary_kind, s) + self.assertEqual(leanvec.alignment, a) diff --git a/bindings/python/tests/test_reconstruction.py b/bindings/python/tests/test_reconstruction.py index c8e0e08b..66cf6ec2 100644 --- a/bindings/python/tests/test_reconstruction.py +++ b/bindings/python/tests/test_reconstruction.py @@ -26,10 +26,13 @@ # Local dependencies from .common import \ + isapprox, \ test_data_svs, \ test_data_vecs, \ + test_data_dims, \ test_graph, \ - test_vamana_config + test_vamana_config, \ + test_close_lvq DEBUG = False; @@ -38,9 +41,57 @@ class ReconstructionTester(unittest.TestCase): Test the reconstruction interface for indexex. """ def _get_loaders(self, loader: svs.VectorDataLoader): + sequential = svs.LVQStrategy.Sequential + turbo = svs.LVQStrategy.Turbo + return [ # Uncompressed loader, + # LVQ + svs.LVQLoader(loader, primary = 8, padding = 0), + svs.LVQLoader(loader, primary = 4, padding = 0), + svs.LVQLoader( + loader, primary = 4, residual = 4, strategy = sequential, padding = 0 + ), + svs.LVQLoader( + loader, primary = 4, residual = 4, strategy = turbo, padding = 0 + ), + svs.LVQLoader( + loader, primary = 4, residual = 8, strategy = sequential, padding = 0 + ), + svs.LVQLoader( + loader, primary = 4, residual = 8, strategy = turbo, padding = 0 + ), + svs.LVQLoader(loader, primary = 8, residual = 8, padding = 0), + + # LeanVec + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.float32, + secondary_kind = svs.LeanVecKind.float32, + ), + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq4, + secondary_kind = svs.LeanVecKind.lvq8, + alignment = 0 + ), + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq8, + secondary_kind = svs.LeanVecKind.lvq8, + alignment = 0 + ), + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq8, + secondary_kind = svs.LeanVecKind.float16, + alignment = 0 + ), ] def _test_misc(self, loader: svs.VectorDataLoader, data): @@ -68,6 +119,30 @@ def _test_misc(self, loader: svs.VectorDataLoader, data): vamana.reconstruct(np.zeros((10, 10), dtype = np.uint64)).shape == (10, 10, d) ) + def _compare_lvq(self, data, reconstructed, loader: svs.LVQLoader): + print(f"LVQ: primary = {loader.primary_bits}, residual = {loader.residual_bits}") + self.assertTrue(isinstance(loader, svs.LVQLoader)) + self.assertTrue(test_close_lvq( + data, + reconstructed, + primary_bits = loader.primary_bits, + residual_bits = loader.residual_bits + )) + + def _compare_leanvec(self, data, reconstructed, loader: svs.LeanVecLoader): + self.assertTrue(isinstance(loader, svs.LeanVecLoader)) + secondary_kind = loader.secondary_kind + if secondary_kind == svs.LeanVecKind.float32: + self.assertTrue(np.array_equal(data, reconstructed)) + elif secondary_kind == svs.LeanVecKind.float16: + self.assertTrue(np.allclose(data, reconstructed)) + elif secondary_kind == svs.LeanVecKind.lvq4: + self.assertTrue(test_close_lvq(data, reconstructed, primary_bits = 4)) + elif secondary_kind == svs.LeanVecKind.lvq8: + self.assertTrue(test_close_lvq(data, reconstructed, primary_bits = 8)) + else: + raise Exception(f"Unknown leanvec kind {secondary_kind}") + def test_reconstruction(self): default_loader = svs.VectorDataLoader(test_data_svs, svs.DataType.float32) all_loaders = self._get_loaders(default_loader) @@ -88,6 +163,10 @@ def test_reconstruction(self): if isinstance(loader, svs.VectorDataLoader): self.assertTrue(np.array_equal(shuffled_data, r)) + elif isinstance(loader, svs.LVQLoader): + self._compare_lvq(shuffled_data, r, loader) + elif isinstance(loader, svs.LeanVecLoader): + self._compare_leanvec(shuffled_data, r, loader) else: raise Exception(f"Unhandled loader kind: {loader}") diff --git a/bindings/python/tests/test_vamana.py b/bindings/python/tests/test_vamana.py index 8b288564..61a7e3dd 100644 --- a/bindings/python/tests/test_vamana.py +++ b/bindings/python/tests/test_vamana.py @@ -41,7 +41,10 @@ test_dimensions, \ get_test_set -from .dataset import UncompressedMatcher +from .dataset import \ + UncompressedMatcher, \ + LVQMatcher, \ + LeanVecMatcher DEBUG = False; @@ -59,8 +62,114 @@ def setUp(self): self.reference_results = toml.load(f) def _setup(self, loader: svs.VectorDataLoader): + sequential = svs.LVQStrategy.Sequential + turbo = svs.LVQStrategy.Turbo + + # Generate LeanVec OOD matrices + data = svs.read_vecs(test_data_vecs) + queries = svs.read_vecs(test_queries) + data_matrix, query_matrix = svs.compute_leanvec_matrices(data, queries, 64); + self.loader_and_matcher = [ (loader, UncompressedMatcher("float32")), + # LVQ + (svs.LVQLoader(loader, primary = 8, padding = 0), LVQMatcher(8)), + (svs.LVQLoader(loader, primary = 4, padding = 0), LVQMatcher(4)), + (svs.LVQLoader( + loader, primary = 4, residual = 4, strategy = sequential, padding = 0), + LVQMatcher(4, 4) + ), + (svs.LVQLoader( + loader, primary = 4, residual = 4, strategy = turbo, padding = 0), + LVQMatcher(4, 4) + ), + (svs.LVQLoader( + loader, primary = 4, residual = 8, strategy = sequential, padding = 0), + LVQMatcher(4, 8) + ), + (svs.LVQLoader( + loader, primary = 4, residual = 8, strategy = turbo, padding = 0), + LVQMatcher(4, 8) + ), + (svs.LVQLoader( + loader, primary = 8, residual = 8, padding = 0), + LVQMatcher(8, 8) + ), + + #LeanVec + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.float32, + secondary_kind = svs.LeanVecKind.float32, + ), + LeanVecMatcher("float32", "float32", 64) + ), + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq4, + secondary_kind = svs.LeanVecKind.lvq4, + ), + LeanVecMatcher("lvq4", "lvq4", 64) + ), + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq4, + secondary_kind = svs.LeanVecKind.lvq8, + ), + LeanVecMatcher("lvq4", "lvq8", 64), + ), + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq8, + secondary_kind = svs.LeanVecKind.lvq8, + alignment = 0 + ), + LeanVecMatcher("lvq8", "lvq8", 64) + ), + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 96, + primary_kind = svs.LeanVecKind.float32, + secondary_kind = svs.LeanVecKind.float32, + alignment = 0 + ), + LeanVecMatcher("float32", "float32", 96) + ), + + # LeanVec OOD + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.float32, + secondary_kind = svs.LeanVecKind.float32, + data_matrix = data_matrix, + query_matrix = query_matrix, + alignment = 0 + ), + LeanVecMatcher("float32", "float32", 64, False) + ), + ( + svs.LeanVecLoader( + loader, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq8, + secondary_kind = svs.LeanVecKind.lvq8, + data_matrix = data_matrix, + query_matrix = query_matrix, + alignment = 0 + ), + LeanVecMatcher("lvq8", "lvq8", 64, False) + ) ] def _distance_map(self): @@ -254,10 +363,13 @@ def _test_basic(self, loader, matcher, first_iter: bool = False): # Reload from raw-files. reloaded = svs.Vamana(configdir, graphdir, datadir, svs.DistanceType.L2) - self.assertTrue( - vamana.experimental_backend_string == - reloaded.experimental_backend_string - ) + # Backend strings should match unless this is LVQ loader with a Turbo backend + # TODO: Allow for more introspection in the LVQLoader fields. + if not isinstance(loader, (svs.LVQLoader, svs.LeanVecLoader)): + self.assertTrue( + vamana.experimental_backend_string == + reloaded.experimental_backend_string + ) reloaded.num_threads = num_threads self._test_basic_inner( @@ -281,6 +393,73 @@ def test_basic(self): self._test_basic(loader, matcher, first_iter = first_iter) first_iter = False + def test_lvq_reload(self): + # Test LVQ reloading with different alignemnts and strategies. + default_loader = svs.VectorDataLoader( + test_data_svs, svs.DataType.float32, dims = test_data_dims + ) + + lvq_loader = svs.LVQLoader( + default_loader, + primary = 4, + residual = 8, + strategy = svs.LVQStrategy.Sequential + ); + matcher = LVQMatcher(4, 8) + + num_threads = 2 + vamana = svs.Vamana( + test_vamana_config, + svs.GraphLoader(test_graph), + lvq_loader, + svs.DistanceType.L2, + num_threads = num_threads + ) + + print(f"Testing: {vamana.experimental_backend_string}") + self._test_basic_inner( + vamana, + matcher, + num_threads, + skip_thread_test = False, + first_iter = False, + ) + + # Test saving and reloading. + with TemporaryDirectory() as tempdir: + configdir = os.path.join(tempdir, "config") + graphdir = os.path.join(tempdir, "graph") + datadir = os.path.join(tempdir, "data") + vamana.save(configdir, graphdir, datadir) + + reloader = svs.LVQLoader( + datadir, + strategy = svs.LVQStrategy.Sequential, + padding = 32, + ) + + print("Reloading LVQ with padding") + self._test_basic_inner( + svs.Vamana(configdir, graphdir, reloader, num_threads = num_threads), + matcher, + num_threads, + skip_thread_test = False, + first_iter = False, + ) + + reloader = svs.LVQLoader( + datadir, strategy = svs.LVQStrategy.Turbo, padding = 32, + ) + + print("Reloading LVQ as Turbo") + self._test_basic_inner( + svs.Vamana(configdir, graphdir, reloader, num_threads = num_threads), + matcher, + num_threads, + skip_thread_test = False, + first_iter = False, + ) + def _groundtruth_map(self): return { svs.DistanceType.L2: test_groundtruth_l2, @@ -349,6 +528,10 @@ def test_build(self): # Build directly from data data = svs.read_vecs(test_data_vecs) + # Generate LeanVec OOD matrices + queries = svs.read_vecs(test_queries) + data_matrix, query_matrix = svs.compute_leanvec_matrices(data, queries, 64); + matcher = UncompressedMatcher("float32") self._test_build(data, svs.DistanceType.L2, matcher) self._test_build(data, svs.DistanceType.MIP, matcher) @@ -374,3 +557,61 @@ def test_build(self): self._test_build(loader, svs.DistanceType.L2, matcher) self._test_build(loader, svs.DistanceType.MIP, matcher) self._test_build(loader, svs.DistanceType.Cosine, matcher) + + data = svs.VectorDataLoader(test_data_svs, svs.DataType.float32, dims = 128) + + # Build from LVQ + loader = svs.LVQ8(data) + matcher = LVQMatcher(8) + self._test_build(loader, svs.DistanceType.L2, matcher) + self._test_build(loader, svs.DistanceType.MIP, matcher) + self._test_build(loader, svs.DistanceType.Cosine, matcher) + + loader = svs.LVQ4x4(data) + matcher = LVQMatcher(4, 4) + self._test_build(loader, svs.DistanceType.L2, matcher) + self._test_build(loader, svs.DistanceType.MIP, matcher) + self._test_build(loader, svs.DistanceType.Cosine, matcher) + + loader = svs.LVQ4x8(data) + matcher = LVQMatcher(4, 8) + self._test_build(loader, svs.DistanceType.L2, matcher) + self._test_build(loader, svs.DistanceType.MIP, matcher) + self._test_build(loader, svs.DistanceType.Cosine, matcher) + + # Build from LeanVec + loader = svs.LeanVecLoader( + data, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.float32, + secondary_kind = svs.LeanVecKind.float32 + ) + matcher = LeanVecMatcher("float32", "float32", 64) + self._test_build(loader, svs.DistanceType.L2, matcher) + self._test_build(loader, svs.DistanceType.MIP, matcher) + self._test_build(loader, svs.DistanceType.Cosine, matcher) + + loader = svs.LeanVecLoader( + data, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq8, + secondary_kind = svs.LeanVecKind.lvq8 + ) + matcher = LeanVecMatcher("lvq8", "lvq8", 64) + self._test_build(loader, svs.DistanceType.L2, matcher) + self._test_build(loader, svs.DistanceType.MIP, matcher) + self._test_build(loader, svs.DistanceType.Cosine, matcher) + + # Build from LeanVec OOD + loader = svs.LeanVecLoader( + data, + leanvec_dims = 64, + primary_kind = svs.LeanVecKind.lvq8, + secondary_kind = svs.LeanVecKind.lvq8, + data_matrix = data_matrix, + query_matrix = query_matrix + ) + matcher = LeanVecMatcher("lvq8", "lvq8", 64, False) + self._test_build(loader, svs.DistanceType.L2, matcher) + self._test_build(loader, svs.DistanceType.MIP, matcher) + self._test_build(loader, svs.DistanceType.Cosine, matcher) diff --git a/data/test_dataset/leanvec_data_matrix.fvecs b/data/test_dataset/leanvec_data_matrix.fvecs new file mode 100644 index 0000000000000000000000000000000000000000..ea76bc309996e42ddfb724d2651f05b328b33ee9 GIT binary patch literal 33280 zcmX6^i9?P5(`}DbDB7eVDilhho|$fnkSw8vL@8_5EXl4tQ7KZXzDiM2QKWiiLP&*# zDB1UYkC1rp&-)+T=iK?6IdkTWgN%$!-bqzSk(><0KG4RB`QPAD$1pfilZPIm4X{#o zBbN?tfRmeE2*({X@yCuApfO?)&3e=iD`_OYXgf}ywBqUfm1ej)Hyf6_sj>BxB|KxU z9H+_Z;S~!8iFRy7aHJZ|4`>nlTFmDH!_E9=)iQp$coffn`$5$EnSpcNRB*^~Kh&!T zhaO$m(X3;}H0-SrzcZSH!3kxsqN_JPe)&io-e!ek%O~^BLnip{%L>|@m5ae;zhKyu zP%H|1+IQUPj0s24~j1EQ*H1Pt!+VW!_Qb#d>ES zf`b%;-?r<}WR-=)@@EZfQ4gm9qr2ex>1sUwza+9691Mm^GI*!m9Oo~zXJ>Cs(Q{8L z1m)HXCP!zX-SbHpD5r)yB?cH1(uXV67vg+NQPk@j$pN#c;IV^KVS&m?niaGW&5YiV zN?YGU z(L*mLlC(Y+N_8!9t^Xjd?{}HhCRgE@-p66&h1q1)Z;NPFf0>Sq*#X}Mj(}DVAC6NW zjB6+M#jjm6(MeIR;>I|0VbI7y^!H6J?29cHD9R82he4>j78k2s<~~^#5~JA~H0J$g zG1$_Qs_yrMOSZCD5jv31#pd!@Cx4tU|EoATR2KUiT!W_mnPTRh^H4oi4!<7{M6F+4 z@yA{PJF>Tt-8yAnW~#?MA3F1~i47d@u8y}{4KaRs4=8(7ON-8br213YqTn9ReOKDz zX4QRAlY5XSXUlV8up?i58EDtn^Dtei{!W))F5pRe>o_QOEEF7{%mMKV_~A!Stmv;P z{K~$AH;fWcc5)mCR!w7#!9#HU=C$H$?LJ(j>;;W}({M}rP%wLZ4r^ET26}r&{55I{ zs`nkqUgnozy-V+Z7~E)ILJ$3q2zA}!aciNKgx8J3?B;&l^q(#H74PEL>A7fgco%*v zJ;2*0=;75btMJ5-DPrl!*>t$NmS#`zz->AP`0G$El!)@UPi{T0)tyF}mH)wn0A<{} z{E4tjW+0oNTERC;PI1E8TlA&b5L+Fu;p^pdu~y#*TSrZS-^RmGS3wVL)XMR}`dXf2 zR!YtGr*UBG5cIr}54*DM(Pg*{#%+s*a@FQ8j z8jS8*8#(829&Wf_C^mPUjl)vp>FS;3==@8++%H^)ZeDH@Lxw4E&SFjaUmctcJxOn~ z3!vnn1&=SQ7Vd4tAKg z3~msVO}x?1ya9?W9C6(AF!8d-VOqN~ms+~tf|)kqXs169K1Io)Q=plkTxg7=HudE& zM@_z{m5gc%kD*F+J#~60uvj-%{2vBc!)>voeYPmcza$?0{F*upyR)65vcx4l4?fi< z@`R?x^!-ygJ?-tu2|r)K)wc<7B3_Ted&NLOy})UjQP|L+kB3s9(&1%GvA54LZjqJa zC-PIEa`Hpy>y-whBx*QMEmkx-TuXX0^C;J~J1%$4fWtc5S-sx|oD{yE&DLbF_oNN< z?S3L{n4}IvXUEWdO(*awpDIKJ1Y*jIyQ2g4YV%1oZAjb}N-uZq!^_`y5M)0RdKwo| zYP3&f%JDb2@9KMqQchr7WfAyj0O<6}1n;sJd?K<`T&d9kChh%U?#EgD%S;#VWNPsL zFz7jZr(m!vLENxV18({(7ag5n+U?ZeLwUY763>fsC?wAU7ml1mlN5IFsaM=#B=Ej$Q~I_)K^TlX zX{f#oCM7mQduJVtYEiZOIA|)Lc0Mh5yL_Pf<|~!cei?J;tLLPVy@XCZ@q+$;t7+Gc zRO&wRx!t{v3v}n5^qcDKanJiJ!r8~y;qFjfZqKXu*B4ISJIBfHX4K@l4}%kHsOzC_ z!t1bE{P^GyetM^ur1#o>=;hGN*6*x%`l;`5Sgir7l}B=yibZ_4PLs3>-Pvf_F!WZ_ zwJUiSW0xadf%)b=u&1ILE1j}K1)&HxPW=HYryhzqa`qT`eK5~$jAGc?F22S}VcW$p z%=f#*ZmVS2fAxRl5ED&xGg5HfVkeqA_zc{C^Q2-~U@Ew#8?kQcP;_xN=Z-O}X{RKI zJ9Ra2Vvift5w{12eAq6|FZ%$w+Z%EHPu0pLPAegQTvSCjn>bq0u?H5_1w(~;8kL$q zXZI@(TsU?eZStJ`j}sj)9DmY-$uu6;to?%^E9=I(7h)PXw@EkM*eW;}wO4$?<{9 zOK4<#sjWexHTdr{B>$0{sNDPlEpS)_0a>LL2LwY%ui3@t!ry^a&r^Kj`-^{dklpCY z{2jtSOFhDfS;tQyaleU3=s1A>Q=l{X=r_i{TXC*YLIJBp7$GV#m`J zlr!!Lq@Npre)_VmM`@+gMzP#(ZJLOEjAwK&NKnJRw z@Ik;u(Yz@3lh!hG5mQ2cpXk zFMhAtkM|50!1GmaPTp=HY0Vd?f8_vpY?Mo9KI$^2l#pkS>0EAckR)^Zv(}&txHV=7 zuljNwADgV=B-;ygX0172O+P0FT$aJ2haD6bd5UF*Y~y7Q_R;!>E!6AAFL+s4#GfM% z^X`7~cx6&Hs=D=nYXeN^!tdXpxFi;QUfcw&zr*;d>l_-q_zKLH;&SKl1OC%Ch{Dv? zgH5fPs41pmW8PkrKlmJaCr%RISpS7c-(P~oJLIR~n^?&w4*Q(16ANp1(S-0Ru*E2V zJSw-EW7<39Q`X$1awG9EH?rqcF;N3^!$0bK4Fhpsy} z@-@#1I8S*1xWd|8lK?wj&WB zr?r>=-Z=rSPsSaB}28||M*-hMhCWq6^wCUuv2C_8njgwaD{;Px0a<;trY#*UEcr#>Ah=Js> zE_`OK3T)SZNH1R;0JD~ba4g^^6t;ea(-zaXj*KMfIrrgmx&bQO8VGpD1b2N%qGY9K z;1OlUE8{PsQAJm3x#_^Y`28Ng~q@}6ec=gU8ULp65 z652WigX;=tIFsp7UKc*HXgHVlTtn_!GF1heCsJEeDZi;ZL3hG>^QWUl(EL_SDA@5I zFYaQ1y<=P4%{=s}aq=%XOr-}>4OKQ=K-F%sG(UI{nOBG7kc$4e_~%BLI3yUICU=mh z#DljicZLD4?eTaXV!^bp!u~@REPEmYawl}da2!fw`%hpWyWP-ebsf6)s~4}`U(G$z zeBnV*I%U3^kLDk4)5mSSQ1^=rrq{>w@jX-V{U=%Y69IHZcO}ET+2ZBr`!S)tg0zgg z;6e4?)IGeJ-pB9fGWVw>^HC3!f3{RQ?GEDV1rnZA@<_PZ_6yY3tJ44UfxOjXK2p4r zaoHGLf5?|KJu=u@<%sy`Pj~2@>JGxeNbYr}jI-l<@Y#VI*gUflS8GhhKwE9e#JT(V zqGSTAUF*l=4__yP2`+qb+EJLdM1>#7nPTT6dxGj{j+_+9<2toj{ZBkM?EES&Sc~iv z`klt#$;6dzp|II^4%6gtK04&8c(6K!j=D|f*ej#?uc8(XA5ti|BxdunXFYIa&uS3A zO~aCyvDh?x70z2T5f9&;$tlHEWVOkz8l2uk z5r1~e7jIa~bILVy%FxN6v6Vai@j=az0AAZ*&1be9vuifr$;-c7BUKx((RwBW@#Lbz z^r6ZTXO`8{6N)6~?zNCJc@>)fmjL0CLbfR#AbETj_|%8hV7cWh^)t0%3zILfr^g%8 zRGJ0r94k;}v?J?|GiG7?0oZv-1$RUp<;#k_aGcIWI1)#&qN7rH8hsL;6cp3c-9JG# zYZTv4ND^|JF5r06FjQQOD6i6opX(ZMsrnpF2==tg{n$q{B|2M(6JYFN<}c6ev$(;FH%s_2f7HbtigSek$En!h z-$c4T%9zdDaYCC6{#f7vSHYWvj+7|d?}RBYOvK%; zXK14PS{%3C3|C58>0Ynd;>V=J;-pWZ>>j_4l;w=DLEV7wO&ur1)J0NUaWm!l#9~@l z5>?DQLv^KT@U>9UKKQsdjD4ld1A~UZ_}w2V-(dsa7^lt!jv01ozV?)NxE%Ew7xL-w zFzj;ov7lA3lXC(B!OQ0m95t^RO=mdmdgV)E%~8nm%Zh}hvFuB;EB zg`L2Or>;`?eJ)pZ0W@XJ3rFM z)j3ktFXAQjGveTDDY#H6Uc4EY2;!b(>b2CL2Ys*?a$=(KjN;KzuasU?cxjKJd|4O1 z(^3m^6HU-Ms)GI9H`2%nd7P9kLxveKT)oBt%U3PnCYXV#mm({}Oub3^RXA||P#*a^ z1Z9>S1-J1x=(SZZ*ggCZCAgJP*{f83bkq|ilP3y#+KvA>asLRQwYmRcce!KuLopZ^ zmfWYvzujR;-ElNH{7%^B^OJnt`}3j6ru;a+hP|%0kmEZKtgMcL-%gRDW_>5U&iW2( zzsHKVCK^h%En0$4gXB0*$)2~w8*slXw@GR1XQ43v4NMJP3_4rQs600hm3n6L-8)`r zbN;xnVr~_T=-wB%aRTSQ$by%9WO(NWd)~Xt7)F!|{G@sa7*1&xwvOvh4K1I=*;##H zX-^Rp|E9rK*Rj~OZ;pboKU$?1*b|TVBFW+&@?<$I8yG;XOs+uILiVWo!nQlGU`99IAsR6XXHZ8 z?HYJ3^&VE8J}TCqSHd4NPgBb<5|SRwMzb+JDu#}Y!`j7Bl{E`eq1M~h?nl;LI9|FE zciaob^Fs&V<5m0Vp57*G7`To#6Z;DO7sq4K=r`oQ?jxjq8pvNd)VLFCXu<6~D4N3P zovI1bFK3LpxiyoMiVS#RM=?IW-W59zo8ZrnDmcY`l5n`WC%*I2V?Xz?{J*zwG(}Gw z_IEH|t6MER?PiRIRb46YR$q1$M&K^Dixiz`2}QEo@Qdqm{9`qp7IxRhH=_}IhrOce z=XIdnaSZyOT>~|Jzp~2tQ&4~J6L=T(k-Rhj&Qe)Nu?K6QyTyL$tdEDZkEXC{a4bH4 zz6BpGd(R^#J)^SzJ!n?MKpZ^Pl3VE;)oqO>pWPF|L@|ZkFP*l%{HlQ-$p0dv$kBYF zql(M)#$oin$*dfR%G+KD_mT_Vd5tu0U?ITLe!X!DAZnb2!VHU4VZ%lpEH z)5~ssx%pK;@M#>v_s$Q*UxyEn&X>f09?F<&DLgj9p2wSJRhC_e;_nrQd5XUxo5)6s zv7S*F=^)FY9@ZSSGns}tS)l*>9CpxMM}t-@!2hk2I0|cz%L{t_`;PY9joi_vUQ5LltFajmO@u>Cg}p33IHm-4&`L=OIjCU z<<#9g(eqh*No5Nup9p&EWJq-QNdAM~>4WiwX!1mLZc_k-`z30#WtCntvD!8rv7s=Q7U=nkza)zXF3v zNwoLCfAD?aR_rv3N6V6Fpftw>PFQSZ)3p+~HSIE1F1lTr_|yf*R@rj2S{jYmXn-T` zSVHvlYMOO?6RkP917wF>rs;#LDO|3<b?~UmEFR-Khhi*_!@XzK{C7zXT;DhqV>4dzhYB@ZayuDZ2d?1??fcYc zLsz^w_!5OKE#u1jwf}mfOLIYG(2{gouGyET)&;^u{aKR6yNr&A@BEEf{|+k?2Gp1UY}=jrjG`h!mz;> z+sEfawV9TrY0C|IzIZpEvDO!!SZU&>-rD%JcLa6qJ`w7U3>Q2DEcmpZJuf{KNh5Au z$5ku7!QYQvQ6q05w)t+rzdtWvVlP8ntRKe_sh#k(+g8!p%S`I|7Q!u65r0qk1v|xL zT4T2kvvzjpANIT9qq;u(KC~qDc zd%B$WXCT@oZX)F)FX8T>x!`YE3pYko*^aR31bZP1d~XEs)NDVpmAGK=QaPztUJGlA zv+3LLH#m3y2KKpE0!P31vVV4U0BgHkfTGtE(Xm#9wqgSq6m%ah&e4#JJgXv%TiK0T z4YS2LIpc7vf)n1UU(QRzw^u}@w+qdCw$lr>x%hU!Huo(Vk0Tt__+f_%s}yJ2EnPGg zU(UYIvmYPkJL&x~@T^pWE$Yhmy8VD{ZKY(m!9tMl58U!Zf}AwHt1hp*v_|Lwpr`xjg~d|p?o_&ei9BJExf=l zq)c$wWx$s^Z}E}*n^5&th5I=!g5rcZG_2SdNnRZ*mUP9gm*e4%KgDr5BN+)lQ>#-MZlGbp}TgCXM;Aub}8Zbyu!^ApTb*8RKK z<6#(2`IQf=G7K;x)E&QVyf0|BujQs83$TRbaoGW>=T@*6*S|O+?0N1(f0Dw%=(8Ls zN)-ORg#cGy&O2+5ila|ZkJDFq#`QD8k1NCI>EQFIS-79O2lmA-uU$Fjl?Hfx?gkA% zOi^P~BzZ--(T%fx=+g2XLc02HGFzy@V^oil?PC@GI&%zue(WHGPtB*dCVlwixL+{0 z;g_&`S1frqb(36mFvWsKK>KlFbo*vs`^fv}$#-cPECtH)B8ANn zZ^^f@1(trD!5*8uNWDFj3WY542nfJqw@1P<`FM6d;zRn9wRq2TGS)uR!ynP=-0kyu zx>cJD3s1cnZP%`V-4aZ&=blyKLys)T{xiw8O+ig)_%rbz2HWP1qxS60+_p@gg9=#22nQ!md|w!rk;% zXb5T(mK#>t?Vk0X6m72)n+~DnZPoOxb}~+T^^!bJOu_#9lzFmu7ig}_K!al`WZ%Am zZz@;{y*5o?&1D_JZHxk|SH}EZ&Qh{#iUaqXra?JcH);5itCgLD)KGu#e?nW^4dGz& zH}TsHb2J%rlq`yRLc^Dt^u)r716Eb>44YW7=R|c(x%G>DS2Y3*OMw~l+QpOO8tIF4 z{_N~oOr1(rcHQ;;(em=DO3P3Ch2~HDQF*;Gt_+Ic-5ZL@Az7IXvKK*wT!XOZtTY%gLZ+#kAZafS9_)ZQ9nXYCmy;;7W)8mk+yet%yrKyu z+i2h93$)~LPf1zyGpb(t?DX2z>Rc8gHCHxXhi?ayxNFyV;j+Odypp;eJ}*q7HOnuN zuwy4j>a~z@c>(+wc!De!)rm>>Pr}eG3vtF&spbpM6f(Pof%3twxVZ8>=@w|Pfl4^T zg3ii0(cv^xh{F^*hy8DSB)`*TVA>%E|I^9?`^^xI^!o6}Y%i=<4})JV-Eomrcdv*v zmg>b_kZF1mg0tu2i0^uE!bA}J&K-y66moG#Su$O9-U@CMB{U`drN6F<$iGKmr@a9t zm}j!zUB>gD(s5y}8pN#Gjv)gdL1xHKX$@{em;Z=f&bTNVS(xFj4Mq|Jm95YqYU1#9 zY2akCliw`s6fcTLg)c9Xh0Y((!JxE~3~hP|`g;(5%-I3?>0Kyq!w}vq*M&;XH`05W z#W%mX3pq}c_~iXsP|8|5Zro4bzupp$d^#j-yD}5^pZo-;x0Ld)$WeIq{#b79VZl2-8SpnsWq&_&N}D>6 zyr1OrI>)u3a(@mNKG4N-U1ywM;3t@b50F?6T}rPzw?J!tHJ{hpgF$;NDQ}4l-aND% z?YGCV&#^c()VqtTy7sUeHfIFbYs68;-O2E1=2Sdy91S|fC4A6Y89zkjQ*-195G~EI zdcKkvSK?2yR{m7^^$HG+8OD9qT@Y?ha)DwuW$cXa{ttuwBWB{TPa#6T$459=eHZHA z%_Gl4fz;Ra47?9~Md#9n3US>kNOJoXg+F;Cs2pDg?Ne0Yib61Xm{!;#En5~%iwE8n?kgRj0v~0*-<$+f-|6DVBU9km zwR>>rioUq%yt(9aepg(xCdKY@v=y$~*$S)At{0l6-fwE!HIV;tl(uQUpxMo5?K~9@ zQmg-Mv1P0eD}`_8gGNRSI)7oSd?GxmI|gri1VY!=&17e>9epNCxn!CQx(FhTHz|U} zUCr74>=}+cIYF3}?fVadfWfP9{Q5wAZmx`dRQ*u(<`wq6@V@fXiKX~<>NDuQ^dSW| z$_Z{uEd=jrMxZ{@3zlUc7F;wHaML|iXm(E$-^R?r!Uz9h9xvjDKEtV>l`5w8zfAAX zCUAcANy=3_zLLkG?WZp=fFz? z<_h|kHgHa>E${w)Mhw|x1UZ5(mmW00^KK^4TYfihz7kHEYo5WG-{W|K^Kn|#vJuM% z?4g%4yWsIdIautij}WVlg?D?Q^UXVg)yk9b(q|l$T`a2Llild5eKP1cMgKo1epZI+ zp>iO=)mE-hT7J6+hOCJh{|Y}U+_*G#mB-e`Q7pe5KEivlHcay2E);ytTTWO8VBL2 zt@FfN?!9U7+h#}_e}dl~_CRC#Pn4lTeBE1L^3`srnBaFDE9QDv`lzU5x1)K|`FIPs zZP815#OK*;@ZcyZMwNIJHM!w#=R`59)F8$K=K*+)j;z?X`gTC7Q17QTRk zsWUOFI6y42|4qlUev2iG`rxfo4`7=i(Actz^x|dO6EP-$>H-F@qt_)kUVmzywR-(KhG-RX1}#iB#}qG)L8m)A&vsI z?g-st-Nei1{{w|B_rdY1fCaXGH1bF{&T=iXYl&BeQ>GVqV##jU8LPxA@^v{QaVA|J zE04)no&I64+BdN>eTOmrE|>(P1EZ;X4+9?1J099fU-N$BGax(JkA6w4c=rGu4A(1# zd(-=1>$ds$Vr(qAcqQQ5=O$P?45fSWI9OZq8}6Q72EV28hsUXAcwoRy+G8C>-{WMc zVTvL?c(9sBysZbnG9%ma@9W8JUmTdlX@J9q>2!$vDwmZ_<{utv*r=t;2Bm6}j|r=V zzSDBW>oYEkcT*3*l(h>0V|u~1?apiCudphp$3F(n$(cGVR+=PeuEu-Q}yS)E5MjeA)*{F&4v+(@mvB`;qfhsruz z@!*&!h~BpwGSADw$Z6lf_pFC--Y18QgC~;FPcuo|#S7Bx+5m7pp^DXk*3{iFl}5KQ z9;|M(^***6Lw>np(Bl1M*)5ZWF)FCEtv{Y_)8o|lS0ULk48Dv=qzem^Y5lSvcJGh$ z#LKUo=#Y{QK3*#IK(h1y`AAKbmDKy*Ot{mT4yL1m;Pbuf%gvWJ)X=4RM0nQ|5g1Z5kM7yMVpo zLvr+-#8HcXLWH5QdaE_)oxh2wIm_R<4dI%g;DUC|$9_fBBl`FfaY^s3^_)COAO z>xmZAec_1)vcaXLpjR>j1D%%Bk)a8K=Jrrdk6A?tp8Gj?d6U>}zaiKAAK?FKq!pV^ zihXxK#z`GHIN+}g=y;9hH}ZX0*l*9n=fv|Wvd6D8yx7h$nunN8<=X2PI54t_&X#Y6 zvf?dhcB}~3`Nu=ZAOlQKn9qJSxwQbb?z~%INst{m?~!EYK=M>N zms#})g~9W_K$wChJyux4-KKTNE$|#-gAP>4kKJJRHMkw#xb;I*eUZAp+sd14j*8_! zO~jUig<|=h7qG~v8iGe%gA1lz@ScYw%A9et{T~K>2Nv?@Jw{w-{tz5jTTrFmI~sPN zgs;?_@ZOvHXk2rE$E=RxlvPy}c;Fr^&b>t=9S=gfUkMdOn9w_qAF!+5lEcFcd9utf z{^KF_PLDLgjMU>K`$U1AM|42f**5$($W+vQAy!tTq>E-7HF2d*38eHGBX;m>h^W3m zx1Np^zI~|$mlxaM+{+#;Q!m{=ZQYHtZXAR7$L@CLoeU-2WAD=6)G+>f;1yl^YzmDz z2t%b_?Zr1maO!6ezl<-XJt?j5bnaQ0<+-o&$-q^tcKeKQ)$BH`ds8poN<1TYj4NQcEk}OL#uEwbKHJ-Asm(#GRp0GpeYu9V-qoqa`d?#WoJpLVuW_3?M zL0Ap`CXRU1#|7VQZWHpPd-%b6b~I_uWwF3}k<=e&P>C~ugcZH9SMMlx85Yj@B}4Ie z^K!JBp~UUam*7i-ASyXvB6+G&&XX0ZFni&8+#cCE z(=f*EUio4-izT?}<8}-?)JT4jM)-BzFz|go7q5Md22a%!_;wPJ?xbh%ew8k3t=)u_ zn}L#xWB8QIT&(%Nkvy6XlV`C7pBj^o7qs8g(v-6h{dYCYJNXgsRt@FApIbPycbRZY zKMtHz4wrw~svy2JK7s9t1!UnWiaVFfQmCmD%ND$%5>Uho4`X=zn@EZ{F^|=SsraYQ zNXgiN+whibrhx6zU2~&8>MojyZnJE7S@TXV{N|5ea|#44Ujw^EE@SwsT^Ec$X9IT^ zzrh}x2g9YtBI=%>gQwN3`M`+_|9a!SCn8BU^yWe-x9XOLVdqChEL;*sU(3|dt~C!f z`NpvK%^171^rw{d$r0pBbe4EvPcQ-h}pm}vHr*8Lz3aVQeY-+B-&73m3! z;OHI7=hg2~xupXRH*W>qsil;k5f7ionNgu<94D`yNUncPdELPI{3BC3tGzGhvCf7V zmDws83uh~b8NI_fQx6Lvs3K9lrzV8x6ZM+u$2+~0c~AHd zylveG>*pV&^3b!OU_JqdT{(`LPy;zDD)ID3L%1BX8z%Q#&6XV-AZ1-SOr?T<9!km9 zi!{U|1C+-N0?<7yOkO(?4!*ihZ$~fV*tz~NDyJP*UkALr={Cu&8cyp>uW;Izk@z+f-z*Xt@;QQXE z^rBw}yzyU5GP-wR-+#R&GIr&3W>K85v(gBS2TbHa7Ix6SzB|u$ucCDSA-t}ivuONB z9wUy{g31;vF5m106AibLi-I~`se1<7i`4${!K0kZboyZ_7%_v6|bV)l2)~F?j8(}*|4L>KyKJr7Eh7&?alRDdK7juPkHcB-v_Togt+3V^SoU)qoIVmp&Do~-j&D->1Hj^}A(HD>ainCa zz$!sU*{(zz_qSGaiZtI~WYYzwuNNWW(@9`2Q$F+J8|~cMA14=v(UaV9{NZ{fsHN!r zdke$T0=e|0bl#ijO!~_*NWol%cLgRuT74~5cPpioKDO+5KNfG7#a52l`x9cb28bH- zck%4GnRLBJI47o9qH4``8tk=j&ZWK ztgyXCthsp#I&SR~&9vKK()yX)wqp{fKN=|spZ<*sf{pm-`!x_&y^r)B1;gXe573zQ zhJ2Iy@bbLr__-mAVg}E}u;&jbRhP&i;mFvz>CMd?9Xd*hoR}srKlcq15TAF(2+HkJp6(_uLvFr$_s)h zX}y*>JGGO}J#+$@DtRmmzbEW|nZ*$_kb?eXLiZF4JierVeYv z@T!M8U+#I4Pbf&U+vOYZRO4B^R-+@{T-+`$UJy^+M=yqXnL+&Q;Cmtb_fqy89*XL2 zcBmEe0Xjp5(}Vjla5K}G{RKl_Y0vPaGN-@~it z@wj>KMxknkJge0Cv904_d=)y54_F7W;+6rt>TDbxXx8N)@@Z_bg#Oh*X;U`2D2-&R zk=YMi$=#qa`@owSk)whJ7fMp>_+KKRv^nCe;p!| z0kx!aUAo)2!|+GAK@#_USfiB4jyJ7%d~gjeR6AnV3=aHCVlHU*JVa~qtFd3EDsJpw zL_u2*g8hd(RG;}ASJ$iIokCX}ld_+!SMOkxF9|s1_EUH~Wf-PJwv$qMS3H)~M3EOB z@WMG6xGGCYh`$|;@tN~!{v>ZcvCWYbeGMcjEp;@n>nwbFU6ma#9_Q{s(_ntkIlF4_ zQ$VwJkmG@!I3{p3zYZHNK5K6I$BAKT*Fpd98ORyCS-PuFsHpin6o-yiUVNNO-T`sNlF0tmL_CR@IsTqS(OjUXVC9O3}anlF>0+gQ(7kL?2#zVqEerD2(;yw>{c~9QotW8D)TTi&Eif>C=B0 zxTkw^o-`+OGP0g@>L;`11SPmKU5BTy`z>C2{vXYpC1B|l0Ttc*(|PYbbaLw`4rnmL zx@}fG@op%eE@`LZW}k%fn`d)UxB||S<|Y;tCy9PXFOr|>5`Gz{Pm@FKK`vzk-}ltv z>=%Bb(XAp<#WnCV|16kYn}?N$4)D4LU95id9O_O5RQ7V-jxXn*qx7r3kT^&{{ZBpl z$D9iBS*ameuSkFycQ3=ZxBi^0t%8U0=LtubjN;N+OK8=*P@Fg2g}-ngiPND{{&d>_ zw_Mpx8sQ(Pu6_c4D{G=V3V$JHd=dF%I7zcoRsS$(JhYa5Pn`hW(~CKKoeuh~d;n0dDP`E9S<%1@nxf^k?!_9PMezCIz$ESl5hlqyirw~2KQdw4bRk?3bv2+@Z5k|Y^Z81nflQW5-T&rwuS^8-)kTCdEb{d_Huzh zzt6B`YXGmC()}MF9G*0jzbLBmf0bdhUfCQ~&OfJfvD?YySRxLQ?w_Uy9TL_|+X;P} z|A4`*K{Tk#Qz1|$o?AZw8b-~hdnE%n=&1siOnFGNk5!86^txhMZ(EEnTg(Gy8&J*8 zZ7lBpBJ{3NfVI~dCS2byq}AMp>;aZkuzUt=x2+Vd=h@=-p$l-4UJ{QQA3&8K*5it( zD=6-rBlb|g1x97RDh;`U9#2dYzO-m#bc{Tf$IlRrN9^T?l_ngzOB+jDMhj<;u7D@{ zDGUMOv?)*#M@Ab-iiduNv-@;;>-_E9-ILa#wP_zI0FFnZTQ3oUydY8oB7F5E>OA4$!v;U-x~`5L^W5t=sue zv>w`D%3)8ZTzE6OFWNhsvh^=L{Gw6_AHVd%uBjP_{{<(bdTc1dDKiZTK+~3?j3weKB1!g5Wdrl6{COr@ zuSus__OfUs?}6~NTKwB*0sB4cO{y~Hg4^LpG%9&6&S(ke2RAF=OOU!WM>!Iw4`_qr zzG?Wr%$<&|djz-L%3$@$p|rJ!EZ&@b&{JQF=dgCG=S_k5j^oF?) z^Q-^1(70I-p(W9kVzO>STvIsy8d}SNKKht?sDu7Y(7>iBN1pP^gq)(? z*T{#4dUwSU)}1i^z7s87t|j`vi-r7VecEWU802DWFypMi`Y91mXr4`mYpp4A^I=f@ z9b~KZ`~!@Y-k9yaq97jkvw(^&(mURn1K2Zi1{@eYS(H6!vN@Wn{(itnbp>hR>EfFFppP zVJbK~-xCr#kFcE04bs!P3{jh|!aYAd?9pEi^Oq@cw%<=k*=^0Q)s*?oqk~wcs80*N z?Zigk7(D&ggbnmUg@JctK)pHV)X=wa9DR2`+}3g+CyM|<6npc4ssfHaGZ7l9B8062 zbYbR4H{Lz)CFN#CU>_NCo|12jyT5v~leR0Jmd*l7gQH-aM=KplcSW3VnH*i$fQ_Cm zjv0_aCEj{i@KwOOHw{s|tIr0hF{C-%Q218=%U~D( zEY#vu^P$~W3GdELy6bU=%iU!(|CXXZkgz}~M_0e7WUmd*q?*koj2mz;7J1%N-qkuwB8r4z)vip9D6Qg_M zTbsq)FCl~5rJNmcrHh??ubo()7KNit;<({!0nA+N52`n9Da6PUOP{8|sPR3ke%<{f zEclfxbhkM{r`qhfP=7IgQ!0q+41R~>Gw(&kxblgT;54*$&91J>%ud}uGsx_b+c zcZ^2&+225?_7?k(@CEC*eiZk67#`POPKqajFs8GJ@aBE6`s0n-*DjII-hsSoz#pnq z3S>!eDSWX1Ae?FH#=UO8sN5Jk4excFgPxM@ymH~CSoR*NF|WyWg3#<#De@#g>E zwcU2vO%45J`D)&BsWyy8l(fHd)n(5<@H5yH))4ngqF*D?N$;G2Q{g^tc`gqYu`R(+~ zZ#G?78-q)3z7Zb`K1}aEd=z$v1%ha$LKS72sNZ!8N0vJCmwh`i>q8~psP2a@3j*j* zHv*mYdhots92`AVBmQ~)m-cuiLy*A%?$C(Ff?hGS_UdtIHl^4iM2+sSE^9g+w( zv)b{PRkx~1Ull3c>=D^aJjUN2ih@G)K>HupZ&CHA1h8}2hbiGBFmytHT)6NP=5BJv z30rINS*mu`i7m=JI#90an?VffF4@isiHSIR+6Z3k>&uxo8Z>EYf-pBz8<&>I<5;gv zaAH6j#(G&|VEl95{pJieIUa!pr-M0>&3WpEpB$AdYroYtQOsGWWWVJ_J^x=x*B#H* z|NZTe?3I~F8JUTAzs{whXh@~4OA0 z?$__X_kZ{CKKI-+pU>wJHBTYG51k_7G9$d%Ya-A>?GS`ccIZ0%Tv~z_u2Y$7*8vruq~p*mg6oQ9#$w`pofOPGv>Zi8GimXm z%k*Ws2Qwvd3iH3Z?C?bwHqGP$#)`b*R|O@}L4z2QJM4vJ`|9!A2QLyH*$$>2otUvS zjN^S+qGj}Ps1E7Gk8{Qo?e|LXc!>#y&&!9YhE?dGxRI7SI-%{#cW_v7gfC;S3%kE6 zGD;)rwBAdE%@$EXpI^=VGu{K_W&)`H3^U!D&OW1;OOn$fq`>SUn#B~jW zV8(t}uu}q!uRh1@l00%NHj+B7_h6>&^`jzA+>WsG8vYO!gozRVU|D%Xg@A`N8yc|; z{p?lwcFUS@iCP*yocIV+``_^T)5r5NM9Uy^T_>9U2uEsNf|Vx*(C&jRK6)a{c7)8v zk#WW3^}%YWtUig-%hYMgb4OS^o5P%ys*mM`>_ct%cgGgG`DX^G=B~$efod?tS_*}Y zw!yr|T4>fUgbP1K;G+kj_`W<4w{}Ef=3MT*1?&LaQ~)Ho6@~{l+VbKSczU`)IHHH+Vu@cHM$7$xu3S@Dp8n=P}IYI7i!@(!lS>B1m`3Cph~)%sp}d z%JAn{eb7*>j%S+?ej0j1^4Vm%>5n?NJLoY_rO!c(xF=cT7>4R+g`lQif~uoN??JcXY^Re4q!wJ`a-6xUztz}7E!p>M`t)S3MTO8A6L zDtTi!@Z$wIo}C4^UGlJI;XdyFoPdG9Rs1%;RJf=ngG=huVEJE3Xv}|3W_`W}cOJ*V zA<0AF*}%soM^fNLvk^NjLjyWah2ewS7GNt928SN(AG5(asRx)jL4^Ij^aTVujAJuS zD57f0X2}J0+4OjzuIqZUNo3&W7pB7i8Ku#DR~u3>MxQLdV;m zas8PUsPOhaZ*N{Q99?cnrETBvqwQ+p%m-~|?q3yXp3p#q?&>nCwvvqgx@!J^8%!`^ zR|YwC>^YKa&X_0Y#QZNeHf|xzaQ7`5Q+eB&S z6HHM#fl7O%Fx|U}zxk&;tA4hFKJPCjrSr4NE{@C6dw7Tj+nIoA?;Ci$--6-2mt{6I zXVL+)<7hCr3FN!dvA%8sS$L}*pZ^TD>$jcHESeXN!z#}3hwdfWAuJkK3NaUkJ;7N; z0nWLKG11i_thh>xUFh&L>YNZuR+-+W;aeKXr|JpTk3YMPhu38Z+8M|-HY&Mbr(^oEX7~nBcUx^gLzeUk@ijdjG~=+Waq7` zv_Rb#tfwh4jJi6`7>q|8UM6GbuqIPZ4%18%~VRHbE-SoE<&+cFYD#o!W_Ty)8WQ)?ydx2JCY}^vSBZyFO}jxT-8~E4r)42*-O7~Ym{U8Mqp&+Y8+UTg7Ma>ZB>Vzk_dg(U zLSpQ`s`Bu@ods58@Gf+rvJpQ zgSG~B%)e#?9lamP<_~2=?8+3hQvpf~VAFiYTnwdkvG1ZN(IyxAelyD4IKc zDzl<1kUyC!u_o&yD4QmabN{)(qxM*0*t>%M9{&woI7H%OV58MEP$715$=7k59jn&lVQC{#8og1WW`JI?$yU+Po53k ztKklr*-Q%Cz%u4*eysOKh3P9-rtU zFEy%gT!dK^{sfngdqQr}-m$&2M}mXS<1@*kQ$fy%>Y*+&PZzDNnY!OO$a+Sqj7R7qapku3Ghd0;WqAVeZZl z>^6);mnYtk_hJd&n!bikaqfj5*DN98dN)XI+(iqu9wTpF2w4~GhHLERa~RFLxOkc( zYn1jJQl^H%dQEpcZh8ugA6U@-4b6N58*W}KV@0Z@+Hr3ww~JaF3l(c8p;3CwSe>YV zjacw@K6IOZ!%dfLk(Zx}_mUR!sE9Ed1RN*c@uoETY9R@Xl43t470?7t32-P&g1_ta zVbyh2n%Cq_Hk$p1!QvHA;c*U{)$iknfIv`FDTi3?PTuM=ABdh=Kr3`=uxYmp^ZMac z5V%u-pb!9muB$Lj^J92gucBbJdpG?i`49|DpOS|KBJeIG4qN30(MmCy`b++RFlQj) zz1Hl6B^+OKgEj-|r#U^i%UEaW1KVS3F#6tUG>VzTA`PUbyAZk5k^Ph2(9Q(R_^|9@Uo}%Z>jzNRqwg!px%GwNUgV3#RVtfhjr_@W{1j@xj2L0ORBpg~#e7kpEU07c}L7@TpWtwT*+L z{p#QqlL_*w&)}YsD-m^AfT@Mg%TlVQ)A?u$(q$g7az!plKWzks`}%O+avjF*WeSy% zEe7@Vdx+liTB^3<1(D2X!@8LoSQt4Ta?js@nR5Lg5Mcpfeu=oq?ka6*v1EJV-x8?u zfN%k8@LT^5XEY0dUR)8>)QORy_6xZ1y9p|`zsFUAxj6Nh5;?F=k=b|Ki2d)=2mX|) zB4d8AEy$iZFa4R!|GN`37ENchy!D{!a0a}L7bn6GJs{D+n9TUf!uf))IuZ--9XCSsu798--Ub0dw|JJ%xjH`hAdC}sMUmitsP)2_*L-_3QCQlCqKz87 zA4=Znvfwf(PFO?N*)N6vUVj5M7iH#k{|usj#T{~s&6(EOi(x{p1a(%NCQ}Tf9 zc1;3@TL$d4BwxHU;tX{I*|_WRX(V=)IOCWQll&7gMK%?KThfrcu0+1;d)z&)4rd*{ z2#%Hslr&Fc<4RAWSmFv4J=ljkH+B$}o_tWMi6q(yicFTvTt*Mcqe)nueUCQ;z(p z_Zg&$Y`vjY6p(<4u6qGUA^&ySs9Nlh6P}L zrW1Un??C&8K5p)oLuU=i!^OXkLCPrztCw6sJMZ802KRdLd|%=FQ-W zsX_nHPC*T=b%Jnt1=0~(O;;=E;nQo%5Px$zn=C8Bl#yD9e5Os@dvA~gXG3=Fp1ovf zI-h^^`YKSvA<`0P&6s`7g-ok*9`Ah(wK^lqs^kVBsVj!YrF%hN|0%Z4tw%c(ZA{ob z5iMU7fOI7v?5dx^x0+J2e)|q2(m6OaN0#0F;|YE1Z%UaDUuoT+7~1nL5q565h>y#z zK-4q^(!OdLquZ56PQ4X}-cP3Z(NP){eKybuxB&Y9ZlWVEC$N)EztfYa80J)J6%2g6 zKxTf;;I9mSKnlf_SsT?6`nP#E`@j0Y^K%fWFV=+BfArWJ#i4ZQ+G;fV-h;MR%izV| zXnN3UkoP;HhH8Ilp?YiX+MSy%&yIU~0v12FVN9pYLC=R~V7ytISyKHJDo#%X8{>`m z#aaj_NVq7nFJ9Ut?>{}Ub#*h zq;ke$;v@a@I5U0&-tQYF(VIPCLYo3fh*1Td#o4sScp-c8!EHEu-30f)en-{|E@F@N zenQtb-NdDJ2k!VM&dZuK4_UGx4QY(TjU#0Ni)B)vcZ35yUCUA2o z4@^hXaK&U{cIL4&aAIaLY|v6;|8lr!-Rrh+c=$S8Fbf-t2O0_CIQa`7&IJ|2ltY%R zhJgxnG)a<)f>Y7&`&BCIdk`Dfw1ee)E25Ya0CU^5$VGi)bUr#6ADW*aPh(BNFYzm> zEM5bBRd3q;;gx$d~+(=mc#=1i7m8XkT62ex__!s>|~MEH^t zbNt>+Fq@N1EUNb1vX)}&WBOUDBF$E#@2%|04j@gIl3 zh{S*iUnu!hg2U!A;9M!i)Hgh$LII^nSRek(+Xf(UXdzZ_lVPK;MWNU1M81{7MKX9^ ziTUo9Obg}@V{~~k?Cgyv()Uke$_;@0Z^mHnbqBTna#~zfzvzM)2jGEL0!XitV}fO! zVASIh$!<)CJ!C5T$xw;8zM}%0dd4$1OEqB0zEs%Ax8StGipa7*%fMbHp0@fNU^{is zz=m;;>8Pa>PLr-9JD|`uY)Fay~mU^c_uw|HKtB#z?~-h$Kt^bZ)db| zxeCjsXQ8nf;2WdOsA%c}3yzgQX!K_qmGA_9n?0lxj88)7qyie6cOPm;ra-}yj+i2QmPj3%iw%eLJ^`pJ|&nwJNQs9NxEl z&;Qu5kPaO@zw_Il#A?fN2f^czUh^Mg}YRyXwV5OxQ)L=*=AxI7DxA6Y54RuBg~QcV_(Rv3d>^U7 zXTs)8Xx0hr=Cs)K+8<)Ylex^Nv>5o%@C5W6ddck#ZshQs#q@aCT^yMHmV~A5qKUB$ zxHVFgk&0c2%H0>R`Q=_FUjI25nw`RfTi?USTOUcwfg^DBhBQn!5QI^Ye2||}1ug-6 z{)N8jc%j=5zRn6J?iR9ar3FiGcy@Age;f8k+5_VFWMjp=e?K^U({12CvBVpKbz~R4 zfQ5Gp$FO-Gi(U~YuYB?=!VeOxb=57h=vLro$}!`AuWoQO|Qq}m9|*M%9a zN`GSfO^n@W-$4`J>ait~b4kr*E%0j7VKRPAV^>AYW9z?fW!79uq^7GQIBmLDqFxZWy9Co8iCN z19Al=xV_8RZn@7&{F6KjX~tZ7L^~Uvb{zq^B}R}x`x}+IxF6U zeRkAVRs~G`?qL3|Xkw}7&%4#tLm!$*v$;L@_?08uk!Kf=@0v<@6FI#i;jgRk@N8ii z@IA(^FR8(eo#EjB*cYRCcD(QQQJfxPA+J_>8Y&qmus6>RV#$n5h_X8l(?68r>UU<$ zypyxg@^=hK9((}5o1Azi?uyvHxC|7znk?;uB znr>2L%@fjK$lVWZ9$5&v5$lL^v#AfbchkUuk}x%;Gpz3q5{f**H76AK*l1 zyqv*G`}xq%88^ts(~hWftQ^+xrs8#7f1=ozfP!AyIO4C1#h!8?ySIt2;?aeS-*udH zLz%U^$mcy7_)f$7%TYhYnsNEEifmZ?4|N<1f#aY&Z$g;`XBhi#H0DA`P9qQp4zeXG_7$i|3R8N%sE{Gvutf) zNxul1PIf}=V0(75*eNKUJR7Pe9LGsVvcRbF8FAC`AiFI7Lzl}o5Piv><;DD^k#khp z6P7;2vVIvdn;2XpJCpQW?<2+yb|2J z?>ac#c*V`jFJtDj-&AIf1M65+M&ZX?qP=w{D5*|>94nT{pEm;UzGA%lyo=6Jl!mSU z_My1vauT+t0DH%;u$w3pMZrr3&aW~DSuZ*8U@)Z78E4TdDh~Iy0hqKI0`s#T){W0HyUA#ku>Ekd5hL!VBmcs{ot$j(;2mXU& zPnrm{Pejjs4VX7y1`|IOV}i#j@U9(icix*vatfSrkn>503Q_F-A;3(_9V8A3<8buD zFp*XhBEOC&L8p5VzOBAUU$+TU-?Y1=VDvQ1vdgmlEq4=!&)lPfoL+6Gr7YW@R)xO7 zk+h*~JbQgBU`w(F%zoyG@4rumiSJTTWmKNYc1VTE0Y>bx^P6$o?>;JAx*Qrzo{%}c zxx`Pw9G)jh;TulR-O1?z^sMBuRSi0<u&gK!b3&XTEYFtN@w=bf zi=L;SQqpbbl;0uBm!g5O^4!bf$I?0ETg%xjcln$2#L#me7N zOudcf>i$O<)h+07Cl{~Jer+e=8A;ySo8iarTraezA11!Yt;n?BNVYwZ$1LUty0mtY z=gt54w#V*)$WRVlJL?UDQ9@OmyjbL!!4BeagigDK11$n=g;PJ`?(Uw4KeS<`j_grAP$*|tPJjSYe; z7Y_4GHWt|F%gGS4y3JU-rswHZ$C8`b< z1ZsdtY#JQbvBhq#ZjswnMZ}v4>w9l8lpMMTcR4(ehDi+B{+uQ45}xb=Lk?r3+sIpb z$cmPgq=7GAlU4DXMVs$&a`{Cq$kd-> zHpu%r8x`2sFtM?m_^a)Mn;d3zz*CpGaa0Un2rgo7YMj$UXCBWLH0lQVYlUz$0B^i&!*u!T};MXawu<6%nlr7L?vJ6(RhJtIrd+j@_F`@2G~XG8wFS zY=sX74np_K6Hv`lf*UJc5L4Spsci*rt@lDN1+H(poezC`1xYpQht>{vX+uXpD$J{g zWcy}PL-h;+%uoadOeBx;`D&mY;=|8 zFjCAjolSU#)AZ?j!oU?q4fV1(?Nn(Y_KctvJ88NOG#tBvLY?DS{-i#7_v$Y4-t8kw z1@6I1Hc?PAsS;*e{G(dNwUC>-5!790W43DURKqzqsUVWSz(y4|JacD1 zomhfQ|34y8eVRBBRmYWtG0sj31y~*&(+Wh7dS2F zBFv0j!D+RLjoBc#`3Pv)y~2w56X@y%=P~LP$2;FX7p0ZCzOWu3z$uHHPxKSR>?Zir ze4IR3&(e>r-?70toV?`rIF7a>pd6Ng_TD;Jm%5ATajV7Y$DV?aVIrA5qm3+=?cwTu zW19bH4a!6=26-_t)^x)FEMFzSWEvNf0l%(_W15GlOX)Z!QTP&QM`z7lj+c}A@JlN&(>`s44)G4+w`wAqhVG%mRW)dI z^*j-&s>Syva`eWRYv9W5{_+J*LG-{srH~8LuAjT#^*VdM48RhSSCF2ffA8 zj9z^n|G}v^IKTM>F^QgnoAspFlgV5yrlbO+KTq?RiM@15;R&40|Hwa=?*+Kl9Ev+5 z@Jz-{bP7~s%J;co!21wfv`~`cNS4t<8hvml^#+7=6@h5WHC%Y}5tP2rf=P! z;FCy)c+xs8m^RFg$Ir2^=mv-XkBQyWM94z-3RK(Jit-<%KyvG2XmN4{P&8+=WK(!k zpR2;=vbVH-ToP)R?M0>d*L35|F1+wpojFqakZjd%;rYz)hbsa@uyl(e(|n+S(>XF{ ztaYxUnUgBB_}wn5t~&{4adpo8>-`wS@p)jaBzwAN4mSPCz$>vUz|)HJmxKxMs$?Eq zUUZ*$3dn)}uV~OR0BGs%!tW=R(oVlcOytE6^!Vj2ax#6CcYARkW=|Di1N%eZ$IYXl z6YCAGJKI3k;vD{SzYKXFj-zd}2Q=oX5XY8z%&)wA6iZ|5R8-G_%~wu$MtuoX{%#zr zmCO27*^RS|K-#_%Zy(mgFJjXnei1jaU#g3Haz4?o1??oI(TLZwA%V!as4^0fX8_z0 zl?n=olzJLyUFxd{yC#FXbH759K|BPxz2%)%SxyQp?(%R`49MNt3GvmQ)FQjxIV*B7Ipl~{z#1FFc1&{8QMVxq-1Tm`O$IF5$P-2zncCQ|;zGCew1VvM_b{*X2i;U#h||nu z*!hx*^s}WOJc!u^0R^v!>fKNrII<1PL}USs^YM~l0dMKF4`g|)FP@kyM0`%kvA&5y zY;c1)9#P33%Z=Yotfg=MEJOLq|47eYcepVA6<^%s9bRcGgYjp6qO<%xFcJ&JwVTfI zt!6#O*N+X^(0Bi!MpBoZ$A{s34R5+3A{H|ef9uU$F;XFt8ggR7L*EzB)Wa2SXqz|Tz2ryeSGD)k}boyK74Xpbr zf$@R+vGlz=8NaQYyW28_ouJxAt{yRCdopX#;Aa!PCT9iR<(;rx;N{r2V7>AW`F@|% zK4z?#ovVUqWzt%DyUCn2NEd-_GE8%0LYdS*TzxPn0qlz>) z_uXY+%|np)Yd!p_7D6F`IZy%KAP}|?)EqRJZRuL{khBa9GO~w+r=zfdtLNTWm64;` z@*GCT8>EEa;()srt((i?k_v0aW+;+V4AK1hJ~DLl6t1@B>W|OrY<~oo0uGnjZD08X z;??8$H)R#D=$|x%7#gtC_+3~A7R>Bq?4cf?FYX2xtQC{H1cy2Zh~^ZANH2Wvz8|-uxmgM>u1R` z`xze2&3y?UIX;q0*cTH2sFzv^E8_+cJI3|n3%IMH!A_5fhu?Xvcva*Qr!{*D%8b*; z{9x$7YWBq{23y;2(6{Rz!n%!vpcD877d}@)^;7}ow%#nO1ohzb{BCBj-NMD2}O)|%z!d0D8|*g&*&W9awAA-uH@xwxWu z1Iwa*!@{nh&0(*)|;j!jlv`dS?IbnIA_InW)TseeOvLirOp$LxN4n}Xc6!d;;%j)u+ zV2bZdc8bF}>gjLBdYcJ?SE3L0_b;nx+aHAw7gmqOgL@GfG$gkUEL9)D?9Cd?piK?N zJ`!OPOin<8zY~r;)MC>E+fX)G2TOuZgNKVgxa^LN^iz8S8#mc!lgIt0z7YKV<#Akpg&D$^BlWnUBjgh?4~d|FBFT)}EV$ozGO6)R)Wg z>T9I)ehtz&iLrELb{Ea}t0lfy1>mBI7;)LsPU<(`fhTGc;q8}k?E6s$Z#b&p^NW zOjn?V`9kcMF|U|2S(%+UH-s*~*gzKta+=peQ{Z3VIr7ZwEJ|Nk170Rhyq;hMX0qN4 zW>B2KiRDu(R(OWme%w1B%U4w4-ON_(8@|R@O6lT4F1)m!3d|FX?qq|rQ5Oy)86|&v*@wQ3mc(R>zmv&Nv;7n9;Xt8tu`2zP8zsENg&mib^ zCX^hL1$Qw|V%PB>eiqZBS4GM(_QHAc>c>Km$l|nAzsErQD@*>hQ>tv^cQsbHdLMi+ z2<8tg$_E2MYqZ<@k^W>u@oHi-#`Wz4yRUh0H7y<`J@r|=X-^=!I|`G<|B{90PQk5H z8brM*2Dc_kFfk#jXr`;kUKHbv#e+|8!)e?(2~w~?5rQ?AQT9+2Ok|?*JADgLM#;@3x>(L!0TtPDv12CJYaDJ3@;0 zJusFDfQR3sq3*IL``IoRcZW`5TffFvZ13a&|MxE3v_2K5CQU@c9nEBL#V*X#iNI}3 zI>6EHD-C@T+TF3Q_JH1!iB~BONbismg zoud;B*=3-!%6ItpWj#rdea-7hJA;eQ|FAPu+W=ArU*Z(AblR^cLFZj?W=Bm&$+MDT z5D$eeSHCS#0B_wV?&X&O^3_pHbX+Y7sSO~rVVqZa(s-R zIC|6-2lZAF)_atDA9=8J)C>0c&L_JZ8_DkO*$h_`;+Ee?uw@3lx8ypU%zp^Ya?{zV zyz`Xp5ob+PGjWMb0JC{$kf)!@^|YskNmz8W7z2CLIyp67nMZF`hq zr|=89&@P8ml&UaU!)wWKJQweubG>X?AvIdZ-5b(SfMhLA*64*0^yt;1z=13p6H$Xd zw?<W)9;(rBTAID7Q3KRgQ7rq%`X zP-W_U5;o@!Szn*Xfh|u$&dKAUafl|!v+NZYk@bGv57vos`Y~hWS_CL4V0zD9;#x{U^6n3|O88NzoWu zwsR|lZ-0(z`N=e3e-gj#-3csG-vzIWZo`7qH$>z5klmGCb@*n%Q`mf_9q)wJqv0_d zG+lO$o}M#EjW5muIbknQVb|l;k}r7el?+61{B8xUMs%JjM-HYZkLAY89%tzwuL_R0 zJ)k!EQPf!^9VLaG@yGk4sQPX^D>3|<-nPuLYvAxY2k$x3OZ)$U-i3F#?2#aQ<>d@s z<3=lRleGo;8@AlLF5!0M3A`Hr{a{ua1g_5}!;9rJAxx(fUE3xwvI`_o`Jyz_acdWj zo{7YFu{pef{o1gYSBWQ&Ndrp^_+nqA*rOXls87lQm|3)rFX1N5+_H!wTSv2K=nGwD zTCy8ttr*9|jYQ&vm^vtw5@n9ZmV)C^VMw|E6k5&|f#H#BIAh*t9ArajuE%5I@Un~! zXV~+0T7AYpebadfAwERo(@h9{v1QB;X1+Ef`}{BSj`e9kvqd?&ype?6{%#0Q9^(f7 zefaAf1EJ^q;LP-SxLKKQbN`zD>c-vtsDi_#Dk^((uDV4fdZ| zHE4xQV5ZD}k5xG{u_P=M^zYB1$16D9h*b?(=KBoX7f>vi5CnVoJ%e#0*P!FFFteW1 z;0WR{DW_L*bKZ&#_||6}dvoV{6jGmzo;7*!TkRbL?fFGJti@?vq9)P$!^2&+!nk|( zGyD)=gF^Q6nOk?pGj&(vFr!QqdX^TVrR`}Nkf((*E^DFX#s#{xz>3v~Zlt2kmDsa# z`Pd9)U|1KEe0Skm=O8HHVr$6KJhYf7&RXohjbc&Cc-BFXEfBc_uN#eU_F+Zlgqk3K zOYBS>S)PttF8zeqlxCFFIYT=B4C7NjWBA>!fbH4-*ga_th-8*hm~=H@<}v}38jky-S{h`E>dZs6Vy9hlHL4klqj4tW_eea;i|Qnm@;!dxEfXS z_b1HdKYCP-@8qrUam7jW7at}0yU(M%z)bd;UoYhyrRs~$|@*VSot7UHVXy{rneU=Z0rcTC# zyUVcSP7ci}yMRw8E3iUluB@YlHHOTW1Am`K8Ek+7p9b)hdqb|?RG>DO zg@89B0CVy=t<8<1WVJq$%qMD08`sbK#J9j7?d7<)@Kr^(o+8_g4^Xqo7tSBA1|rs#Sa1S(yDyuOV>}6FM*tr++m?{-^lwzCy$t%doreYKf57UC z2RMsrflQ|hz^4-seqjaZHoJn??eQ45`!=#I%52RhABd`cN-pYpL)t$>e&uLlNHs*RoB52;aLG}72$6_LL#G27M#B%y( z9Ny{L&vMfSLUQ7isE6?rT)Sl_-|}D^`j0yeZr}unZ*} zcEF8$XK13(5Ujb}3m-@Rf%1(oZm$(W)GuFwxQ7uaB_xjDOAFxd^51mxw~J7$E=T6w zm#j(VG~FO50b&gp<;fOC*A zd#-XVPie9d#!edtr@su**^|GH+2Eo~HR`YT#`|kWae=5WxJ*3`^=OH`&f4%HSBp&@ zT?_$-jnKky5@Rh~0vmk#$@<%0z(ZUg0=Ql-yKo(|zXSU)-kH7 z?*VEZe^BI_I`j0;5lC{cBD)@HGo7)Q>8WRpyiHOJY?i2{T9-5E39~37xG9Y1GISM3 zoEkAzRC5obWcbBLCE>#=Qb%&^z(An&%}5S*Iq27dZq zNzQGAw49sRJO3eXlCCz(tyh3UF0<_ zJppc!x!@Qa&A)Xy9P>q9q7PPcniw|N5G7 zjN21Mvd2Xke$Py)_>i^;rio4j<-&7NN8aH)PTO{Cy$#5&JB#DDe8Pjv6xh*eV$7Yb zSJC%!7*%KFD5;!*_tgfewDb~8R8GReHam`&xDNm7e8Tke;qc~@0sB{T6TOw`3%<9L zF;;&Xc2+-efsA?wWu8YT%?OpMrXP2 za}ds+TuA$CPLbUM)!6-TH!5zlrZts+a8~+RI_*a=o}FS(Ti48i$oHD~VbUQSUiXLC zIhn&YjsqV=dr?R%g=XJqfv5N4h<%AG-T1?l>C$)vSC`un>A-BjpxJD1p9j1A<}LWU zR0py*3bWmf*?5&_$t+r;2a&5X;hPE%4t`z)E-x;@r43JLiLW%)?6?fUu|?PsG><&< zo``E*O~{+K&G2yNMwIyRXEi8poCgJ#Fr85=@ZNr5|W@Mg7 zAAK1413q0&0ofmB%uAn}Tpu-wouPXdWuvE1#hM2&LFXH)B;a#X1IGw_h`6E4oT6RLsjI)F;zX85G$aBk4!V5*xiL^ndJm)C4ND~ZUJ1R zJ0A-7EP{)i-qBHgC6azw3}=`=CeDJqF+cb*jW7qycrdwZ0!|0QNOU63b;-iD3MJHo z)6*Ty$|j$<+Zao&vuRb#67qVkBoRBPgtz3(aMPZTL`Qy*?3h(eWtYXkhTv&1&{&GY zv=G0}wS>~3ohV>+6~-Bh0{dqvVNQUPR}6!75&9O zuCM9+KU!?FjTA#Q#F*bp?O@u;UR*650tImiq~}U4$C;C5>Wnp+nfo;$F;Ewx43t@K zr@ml`?4jRkC@%fTyvtAkByPn3CcO4*PQz(RfjzWn@JuvH6HlXZ$`S6qk2=n;+rV77s|^1Ktg?35 literal 0 HcmV?d00001 diff --git a/data/test_dataset/leanvec_query_matrix.fvecs b/data/test_dataset/leanvec_query_matrix.fvecs new file mode 100644 index 0000000000000000000000000000000000000000..882092169ac5796c008b4bb645f1ea43a8c4c88e GIT binary patch literal 33280 zcmX6^cOaFI_ZBKU5wf!y8b%B6Ij&MFiH3%jQBhKw8X8iPNJ2=7RFqPLaNl!ON=uW5 zXez0s^igU1`TG5N?_c+Q?m5pn=XsvjNk&G7b5$hrYX-pf@QJwhejy|zJcP%$kDz35 zGmLeP;NkM=5ZQaBP_s=R?}pXFka3e~*ipnauiP3TFN6?to0RBba$oSYa zwpggcjWxsY@QV}(voRr6*>3_$hf(B^=^TA~5(jJ!5*~j}2h(Z!ocx~?`%dY>cMVTTSDe3q9v^I25cf$NYGk;+eI);!Zo}S+ z<>1tT!OFN0e9<~cvbXF6+?#QfE<_pP-J=SeoPC}8%a4MGF&emPN`K5;V$1J8{}GK3 zJ%llugJ89@8?LJGz^zlPaLPYJteC99E9S07-TFy1|DO+c`8@%XTPMJpnGa~ASr96z zJf;3C_wu9u<5>Gq7QgBmN%t;35G>k_VAU>p$B-H=i1m!*V}pjFY~3X?Y3q(NeYe4z zn!)_V@glsPbe`s{wT4TJyl~9e?yNgIo8sSAVD^l3xZ|uqe>q2t`yk70y=H?__yMVg z@@Uq7-3y(fMqZOB9$Y;@T^c(2AMg}); z@;KTbN@Y$A+9&n; zx`r+7Mq{aO0q6>c_}9%_v^8cr|JhU|3_q?!38BXPaKm=~;J%FOBV$4Jo+HoN&;!rT zGs0VA6>0E~T0DJcC%(!`=BKd^T(N!}CWynROi-Otb^zK1@WdZtyH&PG}`aCk@%e&iYO@Zpj)z}dSxq*>Ys)K zr>sU9#aQm8uZF26YtS+2v%|z!L9{I@j%F=fihFWC!Xo!bIJ==MdUg-z^A%BaWL7O` z|4jm!t3QOqLUlg;FNn_=6my2m6MBBL7oJzHKpf+TVL|=zR7fP0yW8NEb}h{KmxGGK zDp;x8Niy*-L0L69T$fS`FB@m$l=%Y%J0Ckt?puhLzc@Qw&6$U9Z)P}Ti-WOwjTFp3 zG*d`GH|()7g7@~!!2g=;M1_S5@Z8!6k?Y+$>Oq^-dk!H5wL{!&I~rydaEW-NyUulfA7aV+U}@6TRC-FS<RM{)d76X#wZSn?gk8$I@JjpLA0&=IMFeBu8h> zgKHjfoRhD~lb=nd`9b6O)096@HS0K>`MJ)aAUYdj9@_HHjeZ#D&R5)eUb__aw zfe(~Fq3YCBnEu!r=gRDYuC0dX)}>k8>sv!FOjl8p#uxBd`yIB8_T%)ZHF)B}3f8ZP z=XVn(P}|~1qIZ7{uPik}U zhgLZDd8&uM^&Q!+Ulk-?)4}_$PQ3QzARgmg15^F4Ll;<1YaM@pvP~B3`@RGAJ#^wz zR?4Ex$$Qe1Ci&v0iTVhhGFVdo1WvDhF3J|zaKoC9g65D4+VC@59JS#aebd(B+uQs^ z7(Nn$a~i3*bu>k;NGU1(lSndV38Zt@0Q-K45YCw0f<8-LQ}WZYPCmG|qJZzV$B8yh z(O7%)9vR)u5cW0B;~t*oyx2oS;xJ1AR}|mpu>Agf?6*4J_*eofYi-%=gA0$>tZ=Y6 z9eg_e$4XthQaC*0L0J5zg%C?Cif$F}m+ z)05~=MY2$~D-dt(xx{(vim10|BxOHaPj18a;i?Xdq3M-sx3~$-xFnNTIfZn4zI!A4<899vwY!4?Hi$Xdptc09tyn6 z9M+F5p+>QWdA$ca9zROYT0A=ELhz+)&__3eD&j40=*YiRo3j?5r`>P}?k8~WnGD?Z zPfIfG%r5rtFON<2$@t>tLaf~6fqfOcStD`{gRLX}&L78@;{HN<{#r^nsK*9@hIDey zTH(^>2{3KvOx&{iG@sG4aL}2rQnula4jy+e5honq1`*Mk7&2fJY`sX@FaZf>kL!pJ*KESW%1QQOG(=F9P&(=!9PxwNmVc3 zfatw_D0u!2Q985>cZ;1N&W~S!6My#PQ`>)1_QSdOuBVmo%)1*x!XvOB+Xdt9H$u|u zC~?#kCqB14n-h9kpof1rRrNgrF((g!R5py=Mh!+kYj@h{UP)IbZ{aE*WxSUd2P{qDb3EjV7XB=iXzQ7fms=c7;b8pmJaf&W{%X5(KnGO9)=5u-32m0)ek8^undDhIL|~pF0YN{Cex4khe;({{9i#6T@)P z&Y9G*(1Q)18L)KqO)%>FlJatlBz`**iK2aB!nm`f9N&w3hi#!#u!74bgj4*@ek>Pu z5t8niu;Zn>xGp<{g_ANIQK-YASG;I*ggmM>sj;)h5pD@x%2zH-p_RMpD9^$IWnbrT zvug_Pc&CmZekG%(d%B>mtU}hc9UO8o4CF_@1^toQ9JWh=`ib9Qz1sn(EUxCaQe9Hm zc^ZZfYZ8<1Wn<*sFf@7b5uARHq&=gCU`g9M;mJyM4yl^Nxla#ZZ5LU3sQ8TTEA)oK zrVxJ8-?Fn8iu<31-y2eJ&d6_~sTNUe_hf2b(+UNux}Y7VjG&T%rCu&HX~cbqIoL}Q z6nmeJpWO~2X(g<$ZHQTgM}$Q$11YvWfO&pCHOpGz;TyT|(M%K*ZyX}?CHKMFst;CA z59JAwHmL0#35xaG;f&Bo+TpwCnR5|Y9C*ujuKW^qT^j&~vPO92-f!{4svX?tp%QkT zpaYp*_VJg-V{mqN4wTzepxToPFgj(#B`b{}e2PZLOxl9G9dB~;!Z3WRbPPT@{(;!!OIe|B&c?%(XJZR^~=aAiNDkMuvVBN>tpmEQM+s~UwW<|V&xettSp7jFAYVU@v z88^vsq9NX>vEz$)2`ek}h060EX<9<4=2SE$%w`huSLY*r2)^%{>(@$;zbXJ720 zsmwg(109Q-DHhHvgbN3*)BQ?$_G=l%%XE9trO5u)-aH$1W{tu}ho|zkhsr#l8_>A7%h{=*E5hwPB)Qm)el<%`b$vbT zUiyG)OjYsWXA|La_d5F2IFeeEoM}hlrcOTSmoSQ_q)GUT!Z2Z@UkdwX93V;V2dR&h z9_Dq-Rzn{%e524p^%{}1HYW(KKk~)_zjgwRKFeKKnMu~HLjD>ifym#dY3m>Z4pVG{ zJGY9cm+=)?WLJRqrWkYYJv#j6Ln5gEcOTYlOJS4C)+igB4kI?!!MHKIgmwN8!LX*B zUZ~69>S=>`zgMU*ZrM51)7^_Z6*}&)Q#zcouLtj4ZpE=PH%ce%>BfijEU@qLuH5FE z$boU?qG`8O^19rcAM9}=HJbm4@a@zqje++P~_?Wb$oH}0~uQDWA;}AJTp3i*7^pEjve*sIc+{} zT6q{YIZkjeewxXHpBeGb0|pXpwNs$-d>irU%`}{Y=*j(6obq3y@ceW>wy6Kd83s{O z&9mpkbjA6QpYugLcH5V?H}vD=D_>!W!*npPmWoNmI-tEc39-tG53e7CpN-Ezk>M2V z`}`oNX&JyhpgcIt%F@@3kb+l>T4xqMUofw2}Iv|~pzmtDmosZT| zsyL(%qD+_RMw}H6n^KA;;nkrgn^d=RHwfs!hn0B2sL3L++9+LT0D89U! zc4jH?KFw&X@?A*d9Xym8Mf){#eH!KFr$1lxBWRMX4$_G zghjo?7YnSpWbs(uBsabjgViB2JY}pl`(zD(R*ki^$F>N5xm&W~^B&y1)eu8{Q^@dG zVaeljI~;zQMv+a4Ew8rPOuD@gulMzs8s?PAKr<%E1rPu%5I{+ z=`P;a)`O)nZ|HVR8g8j9!Ke3ag$ufgIQN?tsdemytsRH{LfBCH z=9+-k4U8kBmSAc8L2@1RU8wAN0iOlG5}a0lroj2XDg5qNN@*_QZ%SV&V5Jrg5ea(T ziWL3U{zrRHDC1jGQ~FnJAbD_h5w6fUPG-gZ`R+!2-hBK36@eTC`&k^z^1S$OA$aUf5i$d0 zsb;u0?d-Eknt&OwYSu+aXtT$sK5Amx^!fC4-b`%v&E#iQt8w)5K%D$9r*W5&$qQ|}&P7hN+hsMhM_#P?6W(VDk+)nNsPEpkGs_Fx)F^JsnYfR>gaHLFt+68K&Ib8VZ6;We);RJP`3C4RYe<0 zf_lZ`_?KTx%?zX9^@3d>`4CCX`PPDhvIZ(mcc#XY3J2fIi_obxPi$#QM(!IW=FL0_ zLzi3>{QktjS@-2Q;n!Nc{2kEnML8Mw@j<7Pft)th*WO|A0NlLqGDU44fF)sSJngv& z_f0-X4Q5gp?0+7!RP3N{(Qm=+N*XsD>dgmbkK$z;8I&Kdj@NZmFu6WVd?d8PMdv{r zofpXetAnz{v!(0eH8Hl{Nf#AgqI#M7Hm@?;6hhCk&#pS$zgKYBFpnmR82Zo?a{{iT$&Od5OYwcx!qg}=t_ z6~~n3lYi?+s);q{-~$ybw`v47%^Aw2b9KPMeF`^8x^p8;pb>AH>3Z`FT-Q4j6G!Q? z-^mj2*S?1OF30$rjXPPsFyisO-wU^I*zybCfjBf|1!b)1FZ~||wFx`9c#J(qu0Aid zP)y(im7@&i@~nqH#Su{fXsT04NvZ=`H93a`H{pJKV?$Uyo#W;H66yMUF%TL^Pg z=8EVbNVzx$SxyUo)+=GJw7!z{-&$ywMVN5)_!=-%kiw^nw*|eIC-By}iJW~Z!(qqp z^YmcZF$!tg!g{_^m^o_>C%k)2lizK`j2%bN$7U5Y{k{%{b+>4?gBWJ}pyC?glZL$XiO)`dP~Qxk{Bh9*Hd6aTbR9t4y7k9jXy!xSR zRHh}-kiiWrBGn#tO&bTZ-Yno}zj{D(^+nY1tP#78 zw8IgJDy+IFfa<3kVgJ;}LeKX{NN&R+YPm2JdM><2TOS;!`_GIeo1T_Ywx=82`uIj# zd3-O;w~phC${92w5Q%&LqL)=maq*;Z!QIy#PTK16*4HvHDNh;iA8`yh?T!x|V;31Dki?i_&IMZH^*_TDq};G@i!32?vd{rCeC;2S1zq@ywCeJmzW> z)L#<8XvREVU~rPu%NyWTULqB=Wb)C0wVirm;8F{L2l!Lde@Z-A?F6hH`icg>cH@$z z&LpG18**Zv3t_WX;DkSsbk3lT=H(m~_1c=C$k>~!UFPuZ1Opzq#tM=~9H&oi18`B3 zx#-sy*e2PAJy#FHBPmJHc1TO2zzMW4H=Vo2jD?-4gYfQY9b9QKn9d*afd-EtVZp4i zJXqO^`&gIJmh5{tl`G)eIxT#aG8xZU&+GVn4)brTV}(x?Q^P5^tI{kEy;dZy(SHZ6 zL0MRK?Kdob8bigEi*V-MC$w?REeLPXm}4I*wP&MsRB6eaQT85=I2%z+25oD9S5^>#lIDfbi z+}A6llO@CD6-$S?65vujq}>J{Ae;28?olW3lq= zI9Ot-1vjtH5o}g=XecvtIKn;}=8--(jM0YP1hnqiXnbI(g&V4SpxebX*mxihdnr95 zRYf1%ptv0C=U2j$SL^6p;%dC*h2S6H&ujf9T(wvG+ zWh@34$d*7x1cYiE1oi>2SeaZ4bd)!2dOJ8sR8NISYU zWgcjGPlNPW{jCT&Fb46qqeD3bV7pKdC zs);L9_SM75`gdsMxOf_UZG#|;k8x02dzrl7^H`Vt?OCN--G0 z#)p@Hc}zO^N5qoLWEIKRw?k01Sr1p0&ZG~e8jkYbA8FB|>m1>*8!MEJar&Edo~?En z(gJ=7(=8s+cZDX%pXtHA6iyqTMbLQFi}c*HFRs&Z7B0%|W~EG53O%2IBW5|^jqQJ7 z+kH*$IY*oI!nQ%rz*oYZ_-43#)DDaPUKbb2{TA^3Q|XLWH8FCoWhWo#R@RAolEe6i znl|Tn9K@9a68QY)*KqQp6^~wAME~AuVBV~+a1zQnZHojKdS#OH&nc)F(w|jbW*|-< zf`L=gp|JD;N+$mSgCz$=?=oYyTOLRr@fQ3}Wj()LIF@F&T;$mos`<~BaGcw2AdxrsB=w-;M& zyTDtAvBJ+71+2Gp29K|&9iB{5;{#v2a?*M!9UOOsPPZ=d@E7#$r?ISGim?2G4eWe2h_j_ul2DCNyh3vvT`Ov)m%Y-YO(*NX z@Ph_8)eFM6panFh7|^-fc9L0c0q>)hknI*XUh%P%cNfeTE~UMN^o~7dQujtE@LmqK zMZTn&be`n9g@KpXDM~S$C5)Io8PB&(lGfk6ApER2h_)XOK+38$tZyGrx7Ky&)fx_+ z7-;I*u|-r9IWJJ66!n=<{pcx7EWXJLTCLc;y#hW*WsBVcEGaM7Ncw3*3^(OQvFs;9 z9JXvB#z(BDmp-!vTjNT4b;J_Sjl2%Np2ql}Y(CwdbbxvvRKX!XD+SxLXX&WzCY&`_ z1y>$^PJOCxke}oeoyydZe7@R9<9`1q%o@}}mf7yet>@vL=@FhgHcfc-WjdWFHXWXkegF~!p;Je?tfNgs>ZbCLDtjEXGzWHMb#P== z1=tf$cVYLGA8^mC zKh{U6pjOHW{?jVOn_H4_s(ep48o3xv1!IKF9USWtgo(YfhzHLV(%)F%@jr%=n|YVO ztnMjHceaQ4yv01Qu3fzE^;wv@Em}NmA&1?|FVl~aT0;0oZQ((f2Lwz~p|P61`Q>s6 z#Tvh(GTYgFy`PdO7WL;vAHKl^gJxK!q#`UVlf_EUBjl%S%WF--aD#st4bK}Opi^b@rV0ksmrF}dn5TbDE*DWdh95z^mxbb0#J zaQ5x%MLM8|rpEqc_#sj})L;*~tuwGzt^vkNS zE%5!DBjWC`?Q}M}D{Z`R0Y`?}viw1B;U6r3M@a*)>g?~%xnR588+{AUO0&8}^JTYi zbSaLe^Xi)PdHqWvJ+Xj-cbW?y2PKm3)X%hHM~3vCaSHtD<{_)iW?wy?I{F;+vIrnM-E^{Dl|rkny28X!KgoF69~$r_1h%Z{jWtTe zFjwY2On(p|w(J@x8GNb_<}YlNCd{`-ep3yy6D%C`>K;OW8zmGs93zX6Hu|caDjfdx zp8gd|XjH%)o-ta&4?{h8@lz#?ICKj>C@zG3mU=Ky@hpAz-+^AeHbZp#JJ{Oi1g+OR z4b5%+c;@;mTzP1>kR(p&tb_75Zb$_?u*+r{G+X40+w)6V>ZDC&8WS<=T^{7FzfEOn zmh{$iwDk0?d0_os9`d;toSdVFBWJ7-w0#eY`aY4kZ+I-Ob(q9n3u{HCyB*J>QAwv? zh4ahTd#HZqQ)!6$M%Hr~#bKM8VVm80(XlyG_%(AJ98b-|md!8e^{7f1U}`8CKFF5q zHO~p}*N5|llOB9|-cVY;__h$JFp7^}bi&rvXN8&9A~^5CdUD@A92;^xc~RItT7fID zI_?Zj8={T|wnxz;)f`Q(mxJ@;A8`Fcy)fzI0(cXC3?>JVpbWD`^?s?4{CaOECr<4% z3&PhQ5>q1{(9|w6=sY@w2fkF|TjR#zpC8M);;0FZ)p5eMxAX8-vMf$Mn=L9#vY_v~ z6!DFWB8sh(dAsLOFjAO^4{ujX9cRkoDeHyod0U5*4Dw;9)fM}2;}nc;`X>Zcs`9-h zx|sBBzcl)y5zSJ#3v(tMXN5nr@QM62n*8e$`!(xIt~*W@Miu6xbCw+CSGB^Hld&xK ztVlTau}=E>pB3HGX#?|=3h~vaA_y+rL^kFwZ2fa7x4hU0k;^U-e9;qIEL@!ZbqY3H>Eae^VsW^4L;dr%U3l zz)@>C{+p)f@cBz`o^WKi@M(zw4tpSvOHcKrQJeg!ap-A&Iaz@RY}DqI1^GhmqI%f3 z&mJ?3m=$L1X($h`Mi$U3#n)=btm@@4kJd`SyLmr&!AGBKCu}oHE}~w`4!{ zvGg)i4?}f(bYhSruTGP*`e4Pp`QUFDK+Q#iI3el~%qo7)w?>_VEpmmFHo=;kU%Z3I zkO|c-I(TYZFuHGCPdzH5F~-sjcWoX`riZ&gZtq{P%=xc$>s(_#HoGHh*83p+7_^vL z;zv+WM;>WT{(5d#djg=-EZ!e|oTg@W_ysA8q3624;tt6R;m$)lKIW!_4$JhoYO#{U zWa(g`EdHxFze5lGV_OUB?p+jy%Y6_Ym$pL>^TC|$ITIJoeod;y?kqd3D`af$PBYtz z!Q@j~$p?C7zCFRvMmea?3D z$cz`XX68FGHBJ&$UOopU8F>`XC*iCZ9WeIoAv)hm5;QzcQ}^8oymQ?H3fp^xmigFl z$|enbdvZH|ico^7;z)o!je>3SFx<4RMXI!^f_|?ZLF?)*Byp149eIjmnB7eee_<3Y zUwxk(XO&>u_&8y6+)g~JKNt7qoTpK%_jAPqSu8CZhX*&=uyV%^(OdJiFf=5e97fdA zBCY&~O^D)XXL zTWntcOnCTeAUow3U_z;qFln|EevThPv+tyUPohGpf~6||StF$}m*-&qvIj!@QC+_K zrU<@Vy94_MY@$BPNAZzy9eSRenPmFdc)b2_D{Id=M~Al@qm~_8&_gm9&l`+nnb9gZ zMejoCR^@+m^Q#wLZZYlf@n`Z0OKaG7a5&2IHZm9$DZUxAjX|cBy!vhB>(d*=llj$j zum7ITyRrG(6>;F9Cz$s#8|AuQ6)ez^kJihwwohM9*%HUvHQJb2>cNxa_jBv&zT6U- z&cABgh_BuR%^%Zn(tqigV7?39ba%oo@h8MGk4-|TOPc7?uuMEHzkz=wzD1LrsbXN@ zZs4&jhEEM9kD_h(*);oftQ`u!gG%e{ni%0008q6wy7xY2nteVr=u@tw;2^Xp;AOxjQ3 zOD@yGDJQr`)hr%3ybUI5o#5cAIDQxx;fw<+q-59<2o85G@jVtLaUqJq^|(fM$}pSK5&Gua5GouLv)`rDD&DYv7zY&f%S_p~UxUF=+=b;gn-H z$@SPL!7eWf-hbNwFC))ESP+<#vPjyA*!BerXqP-}u-wyV%IijD~kGl4s#^;Y-gB$82#A#=$;8~9Y z>_2n^KV8uAe+If^(*AFhP&NZEn}yNUZ)TEXiD$TN9^qK~)%c`m3}elxSa4tjN6s5dBa7CPf0QGy{HzVX?W2Wz4|GJwdotWlw@55hRqo_O zry%0Jhv%?wt`2TKazR{Rv<8(GLUC)@bMo-&iKmJS;RFB{tjPuc&*yM>)itUR-@<_b zMtsP80)|BG$CQ31T(ZCmpGO>~-VRw5-v@cl%5*GGttDCeB)I-%C489t2}>GXc*2@3 z9G4m^g!-GnuTVXQSAq>u)`DDuBE|1_W^qeB1>0YT+=38RyW>Qkc6+n_ z)=4-=I#gmjzz1i_z7%qX&!+34Lve4zL^OPhoT|KwO?E87hTH~WNR+C>*A9Kx@U$wv zTHY?y#5LkG-%mnn$~D@qm5vVPgIMcGROf8`HcWv>TE%SN>0zg);|j1d07@$f9?v;-h86aJ&PdY^k9-yDze40V)5qo z$6}MiG3r&H4gYk8v3te`vJ1Dt^ZL0ERsB)4Xgdh*EhFf6Z4lF{@Q!?(4ezG?WrXDg`N-V=Igk0+zufwX+ZCwzBKo#b>CB}rk4LT1t!8XWJ< z6aSXdQe{)De-{94S3+smh6|8)eLPP7bR5+Z@4|cgGZ_Bsh;a4eBG^$mgMDH;@&UU< z*!0q<6N9Nu59qiU2dlb0kskDu6L8>Ysb#}sdgkuK*?JBjceNJglxX1d0Vl~eQjWg9 zEah((R57|n4*!eYO6tDfDaK$sKA+SB2HePiFy9X_{c0fwnuSyGClYS9EdqzXRicW4 z1aFTZ96^1+&}tqJ{GKJKyUF6m{+7JFdoy3TSScLqI)n783t`mUMzSzj$rvHPW^0d* z-c?7(KY66SO&4GF3Zw8|GvUxbZON=sKg!$+uy~9j9{OO+!R=>+!~q6;@!3*}JtxmQ z5)8%rQ*F?>|6j1$W6iC1Qo%POo0f--p@G{ULP4M7oqQnEF3+2zKao#NzG(HR8JwqT z@~|l`WZ`+$p_l7Q{8_*>_)!+bn!OXvx)oFRPaWQF-|_fFX}Or`}6f_Ocq>?)C%%=WU{@t#8;;nhj2S6ZnR1I&WHIk6PUV*yU|^IOVmSEI&JN z>&1`MV}Con*SJohHBu`0`Vj-B`@-@N9h`hfgkCKJxhDA`JRG=>miD&7$0)}VBX|6} z*-~=HwHtk3`;an%_i>Ntj`^Q=jl-w6gZFzy^ilf;r%p>DhV^(=hsHNPLJeK6nNgs+ zEgzCehmoxdJL|yiQYb(6*XGna%V_zwF%);Y3tRWOFO;+|A;(Mm>0Zes?)H5>)^)h7 zT7~yvg~ML)_{12V(<6==(%0}@&A#~i<0UHG>BTb(yYaZ{K%6u-g85deeV*qeuDExX zHH_-vw6tApu}H+{)@qq%JT z7UAlkBQPmfhIRKxaQ|nvcw|De=$6-0*z|J}|C^`AgMaE{%I1aGP38ssyBb3C&ziE^ z$~LmRzYTuJYtgyy{%~BpOFu#u!lTh&ATc&tXsuJBDzT`#Ea$J`DZ`&(Xd_D~ZbIPw-<*G>*)aDZBD=JKeo_mO>URgJ)BEl9Sm1 zs@)cjkEgBUfe)v!$=!P}Mx%fSxjgUO4_aczuzi$@?wkM;F_buq={fmOm?i7~qsiL_x zhP;1!4B6(YinL_{YO7hX-OF-zJ+u@475QOxM}9XmI?Lfj1dCPrrL-nG9nxQWvcuIe z!j!Qt>{h-Dcl4W$Wlh&$$D$(f-SbMAQqq-+%Jo@cas=#XdIYmO#>pQ20C;l>peE$D zn7%+)l63zTC}r=#8EL=73pL+JOEQ(8Y&nQm#+h)xPjfk1*_g39jdJfA@hs<9UM?x= ztb=R!mfp)to{!;LQDVTi`XTXi_be5<%O`~Uq$t;@H;YgarBzB!r4$z2zpm)Tsq4A@h z!sBDP;5`2`Sw5Wr8(a5c&7(NzGTF)Dpo=MTOOn|A<66;a$P3u~$sdPgyYQ5o4gBfg z6)-H%1>4`zykU0)EOyJH(+zr(jjNQnsb5FFTlXzxx;dcsszi9vBZ%FF@4`3n5MTN9oh#iI6BNV887G<{rxB<-MQ52IY=y$DisFZL8gwrpTnb zSrBv!`k{NzJPK5*E&Z(aR2B|v!g3X&2(AIHh4QdviKsx)7=zXjRPOt%LN=S*!~JbrC2dR(Y*_>s1k zt%K#*%H4sxg|)-J4ch2+A`!LJcalkt4o`_PgDYOZ<5Q82eoUZKEyHNDmJ$2M?BKvF z1QYE?^0Y4R;EYcmtr7=vYv53n&8ZY~`)OlWGbLVU&_`moV=g$VKNL6K*@A7q;&Ii9 z?KJK2T9D7X2p+@dGjG%E_$(44SAYIwX&b=LJf7yu4#)rz5hk zD`x>fmt;;DtVY>t`Up9FB!xkhpqM>~pKT4}m5p8K&dONWo_1BJD(cEU4ga6PxHxq> z_X(T9?QuTnT%ITP+-Z-0txU;BE*YxqXToW{3ZYkQf8p|_I}rCn6SW@6(B^`8`nhtk zaHcVX&EGFXyS^H%H13FKR2srd%e-*w#$9|VV;u$s_r)KBN5iEZRpOx1MNp?Ag7Ka_ zSfAq0&AYU*cTo&q{=6U7%QQl{jwQd@_!O!pM!=M)E+{u^zEGjN0s3opc;!_t6j#;_ zMyDjuusCYd)2#&6T`8=UFfk&@0UTRa=rv;@n*~|$IhT_>cjy7I&5h@Adb@CWuY*EH@p)LMB+DufqAfAvqL|M4GLIpWxBsbW);H(y`gkAC*+FBQB(&~o1eQDd?z%LP6MO(g}csj|S# z8K2<%l8!u(>Q$N<5C?U!ufV6~pj6ZNB(&S~#QH0?;i6fP_GKnImYj63Y%}74 z#Z%G6r7NE)j^f%ycW~rYWl5bJL9pL3I%cfSBYi_bbEI77?Yy(@AD#VhTXaa+#b2^? zXzrX0xN3h5{_Hvgzw}aQ_TV{Cp`H&}DZ8;zr$0CSQ~>SC;nInpO}Qa-EJuu9KwBz) z(s!da^3KqsqUVp`>3Dy7-*_C1a2LYctE^hDjP)zEIri8;xZ6C7HKYEK>KsKHDKi1L zOYb_VRS!jDa3qr0U^Ov%(OEHlkIsp;+rd{*rbC?x2aU=`d0Mx z^#RqcMbe1oqp(w@Lu0buK=Rh!Lf0qXsn@)`<+EuF@R0lA9m(Cg9=Jlt_>{1IOda(}ScvH|(K@H-^#J19`mfwHI3No+IqL zx`PJIp28DrHgMdG7&>tCATEq71NpcaQ1jRujNWgf5e1FH$zhM^$IqkEM#W2PeanUK zKdyz88d+8-+biy|vZD#l?h3}|?eO53EShBOLc)hGyu|T7e0||1OpQ;VwuS(l{5oB@ za!VgQ^KxKiaV-RPHRfIqdq@}cOE0Otp8yLNX!1*cb>0+ym`9h^NWa-!5d@3ZLc z@zCp;AjwiG`*y?$8ZE{`L1`-)``hrDkcp`J+W?;j-4ZgDbUAXJBQG=|(miU4hc7;Y zDdCoU>-s&6CnToI3oOfu1^V09G zf^R<~$^5p0&N_HHV=>vxe?y+*rgQeLAeb1s3VSEJvi&C$4)*&5QBTwGKgTp!@PizB zWsDMRevSdHNENb`pA07BvmGTBeY@-n=9M5aCHE zUFxrj4-$)^YH>Y!sw$Q(P*^VBvbaRo`V{a!**M|Rv_X!0rj?Uz&`y}HABBsKjK`7V zt#Czb4MuPG#_8b~@JORx*@1@76f&ZF*{IX2P*x#?%`($)du|^-e!`u98~IT|cBcJb z&tB*g@*4t*Qeh)!;X8-_E9pG^xqQDrZfB2@RkoxgDGKlF+$0sHr6`4@(k7n@4T_A+ zq-kc9RaqtDeVtnpMWul>L<31n%ZmElKi_}hdR*5z*E#3)dLAJtd;JrQg0G>=>BD45 zBL?%rjd14{3AXm{ZxG^U1qDmTF>1e`VmKV-UH;)q4*lCi{EUNex_2uc^wEUAwJo?L zNr^f0%8{1+dyd-!nu*KC<*38|OQ)D6!p{}?I4No>qtdK`yKi2>=#Toi`*qLQJqY@C z3Dw5ipy{YPU4O2ehW2b_{uY=pt^tKKGiN#LR&owkEAHoKwE0oVB^QW+%T`?1P>S+u zvx$@QQ^NE#x%s;1hlLoG}>_V^hIF(G?E}lu^fyHf*=$ z?$jY|W}eD?OW4)nKB;yf0wEX>u<9r_$lc*B8$OM$m-!%ZFbC!@sD}CT{$i}_DgODgCnRsPF$5pCM3o3* ztUR~?CHvOl{(Tknf-<7pL^1Z{za~H$hN8!-VaCQ%T15;U4E~UX zCrrNJsawvt%Sf8NQL2j9cDE8^i!ul*JBvl3vDB;06ACxvqw_P{vAl4`jnnkDH}EIy zE&=|1&d*(p@++K0O{9pQcv3(KnN z;rogAeB!wQD{KJi+oE8Cv1Y7i3orwmwGSTVA9RAH|)4)};hJKpz8dn&ov$34<>7}SF z>p1X(zs@9;+*eV=?hv=E zfMb>rDCvkJ0iSBYr_T-Ee2^LYBl%aY#-uxCkUhx*_}N#ej-?)y*iK<$RI}g;%puQu z_TZ2(msj1@sd$ka&D?gLJRdque%+o20=y9B)2drMLBBh=ap@0IpQOM(y4neos<)ua z+8#W2HyB+@?sER{d6@Y3HVm#QqVw)8Cibe;G(sbQnbYZv3VN~lD)bF+URo0bPnN`S z|8aRlrvmza|3a-t&!AIr0UN6lz? z{p2ZXd;dWVoknmk6J)!!C!tty1RifKgQ7qUcJ}W*(0J(t>^ZN2%k~T7QU4dP@R2^_ zI@&-7Q@Sv!wV5jPzXFBfS5$MC2v0(P0Q2L1g2amF{Boni*p<_c>dE6+ZwV$3RmW8kx`$)}6 z0|x6v7%g@j9euJ8Wp!SY_kne&Br+Fo#8@)_pBujySTI!^N=d`GO6u>r6*4Wvv7dhd z^=e*2aKV03>-35At-Fi4N21aDybw-wP^>VCkzwz}WKhSn9^yNoV& zFY2CR_Zm4yRmKm_WT-$;y$rJ>^*GzH_n+mIjxMU|{FcDK)6`n-EUBKqml^Wu=6MUP zB*I_j60_3rSUB+pdd^kAf8kWniLO5v-j18-v~Fg5<|kh%e0Fz1**!Jc%|&W|E8woNE|)s8i3 z2ti)BBZ#P_l1S-OaPMt6ZqU-e;SD8l*G`5UEYC+Pp*fIvuN6(AXAt=~QD(-&GU9Yw z4y4ZOu@V8opq+LY$k*?L$7hIf9SaE~niuoo6j(jkVbzc>vpk0MNkL#rK_9p~?$%hn>k@U?TKT>l9_wbmRpc27l5 z-+p{1c9&GYNFnyu!*H{jDoL@b#a5m#gnddRYX5lXQnZXUnZAktz|REFl)t0#D(c&F}(&FzBhnIs(|NJeP*Aj4$2q4rIo4a^wpDo za>u!rPAc&tx^>&Ay3Y>?T`&bt`PqQe$5s+ziU}=5Jl*@6&_j63vQi?b!%=tmrpwy0HWdMiO}QSD&Xh>mNWH=RXhH z^p0n@O_q7V-^Uz0DS+d`+hFagIudF;8^wUrB`k}?uDdSmA?+|wuPe!gED+Hlb1#4%$p)N)10kpU9K3 z+<4gJD_ph>#YL;8ayn!Yv?xh|lw&%~&u8W2SeZU^Zo?a5b$Tw)5`T0$Ed_~sg5>OC zLQl@W1xs|V5~-#MuuRsRCTiq^?j14qcyK1^K4lL)^$}j{Y)4$t=YUf~qR6&=_StneE}>|uaP-r@sOIT z#9TfeiA{Zgljr^>K}U<|H=EZuG`|P7NG_pkIIY2h!5yG0a~r?+$g(q>ya9UZ@Ys`= zlyQ;*3p>L2S9jAFuh!80!s5*FU3YNzr{^U4YuwnJY2zewI$f6!zdOgl@h|4ssIr5f z&^1h^-1p-pxb~6JqjkI+PorSDk`ewnw-;<;o`adw3!G^bigo7a(P?QNHy(aoIrqpk z=E@HWL;pq~$T=Jwvw|^uhZFAgYT$PS$v{@A5o@$QocoJNq3BT;R(hg1^E}vv<6Nv^ z7bI=Mo_De6-;|AU--EIFd^j$S`j7iJZbmVinf$c}|3am%GF;*fz*p}mdc6M;9up7c zmDH`r|7sWDnwbr_H*5m?*{d0vnva5DRuGD0?t_eWSGqHJHjkysL9x3A_&qhCGh!a|S{{DE(Qy^PbIgOC0)IIGwwQ z_AcL0KC6dDt9^pe*ahU+0I=Jn4S@J4GQWzW(8je8y;@d*uUQkGuL$D2;0BCIPcMzy zErj*yy6BrO#LV{m$J-N>KxhBxgT?s+n6neWSpFiqo36oCjgnZTqBKMY& z6@oLst7sfszgL}^x5<*e{ayqA7Uz*ftp>VJ`xV*u{TYsGiJ*p-EqrRZ15)RP;DDnx zXv|B;*IUZyQ>j_(Vee1mc>EUF{>=_L`ns{VMHYt0O)$@#!LQhxfJe{&rgP(e;-lKD zxciDY$<wiwASA%x7$R>PfJa3%t6o&z^5Jf|zwVU^qLy(kmtq zmY=C6|1!3sQSEbn`+fo37W^7{pGP3JQJ8NuI|7z+-((_q60elVg2mziI&GZ+YvVTu zci~L-`96*>)2;=4uP*AjLKE$_tU=w2XFxs23q+59<-Ajmz_;r?&&cC6mS1&)y=_ML zF-nZp)DGj7|F@CcW_wVh<%i|Sr4`urJ|EfyGWceu>!EAU2WUB`$QU*mknx-bS7Np? zBl({m-2R|NLvv3-ebOY7dnz9q5>_xOBF7;h%A6Prj$<4LSAxcxS1{u{3lk#BF?Xuq zSZ)lPR6$x)Y;b08Bq%i_u{OC5mh<)5y>VekPMLzOTpDV{g`@oKD!dM(5S~Az z_;(D#oGbYKb_-hPJwc13C*i8m75Z{?I_t468l@5(Q2gX)Y@DMDKI1EaY)s@W|Dw!n z{L=$E?hk3hg=*d*B^Op^;2jxBsDPC=XXz1>QL^Yy7(_mvfKGvumi*B07cAdJ028I-M0gT1O6n6vt~#ixn-?3xTY za%FuSY<38x`?d)(gBoi5ibJR1)!A-Xo!P?=4H6lP2U^@r&7--6-9>Px^%X78y8vrVGhoQ`GKlN6;3KOf+B7AC@!UBVjeL@zQC1qY%cJSW zIl{pAO(Km)RMEvtpFQ_n9}nLTMd8#P7?eMk*)QHh#}5gze>xNx$L%TbF-4zU)o4L2 zLa*ZKUtPG7se(a2MV6cE!gQsnOjXcFzM=nTFcUn6#ktMI$W0a7EL7plLJy7&A&K@= zd-yMROJeG`=e(!p8F(r8CTw|G4+5$;;OOxW5a4-{P7Tw6gzh2ld#cMwiKPsVe@3*yW({Vgd-JSTfgdRKleft-L!`?`ZEXdA9j? z7~0=1gCC_%upsX~hDhJTq~>1Ao2iN`-jza~cscZ3Xa?(#5hOKv5x($D#Pce$?DUoW z)WA`eHhq=Fn!AaVJ+KEJ#-`!dGr15kVFA(pyM%GS6-h*9KP2_Xmtaqi5_B9|O4WND z;DfObJ!7HD9@xp!!&j{ti@{oWAm~gQeXMDX=~puPQj+aI@SJ}6yPf_2dLiRi6bxLH zhHM{QR-!SFdUUPB4BIaBw6>7-nkcBMrGJH zrCUMIP6W*Zu7kh8L)yH2Km8LNOqDer*z?dBS`8Fl+2B{Jfct2XHP%#Y@4?CKs^bSWt#`8sO{wms-zR z&Tjda48;jfICNYP`t+??uS6l%*y=73QQ3g@Qi2?_WEROgEej6AKgoUp1FZJzggXx= z;EhXhkS;F4JboNRPI2|&14BzxTPlR6mLG{JpO0%Siphk!etPoo5XRdZL2Z!<_CK3| zMQ00P-i8Rc;I0en>^ZjRf8p>t^%0fT8|JIWT}5+qf26BkgX70C5_H!Bt`9H7!*W|8 zTIdGU+sLznu68i-kRnK?U4en(Q)o6>lHGGN3S9U}aJfjD-P3M?3bEdBVk95_9UC6Y z3nz6Bp~#av5cw<{X4%eSqq!dX7uOR!i6&X>4fdxy76#%74WP-`JcI@Tu_vnu^ zkYmj;vHmGB)6GqIYfGQQ(kFKze`Et+!P*qon?Hc;zjGNA*MIPs>)Fgqi6P&f=`t^P zd|d4L6MmGHLsxJuG0+~*oIWiICEis$KlyO9Z`1@6?UgX){RCY2^azev%ChgJU!dW0 zMJ(OFlULR77!+;OG5pTC%K7g;^Mg*PGNR`K@Xp81v3Ss1Qi%1p_Cd|u2=cahB7Qh_ z7te^=p&I``+Wbipnfu=$AUOdA__t`4Bt_elZ{%K=45PoRg|16Ik0pt={MU_=U{-8^ z!+V6;!)B4#y5j;r^DL)9kCtcpd!1=?_zz6hzsPw(19?T3r|^*PA~49%gc18nG`Lwp zggS(9|LvU+Yjg!9T~(M=|0y8KAepm035+#0*)8cZ%!2(59K%7BQT2XD>`%nPlTKS) z>iCMp*R6m9(myzj&rVk2)Ot9)vW6CHUyADvP>{U)15PAYki(5Oz-CraqqcW=AbP>}`u%y>}Sr^;A7zm5re8t6OO5;(=_nfn$* z;(-STS^E~QuDrYwvWk>&wR|wGnw>*K3ihz|msir7x6#nnbPF!s5n#3F&tfaydV-{s z6kdJP1F3_Dfu2bsA2;YQaWC%UNzE=^g}`o{_2LW~ET70*^ZY7hJ0`%tAw5vIRf)dy zBA{}27+Oca$I~JbG-JniYBjYQ#5O*M+S_W(uGZ(&V)0_~-<)VjTDN*E9z=@!V8Hca zc>DMdT@1ds<3caJe{D0dPO+mNqP3RNFFLuqwH>}33CBNnkvy9v2T881CUb1BA>LRt z8?>kYM8OAqu&l2^$IrQB_kzo43&COE~P7fXB9W!@=rqG8DHL ztmMYQMK5WXG5;nU9L1(F@X!t9ED$<#oj6;3LevJ(|3(OlnM zS^K()OddrTIm$zOhg7nf4Z#x`iDNodnS;u(`707zubtqhql5v{579cdAH82ELS@Wd z8t{H6YA@;qsROTo^In3Bi!bR}F2+7x&v|83rm-hXnt1|7!Vn>#!<3)_n;&A!wkd35 zu=O0Z58B1))XT~19tW&Bw2Wlg&4KE(!K7Pi9RxJ;;a1^9M)Ow!SyH}$36Rr*XNU4J zm-6xb3oSJGWWi*?c6fWM1cy1MQMTS5yl!QUDz=aL?e8yvdDcFN*C|TJ9&$PM!;ZuvBjTTMmSpBg43LWNXv>mUSPmCGbQp*0hp$XDR$>_Q?~n#+6SB!HV`{-JlsBmT&j^=RH7gBijG zBza0THo4BnObZbh*nW%^SGb3d{UV_7V>s#+yO4M1cH+rFO>)Ih7v%zE**$B9P+}kx zL^$vE8J$9GkvCy@*PKzHGzShE*FfaSD&FP4#(3;Y2Ha^*f%D-U6V$T;2mdT%)=$=i z(yBs~{96Dv=VX~x0`0i7MTc1!xePX{*HG5$Jc>@Sq7F6zm>8f3Dt}xc&HW(c%qqd_ z!8%kkEDDCy^ibVpCHPL{7**|i@xA_cVzW(WEGDu7#r&0@%TUSYB1Y~N!1u~)@K2^R z+3&KDjrBf(hpboO_C`~j-FA{ZmtKIAE4qnRsUp4|f0?vjNyFh6YK-!m4|Hj%HQrfq zjqZ6l7n;SyafkD4mUWp;c4&A*T1Wx>{m91|KH{v>lTNgkjfaa1ZE%yG3~y1;TEedq z#U(2Q*`@&=JPzDJF?J4XcPEl2sCAJ2DO*u?`)zPaYoZdj zu&@NLT=^eY_h0e>_S~agXj5&9+ZtZLQ;CU;b&)QUvgZdhZ_mKE!{z*aKF9g7O4n)5 z<04DzOS@p}a`tezoqS#=tj_@p0C zKK@NAW31RaheoM#wi2tjbR|jOWWs4?SD=|$46m=ai7d3zAZccD%#?*ntXU6K-_%&Jf`@i!7leqFtx}g*!_|Y2I?@-{utH$zw$4pMMG)r zT@*TQ$kuH;%W1ve(?HjW%;VA!wA!n~#+y7LRP7J#f82?B7B|NJ2Z4qn950H~QD)0h zW{~rU2kpYNy_tyZ1>|0X7Q4>N52gv(phn7K*tvTHp8Z>ZR-f`wPuhYhl6ebt-^7`$ z(hS;c&`6iYzTs8hDyFJ7EI0*r(`ygA_=Ck(v`r`mB;PFo=Pw_)-p@29rFtobh>9`p z#_G8B?09H$&%|5)=ipY*V`SGS;neg8bXpZ*^KBo{!8@B4B-oZ_q_rM|b3v`LSE=}W^vQKtPGPUBzQ0<~Z@DkdmEXiJ(Iv?$Ftijb|K7KY9hG_3vOj#+*7{sT;s{&niXw(=@rnb_J z&F0|tL>hi{eInV%jUc;H6tlHCrjkVg+-aN3t`yK_8>XjV7w5@Lmh;A=W=&&f(3mB{ z7)^Xa{IBF=<%#3)eMT*8Qr|^Wq+VJa8h!$|7Tp40nJJ)q^AV(die$y*Ux1WLICfrH z0DqN@7{ATKpdFS;2QIv*taQpHA9ux~N#JX2dgcVbIq$>mudA?ff)k2z^L&TW4a~F2 zvW!k_HR%a>h25czRIT+Bky-AB!&bSt-XemRy4Q?6%~eN#bwPOfWe9xT8ZDzwC-Gj) zHoz{cN7&ZVK~5B9@Xwgn0VqY#%+z-P5stif*>Y^UX%F>VlFj9DBSt)IF8REhh`je^{u@YLp(ju zx4D;u%0A#~;0fSsnhw)Ox!S=w3ci?`k~Mg^GyLXwdt}>M(6oC8zT56OODX%$`-sSl@E#U^-OrKJqb?!(}vnV z57FIv3iIa33iN47pw4az)VNlXb=}kouhmWvJ zihrfa9l9K&bOSLsqXjk>r!lqjwNO# zhle2ZcLEkinV_KLM!2i68D<2@fp3d94$nq1J-!@IF1E*!%mt|aHXU*+PVt_khoEcL zWhyK631v?|0C|2H7Jmza`0_eJ)-U2aB%7ggR6X|2@dLBvGsoV-&+Gs^_|<~>XF7#h znLdH6XzrrwZ$y~P*cmvj%@d28R>H=-5z6;V#VghWnAB~=nkh|zEs2TP(DDHeetA!S@Zz{V6&`RnlgoEY zQc;X^_7u2WLn>lMbtf2(oq z-CD;}R6fD)Jy1q>PRRzxx&Si&UMNxjSVL_0WmEBi^|(5Tt3Pw+FuRYs!qYBEm}4M= zvn>7TjGxsgx734(CEY^f{muL=#aw9rI>LKw<_oR{*HEMPC>ZMB#ypv1OU6uMtY*?5 zxsOw>{RB}zS*Fakqe5ON7J8N(C);LD!W8|#C^**`{C6wC1nJe5;-{a}b{dRf$E;{y z)=~)7)&PeEL6~y45FZvyVmgm_;AeUsXA4Wg)T{^eTDvIYc_kfql2_o-+dO=8s1bg> z?id|38LI#P1&=V*yCOPqCb3X~T=h0EjZ zU^+Naf8$~zdtCVWX5M<=FW*sQN&hS7_SP+FREap#8(VCA`QnMO0q569I>AJhm>4%hNqpY z$Okbo2voI%R;@7NXs-xcd{bc3Cu1ny{|QZUfL{9M!bBTo(QT2p$ll1?JeQm|_+qUH z+ZcEpPTUBAO&XgycGWAeOi#wXO~p{UCmfIevxjAOt;l`V#Y`o)(}G4@Rkm#og$Wy@ z@V_D-@DlnT4<+y4cvf=A1RhSii*KrC;M@R?1$$y4jI881sA-q!It?}G_3y9TqH~(e zaZ_M)d{RNxkJCs+z9l!VXF=dcMT^>MB`mA{3 zTWKSI^S>^#RyBwxt-Ou4XRSwp%w}}E_?p(3MU&_v8RmPC78=>t!I#~6+)OJ6PLmTj zSi-|ABpwtbSl-1+*XZ9MMJC{D0xr3)#_}|U7`=NBp)>g^Xe>*@8GF6}M;kQi$c8G2z56u5zrrbZgWMRl^{5m}r4^AeR6RUU`5Qz4K zmp8*`YUx*OS}K6@pCw3-=_FRH@i*8hOvlLcNjNFY95mbi!NAdCY;b%Bz7A!4)frK! z`uPL(?>mX=O>Xf0P6Eo0YO(vg-SO}%MYiO{baue569f~vogR4`82?wf)b&j|UI_I7 zEpI8ty~-C=%5phBdOcRj>k*AR$M9-s4>)-UVdJL=nBX^weo)uIOGhpE#!M^BT=5$A zAL^!wR)yGJIG#OV6UhI*c|R!sSPzeH{2+B@?s#sE7hWix26pWi(PY*NvO?@L*_*{_ zW0dBQUv6@&_f>IrWd&d~n?CjyJk3w>r#WxM6KbN|%*PClpA)1z+gmX#sT|bgK45kZ z3p%pLv7I-I-j?e?>3@^h0xq5vRI0FFOVw~iurV>~^ugSr!*u9Y5)QvR0j)lI#B#Ko z7|G>>Jx`puDH%#soZUe2>=}5!;2l0HHzIQjEMU-40G@IBzNjZ@^p9K&K0EXtrZx6f zdOpdJ>xkr?o;WUzLmvd(HaaDE?t@c)fOjN4^#;TKdo6p=%&k8#RWN%qhi24-;#{rs@0jQxfN znl9_fzqook<|bYN^LJtRxWxvf0@bi(l{)5_72R4gYE2F#q*hHArDU zzyhJIu*6LcUdAay*TYzl&by0~O=i-~E+=ti&0zWepTSJ`sW_2iVO8m;qH>KaB;20N zt~xlI+1PKwiFKQA&YYRhRi&cnf|5@OIR z&*?9kK(Rgq(?(XogDIg1hx15PZ7bxiih#{&npnKFlC(DzLFk*GBz(0Dvwz7wo>fE^ zRxc6*lgBzdiTHl{C2JfUc&>{nFRw!S!*-~Q{>|5W^ciP;Uc*X%9_CA2MR@UK0iNe( z>}oYEPU7m%$8B;<)ebrIws;ORcc{bE`2x_FGfHo|exg<%%$e75&*0@69rok=%TOX- ziw)LMuA6dA{bm)o z^G6g*E(F5&{QKx6R|xu+AfWf5;S+vZbu2&Iq3rU z6qeCb^@p(d`gItHeh++`N}Ogf8S1z`fc$4cR^eJO{++{l%N5t+bg3XrjEaE)>#Okf zJEy-teF}@m0joF722OihvkoS?wC$?_=LVVp$`Fe+KgBG6J4RvF`^K?&F#As&RjYdd zGAfnuW~T-dqRL|7?s3dZ-Q8e=&KSbiV8`{mLyt})yk(mTcYS8S-wlLGocR%RO4bnF zPtru$hp^?#a=`vwCO+Gx!8T-7L)?HoHQn2cLWy>ucQ=l{kRQcrz9@UmJO{fZ71_Bf zCGbsP3G~aoBK*93tQgo}DVQbBMjzLJieYc^INuDC3tVaP`v1uOyexb)!y8LX>dC6U zO2kE~Z0qu4xb)Ub)NmYy7R{-MYa5A1sRsz%osOChl0j0$73vIyQK~?W4sCU$Uosm> zRnicIyZE6~c^6$VK??#~HjMQTdS<@hZBEUjss~oXpS(G2bn$CAxv~at-Z_VV^|e?7 zMjvJOt1+SxQ?Q}Bj3$S7(TiFG)F4;|_dgMZbpo?NVO9rkMd~wRIjaGt91?+F>y+3h z>fG*toeDUO{^TtoarDEIaQw0(8^b5aviILyqTe0Pl6q%BR()+SzD=0S_qllr7UeJC z$2m%2*VhhQ&+UcUs5Bk*of!NS-)yN{K9SAJy+9W?3{tqLj5mKx0BI*aSsfk1d9HWB zw$~lJdrp$f1Tu>$HPDCoj~pwzvOIV$dlzGf%PssX-HiDYYw2C#uW0(Rnx2(SfE2%1 zW4W?A_f9{Nfe|M))D*<6F=jz|*Doy0@#m`i)>l&VvkLMMs zKEdVTEm%M8AxMAD0PB1I$bEwe#NtW^9x2&Qo1T@Se8D1Ol(86!Zi}Eo+i|$F@Bn=| zS%Y=@Pl_Gt-v-m|qWRMeE8%KZ2@PSB>BjSsn3#DVOFB80;mJ~18+sXS7wWS0qbwM9 z$Ke8lCbD+?3D~9l(;|8#2yHIQGByjQVV{ybYt*!AEFPTam;!&c3UR$=8Q9`5or-Nt zgeUPQ@Z0-tSSFYXul6^i#SbI0WMBk~+8aSd#*}W2+zOlam%yVpdFUz3Kz&~bJ#bQ% zdZ@-BEV5&`L_U08F2G+c99gPqqWOon#aLE0S7SJ9nY~4_aOFx9IJRM`Tf<09=pShu_^_^Pi6!LGhX(kXYMG!#t<4cvqHj z9(qblUOtEErHUZ>z8!WuwgcJf!@KUCg@4)^tXXn~ymJ`FsGKw8W9N3Jca<$(+$hLy zgnEq4AWVvM^w=4cbpNHt+SA~f-r@?SBM$K3a2%fKTn?WK=do*NHR9uMV$2`EOzI~l zz-k7mSq?OE8aeNyc<1jj@OHa^{z<=~N6mx0?QY;TKFGjt=9%R7Ghf&mbQ`~PzoTyV zBWU=D9XnC8joc6`fb3VLIO3oLlRvCRjfAVH4tk8MK`83W2k`#n1mUIaYhdrOmEacU z#q|^B(4vOPm=e;7Vs~7yP={j$?V1Hst1{rto()i)H=pcuxm_gwcMtnA{ti0{uX;S1wsw+1xocx*U_9R)#onhw zjix6*dFn()Pc@V1Eqg`xDQ%_~+$5-Ziyc($@xfAs9;zynjmc}5aGc^xbgH=>de5(c z#+fT%mZ=>3aK0TAa?g$LF_&T+r%dz}8{m6no<^zLC-Kk;8IZ7F#@jURBAoQwi$0fU zVCTc{v}Nx?_RQHth>AW%eMDB_XuBC}$ju+bPg>!gH4^OXD`%)kc?FqcHGm(eAM`1S zFiq<#5XDbJ>`y2D71iIQ?As6xbKcFn`c#_na24Ver0>R4juOnRhI=$_C+F4uV*s`0 zy6nVfeWWAdHjbZslR9vHy2`7en0m}-EFPegETink`I(}sK=g+{l8zhX!cvZ@zEA=E z)6%K=6)`Tau7rg(|M91%EyDdTu94K~c|@#*+pS`M5FSMCWiB0E%dWHt!G%u@Vea&J z^7*zXQ!U!U^OH7UZ+%(E+rGJpI={BWDH41X*{8y!UFLLly3f(aQ3O=B459k6kI-RC z;a%!a5P5r=mtKAX>McK0lOJwyVbN1u81kGRVyE(hmYu{~Ln|Rr{RSkdX+sAq!Pks$ z#-+1a@Z0_ySLfDZNcTGY7n4b&KX8nT-!oyO{ywl6-i)zv@6q&}D7f}-#3MuZaq;Mi z$~&8q#&TmQiqMzdb#O{g0i)({bArNRe4DxuC*_^P<=#^4Erloa+Tkz@-yaLHDryNW zo-D+irk^nAu`qjUS3l3^VLnW2>H$|At>CJ?PGFD^vK>TV1FF>}+GQ4&? zrvcz&-cealZeE4%ac%f!*FCTbIRo|~o8im`do+l>f<3}_$@KY1_QduByI&ASx8~!) zjbdo!+KCx5<(2h#zwQmVNLHaz z@&TAF{t&h>X%Jy6!tAd~#Dw#Apu<+2d3bO=zA2VqbI-Y=pXXn?eqSkEs(laTM&D@U zk%g9>AFFwFdzRsiN>yB}@DN4!mZE{ld}dU50%J8R3up9QBf=Ut@LEG2eKAc2W8O`H z=eE|=Ynut1##B>vkt$TySu@r{X;WN;MYjRn4~0Rw?H~A>e;x&sCD;v1%kZJyC%OVZ zaUg=5Fv4jad@aT^zw@?R_L|MbNRb4b6+Q$$C2!CTGD+6ge*Au24dmigP|nU1f6sG+ zto#ZHOBzOSH)c;CP2yV>=(GNUn#k^Q!!WNbI&Zrs|D5G8-{V?2opxQEE#q-J9skT` zE9ULN?C2|KI%^rM|M;r%ZG$l{wj7b!1~@Rz8=nLxk}qM2=(&!^4iA68gN1A9aE2{v zW_+U=Ujy-gv<;YT@WS08sW|I`9k&~<2P!ta!wr{aqdzYg)-;$9?-d1Lly~-j@jxbv z+Qx2z)?ZiPui*|1Ft0$dg}KzBB@r*1in0%OuV7EQZ^Wj&5h9u}K+^PenJaq#=oEKZ z-U3kv_DjrvoM(O=s_CSIz?9W+KvV|J8x+``;}n5uU_knJ0ciU_20En@wmm7JMlQQx zPGA=VUTGr3uCwT-Y%w@n9f6~Ja-br~nk+AWKn!J+8J~R?p!@Y3932k8xOYiBkqb)f z?wm>-FUi$pRSEQJwhbexriIhoFTj+R4d6cN#M{029)8@#`Fb*2VeZKn{72KPs0&Yr z6<@aira&!meOrbqRgRE1dpA{^{~xjw>DU>Z7`}~PcTOfMiSvj~f-s}|i$&Y&G_sPL z#T_3_2SfK$5N$w`(%1D?WZ3j;m>%f6`dML3r8=ZemX5|;{gZOnn z$hFc1u+LhYU8~6D3r#iX{ILgHG`7OU(d9&aO(b}UrtqpqG~xc_8a#QT3{)EW;WgJQ zpBiF?@!HK)wK^G>Z` zCR}|Fn|$x#*4@9c``=0?XV+%xvOtA>B=DNZ42018_JR;q-a}m$|KM-P*uK3Y_ zh*J(URfyw~(?{XUfHScmvq0-{7nWZ#Wv&(rF&R&}9c10vxVh2~d{=pZ-nKZ*KWYol z=2ut9brq7e-99K>I*WPwx}Kalup3ha((ux=YP_k_k291?iIXJfp^%e?MK)5bOeyzh zuHJ%!#oOtke_v_8XBsF@`-|gVl<~K7e%?u2H9`2D6q+l189RfePj2JK@`G4XFn~tQ zTR@`xJeY}0MR3*z*%TFadh!Zb(IkPvZ7R&tm6fn~)Buw7-aw!d4>si%V9^3sCYbwM znOyfr@JCDYa5Jpbv7`@6Ji*ZaCvH}l#0Y%wgXK#aiRCgaX6kr1Dx|iFXD2xy>iI)7 zp7)a;cl%D7y)8ICc{&cSeS|B6Zo-mtB6vq7l2=`&g3I0eNN9ov`X?%Z+fV{+?Tp}d zCYfL%&Vz@&eYp0x4jbC}j9%`u#1+F?7=3#lBf+1_^E&p0?$@lsgrRbdv6u?S&f1V% zonORET8$BWlun?bVytE|EYE`9fsL?ARssK>|3NC6#UL@7<9p1@fhfD#5VgXLRh}@P zJecQ+jccO7A?qd0wJd}ELY!t)YZq>x6^WnPtgtpD8aK`pWOZvx6{B4_1{2l{tlQQAqB;|ZxZ&MFeB<4h)-l+W59E1^xG7V zS(R~EuvG}0-;}~Mi%K{vZpmr)wOB3bY}EG1rRBM;beTg4sui7p9k*m~a@$naAdQR1 z8H1HSp9(ON?g3EotDE?4zXxqjN!-8uG;h6{3OnfwkF|T0O!^G%*z)kmv3MXM+JwK~ zt%nd!t912fE=KSChK75*;q#Cn)3TX`U;zcDU&New{(;@A9UFhC$n+4Ts7%i zVo&nhL-^AMWQe15AV?>sL95-l@>NPFao(3_IJB!8r2lDSPsT$W99c}S&FSP-cyoRc zV^4Hh=Y)Hti=k$<2%~J*%ai;e4Y!}IskC^n1&KPNM4=-JzOQ|OJ8!*4uX8itM|m>l zO#RKx4=1qtZW2tt^;KLNa*uuwXy++q&BJ8fe!6gEJGa{*6aNlO=Vt$oc=%Bd?lFpm z$y-d>T29mZ{ed+EO+JN(zdFNfCV|LA3XkQ+2g-bUFU1lM&zcXlVp5Ff3qDOeG#6_% ztf6>wH>rNAjK_HC(6{sl96l0E*;aqj*I0ltZ@uux_RX%E(Q%xN%+0m9WHIlg@SqIRCnqG zj9r@o3oaL9CFiwI4VT4G!>Q!f;4`ResiPhPm*MQcY~q-l${$)pf;cZ9K1Ej ziPaKDBu>OwLW3!ha(>j#=B84Ys?cV{S(xOuo?0 zOW3m*bSHGdx(k)`>3+g7Nms&hb-+mhmq=a2Z|XBu7_PQjjNOBi+h#HmBTHby3ro~) z<}okb?_tN^OiWx5NasxFyk~na60Lb&uueOge%`Q=n7BCdwEv3X_L=5*dCNHH@Rx!< zEpAs}%~^Q%Wis?GDZpfzB3#p~1R)a^qsGzm5SWkTzX?m3+abnq>)K=T;fxGi^U0vE z&M7c<%CY!u?NXFVR%A8!YrwT!jy-8_&j>_+Bl7$bvM%Kw?U<&+4h`9d@rD)f3tR}l*hFU6Su zs$?1UJ$i6+ssp5%%d+=aGFCU5C2{*cKEDOgZ95Pze5SF7FTlmDrD%VBJx-i=6$O&# zG2&^1C=sSgtnJ4!E^iglXO9xS#lD4SoM5IUehQ`@UdqpU_k=7q1QE+_OWHvV<9t&&TpifW{zgcZ$j(MoT{{e&r Br?CJ4 literal 0 HcmV?d00001 diff --git a/include/svs/leanvec/leanvec_common.h b/include/svs/leanvec/leanvec_common.h index 3a93a034..c6d948f3 100644 --- a/include/svs/leanvec/leanvec_common.h +++ b/include/svs/leanvec/leanvec_common.h @@ -26,6 +26,8 @@ template struct UsingLVQ {}; // Hoist out schemas for reuse while auto-loading. inline constexpr std::string_view lean_dataset_schema = "leanvec_dataset"; inline constexpr lib::Version lean_dataset_save_version = lib::Version(0, 0, 0); +inline constexpr std::string_view fallback_schema = "leanvec_fallback"; +inline constexpr lib::Version fallback_save_version = lib::Version(0, 0, 0); namespace detail { diff --git a/include/svs/leanvec/leanvec_concept.h b/include/svs/leanvec/leanvec_concept.h index 69e3eb07..1ce688c8 100644 --- a/include/svs/leanvec/leanvec_concept.h +++ b/include/svs/leanvec/leanvec_concept.h @@ -150,53 +150,77 @@ struct Matcher { public: ///// Loading. static bool check_load_compatibility(std::string_view schema, lib::Version version) { - return schema == lean_dataset_schema && version == lean_dataset_save_version; + if (schema == lean_dataset_schema && version == lean_dataset_save_version) { + return true; + } + if (schema == fallback_schema && version == fallback_save_version) { + return true; + } + return false; } static lib::TryLoadResult try_load(const lib::ContextFreeLoadTable& table) { + auto schema = table.schema(); // For each of the primary and secondary, use the combinations of expected // expected types until we have a successful match. auto primary_expected = detect_data(table.at("primary")); if (!primary_expected) { return lib::Unexpected(primary_expected.error()); } - - auto secondary_expected = detect_data(table.at("secondary")); - if (!secondary_expected) { - return lib::Unexpected(secondary_expected.error()); - } - const auto& primary = primary_expected.value(); - const auto& secondary = secondary_expected.value(); - return Matcher{ - .leanvec_dims = primary.dims, - .total_dims = secondary.dims, - .primary_kind = primary.kind, - .secondary_kind = secondary.kind}; + if (schema == lean_dataset_schema) { + auto secondary_expected = detect_data(table.at("secondary")); + if (!secondary_expected) { + return lib::Unexpected(secondary_expected.error()); + } + const auto& secondary = secondary_expected.value(); + return Matcher{ + .leanvec_dims = primary.dims, + .total_dims = secondary.dims, + .primary_kind = primary.kind, + .secondary_kind = secondary.kind}; + } + else if (schema == fallback_schema) { + return Matcher{ + .leanvec_dims = primary.dims, + .total_dims = primary.dims, + .primary_kind = primary.kind, + .secondary_kind = LeanVecKind::float32}; + } + else { + // TODO raise exception + throw ANNEXCEPTION("Invalid schema!"); + } } static Matcher load(const lib::ContextFreeLoadTable& table) { + auto schema = table.schema(); // For each of the primary and secondary, use the combinations of expected // expected types until we have a successful match. auto primary_expected = detect_data(table.at("primary")); if (!primary_expected) { throw ANNEXCEPTION("Could not match the primary dataset!"); } - - auto secondary_expected = detect_data(table.at("secondary")); - if (!secondary_expected) { - throw ANNEXCEPTION("Could not match the secondary dataset!"); - } - const auto& primary = primary_expected.value(); - const auto& secondary = secondary_expected.value(); + if (schema == lean_dataset_schema) { + auto secondary_expected = detect_data(table.at("secondary")); + if (!secondary_expected) { + throw ANNEXCEPTION("Could not match the secondary dataset!"); + } + const auto& secondary = secondary_expected.value(); + return Matcher{ + .leanvec_dims = primary.dims, + .total_dims = secondary.dims, + .primary_kind = primary.kind, + .secondary_kind = secondary.kind}; + } return Matcher{ .leanvec_dims = primary.dims, - .total_dims = secondary.dims, + .total_dims = primary.dims, .primary_kind = primary.kind, - .secondary_kind = secondary.kind}; + .secondary_kind = LeanVecKind::float32}; } constexpr bool friend operator==(const Matcher&, const Matcher&) = default; diff --git a/include/svs/leanvec/leanvec_fallback.h b/include/svs/leanvec/leanvec_fallback.h index 362cfe64..87e48151 100644 --- a/include/svs/leanvec/leanvec_fallback.h +++ b/include/svs/leanvec/leanvec_fallback.h @@ -151,8 +151,8 @@ class LeanDataset { return LeanDataset{primary}; } - static constexpr lib::Version save_version = lib::Version(0, 0, 0); - static constexpr std::string_view serialization_schema = "leanvec_fallback"; + static constexpr lib::Version save_version = fallback_save_version; + static constexpr std::string_view serialization_schema = fallback_schema; lib::SaveTable save(const lib::SaveContext& ctx) const { return lib::SaveTable( serialization_schema, diff --git a/include/svs/quantization/lvq/lvq_common.h b/include/svs/quantization/lvq/lvq_common.h index 10d44502..bd5b5f42 100644 --- a/include/svs/quantization/lvq/lvq_common.h +++ b/include/svs/quantization/lvq/lvq_common.h @@ -59,8 +59,10 @@ inline constexpr std::string_view one_level_serialization_schema = "one_level_lv inline constexpr lib::Version one_level_save_version = lib::Version(0, 0, 2); inline constexpr std::string_view two_level_serialization_schema = "two_level_lvq_dataset"; inline constexpr lib::Version two_level_save_version = lib::Version(0, 0, 3); +inline constexpr std::string_view fallback_serialization_schema = "fallback_dataset"; +inline constexpr lib::Version fallback_save_version = lib::Version(0, 0, 0); -enum class DatasetSchema { Compressed, ScaledBiased }; +enum class DatasetSchema { Compressed, ScaledBiased, Fallback }; /// /// Support for deduction. /// @@ -73,6 +75,9 @@ inline constexpr std::string_view get_schema(DatasetSchema kind) { case ScaledBiased: { return "lvq_with_scaling_constants"; } + case Fallback: { + return "uncompressed_data"; + } } throw ANNEXCEPTION("Invalid schema!"); } @@ -86,6 +91,9 @@ inline constexpr lib::Version get_current_version(DatasetSchema kind) { case ScaledBiased: { return lib::Version(0, 0, 3); } + case Fallback: { + return lib::Version(0, 0, 0); + } } throw ANNEXCEPTION("Invalid schema!"); } @@ -101,6 +109,10 @@ struct DatasetSummary { version == get_current_version(ScaledBiased)) { return true; } + if (schema == get_schema(Fallback) && + version == get_current_version(Fallback)) { + return true; + } return false; } @@ -122,6 +134,13 @@ struct DatasetSummary { .dims = lib::load_at(table, "logical_dimensions"), .bits = lib::load_at(table, "bits")}; } + if (schema == get_schema(Fallback)) { + return DatasetSummary{ + .kind = Fallback, + .is_signed = false,//??? + .dims = lib::load_at(table, "dims"), + .bits = 32};//??? + } throw ANNEXCEPTION("Invalid table schema {}!", schema); } diff --git a/include/svs/quantization/lvq/lvq_concept.h b/include/svs/quantization/lvq/lvq_concept.h index 4afa0169..a2c882c5 100644 --- a/include/svs/quantization/lvq/lvq_concept.h +++ b/include/svs/quantization/lvq/lvq_concept.h @@ -90,26 +90,6 @@ template < typename Alloc> struct LVQLoader; -namespace detail { - -template -constexpr bool is_compatible(LVQStrategyDispatch strategy) { - switch (strategy) { - case LVQStrategyDispatch::Auto: { - return true; - } - case LVQStrategyDispatch::Sequential: { - return std::is_same_v; - } - case LVQStrategyDispatch::Turbo: { - return TurboLike; - } - } - throw ANNEXCEPTION("Could not match strategy!"); -} - -} // namespace detail - struct Matcher { // Load a matcher for either one or two level datasets. static bool check_load_compatibility(std::string_view schema, lib::Version version) { @@ -119,6 +99,9 @@ struct Matcher { if (schema == two_level_serialization_schema && version == two_level_save_version) { return true; } + if (schema == fallback_serialization_schema && version == fallback_save_version) { + return true; + } return false; } @@ -138,6 +121,12 @@ struct Matcher { .residual = residual_summary.bits, .dims = primary_summary.dims}; } + if (schema == fallback_serialization_schema) { + return Matcher{ + .primary = primary_summary.bits, + .residual = 0, + .dims = primary_summary.dims}; + } throw ANNEXCEPTION( "Unreachable reached with schema and version ({}, {})!", table.schema(), @@ -191,7 +180,7 @@ int64_t overload_match_strategy(LVQStrategyDispatch strategy) { template int64_t overload_score(size_t p, size_t r, size_t e, LVQStrategyDispatch strategy) { // Reject easy matches. - if (p != Primary || r != Residual) { + if (lvq::check_primary_residual(p, r)) { return lib::invalid_match; } @@ -206,7 +195,7 @@ int64_t overload_score(size_t p, size_t r, size_t e, LVQStrategyDispatch strateg // We know dimensionality matches, now we have to try to match strategy. auto strategy_match = overload_match_strategy(strategy); - if (strategy_match < 0) { + if (lvq::check_strategy_match(strategy_match)) { return lib::invalid_match; } @@ -297,7 +286,7 @@ template > struct ProtoLVQLoader { throw ANNEXCEPTION("Invalid specialization!"); } } - if (Primary != primary_ || Residual != residual_) { + if (lvq::check_primary_residual(primary_, residual_)) { throw ANNEXCEPTION("Encoding bits mismatched!"); } if (!detail::is_compatible(strategy_)) { diff --git a/include/svs/quantization/lvq/lvq_fallback.h b/include/svs/quantization/lvq/lvq_fallback.h index 63afb6a6..b464add2 100644 --- a/include/svs/quantization/lvq/lvq_fallback.h +++ b/include/svs/quantization/lvq/lvq_fallback.h @@ -149,8 +149,8 @@ class LVQDataset { } - static constexpr lib::Version save_version = lib::Version(0, 0, 0); - static constexpr std::string_view serialization_schema = "lvq_fallback"; + static constexpr lib::Version save_version = fallback_save_version; + static constexpr std::string_view serialization_schema = fallback_serialization_schema; lib::SaveTable save(const lib::SaveContext& ctx) const { return lib::SaveTable( serialization_schema, @@ -168,6 +168,25 @@ class LVQDataset { } }; +// No constraints on fallback for primary, residual, strategy +template +inline bool check_primary_residual(size_t SVS_UNUSED(p), size_t SVS_UNUSED(r)) { + return false; +} + +inline bool check_strategy_match(int64_t SVS_UNUSED(strategy_match)) { + return false; +} + +namespace detail { + +template +constexpr bool is_compatible(LVQStrategyDispatch SVS_UNUSED(strategy)) { + return true; +} + +} // namespace detail + } // namespace lvq } // namespace quantization } // namespace svs From a75439b813e7a6f0ed239aec29e44f3887b1ff34 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 22 Apr 2025 15:14:52 -0700 Subject: [PATCH 14/23] header fix --- bindings/python/src/svs/leanvec.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/bindings/python/src/svs/leanvec.py b/bindings/python/src/svs/leanvec.py index d1492dee..9da95261 100644 --- a/bindings/python/src/svs/leanvec.py +++ b/bindings/python/src/svs/leanvec.py @@ -1,15 +1,16 @@ -# Copyright (C) 2023 Intel Corporation +# Copyright 2023 Intel Corporation # -# This software and the related documents are Intel copyrighted materials, -# and your use of them is governed by the express license under which they -# were provided to you ("License"). Unless the License provides otherwise, -# you may not use, modify, copy, publish, distribute, disclose or transmit -# this software or the related documents without Intel's prior written -# permission. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# This software and the related documents are provided as is, with no -# express or implied warranties, other than those that are expressly stated -# in the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import numpy as np from typing import Tuple From aab98cf5d8cf8ed5f6c571da9261bcc20b3867b7 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 24 Apr 2025 00:48:31 -0700 Subject: [PATCH 15/23] remove accidental print --- bindings/python/tests/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bindings/python/tests/common.py b/bindings/python/tests/common.py index 0497c8c8..b8b870a7 100644 --- a/bindings/python/tests/common.py +++ b/bindings/python/tests/common.py @@ -24,7 +24,6 @@ # directory of the SVS project. _current_file = Path(__file__).parent.resolve() #/svs/bindings/python/tests ROOT_DIR = _current_file.parents[2] -print("Root:", ROOT_DIR) TEST_DATASET_DIR = ROOT_DIR.joinpath("data", "test_dataset") # Main exports From 3cbe104bb4eaf34b6bf2d30f282a2aa21335d7dd Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 24 Apr 2025 10:40:36 -0700 Subject: [PATCH 16/23] quick fixes for review --- bindings/python/src/svs/common.py | 2 +- bindings/python/src/svs/leanvec.py | 2 +- bindings/python/tests/common.py | 4 ++-- bindings/python/tests/test_common.py | 8 ++++---- bindings/python/tests/test_dynamic_vamana.py | 2 +- bindings/python/tests/test_flat.py | 2 +- bindings/python/tests/test_loader_api.py | 2 +- bindings/python/tests/test_reconstruction.py | 2 +- bindings/python/tests/test_vamana.py | 14 +++++++------- examples/cpp/fallback.cpp | 2 +- examples/python/example_fallback.py | 14 +------------- examples/python/example_fallback_leanvec.py | 4 ++-- include/svs/fallback/fallback.h | 2 +- include/svs/fallback/fallback_mode.h | 2 +- include/svs/index/vamana/extensions.h | 2 +- include/svs/leanvec/leanvec_common.h | 2 +- include/svs/leanvec/leanvec_concept.h | 2 +- include/svs/leanvec/leanvec_fallback.h | 3 +-- include/svs/quantization/lvq/lvq_common.h | 2 +- include/svs/quantization/lvq/lvq_concept.h | 2 +- include/svs/quantization/lvq/lvq_fallback.h | 2 +- tests/svs/fallback/fallback.cpp | 2 +- tests/svs/fallback/utils.h | 2 +- 23 files changed, 34 insertions(+), 47 deletions(-) diff --git a/bindings/python/src/svs/common.py b/bindings/python/src/svs/common.py index 010ac7c6..d827e191 100644 --- a/bindings/python/src/svs/common.py +++ b/bindings/python/src/svs/common.py @@ -58,7 +58,7 @@ def np_to_svs(nptype): if nptype == np.float64: return lib.float64 - raise Exception(f"Could not convert {nptype} to a svs.DataType enum!"); + raise Exception(f"Could not convert {nptype} to a svs.DataType enum!") def read_npy(filename: str): """ diff --git a/bindings/python/src/svs/leanvec.py b/bindings/python/src/svs/leanvec.py index 9da95261..15284725 100644 --- a/bindings/python/src/svs/leanvec.py +++ b/bindings/python/src/svs/leanvec.py @@ -1,4 +1,4 @@ -# Copyright 2023 Intel Corporation +# Copyright 2025 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bindings/python/tests/common.py b/bindings/python/tests/common.py index b8b870a7..8b34454c 100644 --- a/bindings/python/tests/common.py +++ b/bindings/python/tests/common.py @@ -78,7 +78,7 @@ def get_test_set(A, num_entries: int): """ assert(A.ndim == 2) assert(A.shape[0] >= num_entries) - return A[-num_entries:]; + return A[-num_entries:] def test_threading(f, *args, validate = None, iters = 4, print_times = False): """ @@ -156,5 +156,5 @@ def test_close_lvq(original, reconstructed, primary_bits: int, residual_bits: in # Ensure that each reconstructed value is within the target threshold (plus a tiny # fudge factor to help offset rounding imprecision. upper_bound = np.expand_dims(deltas, axis = 1) - upper_bound = upper_bound + 0.0125 * upper_bound; + upper_bound = upper_bound + 0.0125 * upper_bound return np.all(np.abs(original - reconstructed) <= upper_bound) diff --git a/bindings/python/tests/test_common.py b/bindings/python/tests/test_common.py index 8d283c8c..f371ac56 100644 --- a/bindings/python/tests/test_common.py +++ b/bindings/python/tests/test_common.py @@ -141,28 +141,28 @@ def test_vecs_extension_checking(self): self.assertTrue(x.dtype == np.float32) self.assertRaises( RuntimeError, svs.write_vecs, x, os.path.join(self.tempdir_name, "temp.hvecs") - ); + ) # Half x = svs.common.random_dataset(10, 128, dtype = np.float16) self.assertTrue(x.dtype == np.float16) self.assertRaises( RuntimeError, svs.write_vecs, x, os.path.join(self.tempdir_name, "temp.fvecs") - ); + ) # UInt32 x = svs.common.random_dataset(10, 128, dtype = np.uint32) self.assertTrue(x.dtype == np.uint32) self.assertRaises( RuntimeError, svs.write_vecs, x, os.path.join(self.tempdir_name, "temp.bvecs") - ); + ) # UInt8 x = svs.common.random_dataset(10, 128, dtype = np.uint8) self.assertTrue(x.dtype == np.uint8) self.assertRaises( RuntimeError, svs.write_vecs, x, os.path.join(self.tempdir_name, "temp.ivecs") - ); + ) def test_generate_test_dataset(self): svs.generate_test_dataset( diff --git a/bindings/python/tests/test_dynamic_vamana.py b/bindings/python/tests/test_dynamic_vamana.py index 84d78217..1586e779 100644 --- a/bindings/python/tests/test_dynamic_vamana.py +++ b/bindings/python/tests/test_dynamic_vamana.py @@ -58,7 +58,7 @@ def recall_check( configdir = os.path.join(tempdir, "config") graphdir = os.path.join(tempdir, "graph") datadir = os.path.join(tempdir, "data") - index.save(configdir, graphdir, datadir); + index.save(configdir, graphdir, datadir) reloaded = svs.DynamicVamana( configdir, diff --git a/bindings/python/tests/test_flat.py b/bindings/python/tests/test_flat.py index 9b629111..77847a99 100644 --- a/bindings/python/tests/test_flat.py +++ b/bindings/python/tests/test_flat.py @@ -121,7 +121,7 @@ def _do_test_from_file(self, distance: svs.DistanceType, queries, groundtruth): svs.VectorDataLoader( test_data_svs, svs.DataType.float32, dims = test_data_dims ) - ); + ) for loader, recall in loaders: index = svs.Flat( loader, diff --git a/bindings/python/tests/test_loader_api.py b/bindings/python/tests/test_loader_api.py index 5f4e968c..a86b5e9b 100644 --- a/bindings/python/tests/test_loader_api.py +++ b/bindings/python/tests/test_loader_api.py @@ -24,7 +24,7 @@ test_data_vecs, \ test_data_dims -DEBUG = False; +DEBUG = False class LoaderAPITester(unittest.TestCase): """ diff --git a/bindings/python/tests/test_reconstruction.py b/bindings/python/tests/test_reconstruction.py index 66cf6ec2..c810fa26 100644 --- a/bindings/python/tests/test_reconstruction.py +++ b/bindings/python/tests/test_reconstruction.py @@ -34,7 +34,7 @@ test_vamana_config, \ test_close_lvq -DEBUG = False; +DEBUG = False class ReconstructionTester(unittest.TestCase): """ diff --git a/bindings/python/tests/test_vamana.py b/bindings/python/tests/test_vamana.py index 61a7e3dd..6c8db9ed 100644 --- a/bindings/python/tests/test_vamana.py +++ b/bindings/python/tests/test_vamana.py @@ -46,7 +46,7 @@ LVQMatcher, \ LeanVecMatcher -DEBUG = False; +DEBUG = False class VamanaTester(unittest.TestCase): """ @@ -68,7 +68,7 @@ def _setup(self, loader: svs.VectorDataLoader): # Generate LeanVec OOD matrices data = svs.read_vecs(test_data_vecs) queries = svs.read_vecs(test_queries) - data_matrix, query_matrix = svs.compute_leanvec_matrices(data, queries, 64); + data_matrix, query_matrix = svs.compute_leanvec_matrices(data, queries, 64) self.loader_and_matcher = [ (loader, UncompressedMatcher("float32")), @@ -222,7 +222,7 @@ def _test_single_query( queries ): - I_full, D_full = vamana.search(queries, 10); + I_full, D_full = vamana.search(queries, 10) I_single = [] D_single = [] @@ -358,7 +358,7 @@ def _test_basic(self, loader, matcher, first_iter: bool = False): configdir = os.path.join(tempdir, "config") graphdir = os.path.join(tempdir, "graph") datadir = os.path.join(tempdir, "data") - vamana.save(configdir, graphdir, datadir); + vamana.save(configdir, graphdir, datadir) # Reload from raw-files. reloaded = svs.Vamana(configdir, graphdir, datadir, svs.DistanceType.L2) @@ -404,7 +404,7 @@ def test_lvq_reload(self): primary = 4, residual = 8, strategy = svs.LVQStrategy.Sequential - ); + ) matcher = LVQMatcher(4, 8) num_threads = 2 @@ -479,7 +479,7 @@ def _test_build( params = self._get_build_parameters( 'vamana_test_build', distance_map[distance], matcher - ); + ) vamana = svs.Vamana.build(params, loader, distance, num_threads = num_threads) print(f"Building: {vamana.experimental_backend_string}") @@ -530,7 +530,7 @@ def test_build(self): # Generate LeanVec OOD matrices queries = svs.read_vecs(test_queries) - data_matrix, query_matrix = svs.compute_leanvec_matrices(data, queries, 64); + data_matrix, query_matrix = svs.compute_leanvec_matrices(data, queries, 64) matcher = UncompressedMatcher("float32") self._test_build(data, svs.DistanceType.L2, matcher) diff --git a/examples/cpp/fallback.cpp b/examples/cpp/fallback.cpp index c2e05674..52070121 100644 --- a/examples/cpp/fallback.cpp +++ b/examples/cpp/fallback.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/python/example_fallback.py b/examples/python/example_fallback.py index 5bd98f49..c49bd25f 100644 --- a/examples/python/example_fallback.py +++ b/examples/python/example_fallback.py @@ -1,4 +1,4 @@ -# Copyright 2023 Intel Corporation +# Copyright 2025 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -45,12 +45,6 @@ def run_test_float(index, queries, groundtruth): ) def run_test_two_level4_8(index, queries, groundtruth): - # expected = { - # 10: 0.5482, - # 20: 0.7294, - # 30: 0.8223, - # 40: 0.8756, - # } expected = { 10: 0.5664, 20: 0.7397, @@ -67,12 +61,6 @@ def run_test_two_level4_8(index, queries, groundtruth): ) def run_test_build_two_level4_8(index, queries, groundtruth): - # expected = { - # 10: 0.5484, - # 20: 0.7295, - # 30: 0.8221, - # 40: 0.8758, - # } expected = { 10: 0.5664, 20: 0.7397, diff --git a/examples/python/example_fallback_leanvec.py b/examples/python/example_fallback_leanvec.py index 48457c9e..64668be4 100644 --- a/examples/python/example_fallback_leanvec.py +++ b/examples/python/example_fallback_leanvec.py @@ -1,4 +1,4 @@ -# Copyright 2023 Intel Corporation +# Copyright 2025 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Import `unittest` to allow for auotmated testing. +# Import `unittest` to allow for automated testing. import unittest # [imports] diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index f1174900..8fd7a002 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/svs/fallback/fallback_mode.h b/include/svs/fallback/fallback_mode.h index 37c82d5c..80d4ed5b 100644 --- a/include/svs/fallback/fallback_mode.h +++ b/include/svs/fallback/fallback_mode.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 9bf5e2e9..2fdc0895 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -596,7 +596,7 @@ SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( #else // USE_PROPRIETARY -template +template SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( svs::tag_t SVS_UNUSED(cpo), const Data& SVS_UNUSED(dataset) diff --git a/include/svs/leanvec/leanvec_common.h b/include/svs/leanvec/leanvec_common.h index c6d948f3..ba7cbec2 100644 --- a/include/svs/leanvec/leanvec_common.h +++ b/include/svs/leanvec/leanvec_common.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/svs/leanvec/leanvec_concept.h b/include/svs/leanvec/leanvec_concept.h index 1ce688c8..85e1a75b 100644 --- a/include/svs/leanvec/leanvec_concept.h +++ b/include/svs/leanvec/leanvec_concept.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/svs/leanvec/leanvec_fallback.h b/include/svs/leanvec/leanvec_fallback.h index 87e48151..6cf0594e 100644 --- a/include/svs/leanvec/leanvec_fallback.h +++ b/include/svs/leanvec/leanvec_fallback.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,6 @@ namespace svs { namespace leanvec { template struct LeanVecMatrices { - // temporary (?) additions public: using leanvec_matrix_type = data::SimpleData; diff --git a/include/svs/quantization/lvq/lvq_common.h b/include/svs/quantization/lvq/lvq_common.h index bd5b5f42..b1ca1554 100644 --- a/include/svs/quantization/lvq/lvq_common.h +++ b/include/svs/quantization/lvq/lvq_common.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/svs/quantization/lvq/lvq_concept.h b/include/svs/quantization/lvq/lvq_concept.h index a2c882c5..cfe29632 100644 --- a/include/svs/quantization/lvq/lvq_concept.h +++ b/include/svs/quantization/lvq/lvq_concept.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/include/svs/quantization/lvq/lvq_fallback.h b/include/svs/quantization/lvq/lvq_fallback.h index b464add2..176bb823 100644 --- a/include/svs/quantization/lvq/lvq_fallback.h +++ b/include/svs/quantization/lvq/lvq_fallback.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/svs/fallback/fallback.cpp b/tests/svs/fallback/fallback.cpp index c925a0a9..2ae29f78 100644 --- a/tests/svs/fallback/fallback.cpp +++ b/tests/svs/fallback/fallback.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/svs/fallback/utils.h b/tests/svs/fallback/utils.h index d0767b69..c1475931 100644 --- a/tests/svs/fallback/utils.h +++ b/tests/svs/fallback/utils.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 Intel Corporation + * Copyright 2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 252fb8eceff02e792db9768721c231a1fcb59ce8 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 24 Apr 2025 10:47:12 -0700 Subject: [PATCH 17/23] clang formatting --- bindings/python/src/core.cpp | 31 ++++++++++++++++---- bindings/python/src/flat.cpp | 2 +- examples/cpp/fallback.cpp | 9 +++--- include/svs/fallback/fallback.h | 2 +- include/svs/fallback/fallback_mode.h | 12 ++++---- include/svs/index/vamana/extensions.h | 3 +- include/svs/leanvec/leanvec_common.h | 6 ++-- include/svs/leanvec/leanvec_concept.h | 8 ++---- include/svs/leanvec/leanvec_fallback.h | 20 +++++++------ include/svs/quantization/lvq/lvq_common.h | 7 ++--- include/svs/quantization/lvq/lvq_fallback.h | 32 +++++++++++---------- tests/svs/fallback/fallback.cpp | 2 +- 12 files changed, 78 insertions(+), 56 deletions(-) diff --git a/bindings/python/src/core.cpp b/bindings/python/src/core.cpp index 2b65c5a2..ce5c423f 100644 --- a/bindings/python/src/core.cpp +++ b/bindings/python/src/core.cpp @@ -330,12 +330,31 @@ void wrap_fallback(py::module& m) { m, "FallbackMode", "Select the fallback mode for LVQ" ) .value("Silent", Silent, "Seamlessly fall back to the default Vamana index.") - .value("Warning", Warning, "Provide results using default Vamana index. Logs a warning message indicated LeanVec/LVQ optimizations are unsupported.") - .value("Error", Error, "Enforces an error, stopping execution if LeanVec/LVQ optimizations are not supported.") + .value( + "Warning", + Warning, + "Provide results using default Vamana index. Logs a warning message indicated " + "LeanVec/LVQ optimizations are unsupported." + ) + .value( + "Error", + Error, + "Enforces an error, stopping execution if LeanVec/LVQ optimizations are not " + "supported." + ) .export_values(); - m.def("set_fallback_mode", [](svs::fallback::FallbackMode mode) { svs::fallback::set_mode(mode); }, py::arg("mode"), "Set the LVQ mode."); - m.def("get_fallback_mode", []() { return svs::fallback::get_mode(); }, "Get the current LVQ mode."); + m.def( + "set_fallback_mode", + [](svs::fallback::FallbackMode mode) { svs::fallback::set_mode(mode); }, + py::arg("mode"), + "Set the LVQ mode." + ); + m.def( + "get_fallback_mode", + []() { return svs::fallback::get_mode(); }, + "Get the current LVQ mode." + ); } /// Generate bindings for LVQ compressors and loaders. @@ -621,10 +640,10 @@ Construct a new ``svs.GraphLoader``. return svs::lib::begin_deserialization(path); })); py::implicitly_convertible(); - + ///// Fallback wrap_fallback(m); - + ///// LVQ wrap_lvq(m); diff --git a/bindings/python/src/flat.cpp b/bindings/python/src/flat.cpp index 182f9644..765ed8ed 100644 --- a/bindings/python/src/flat.cpp +++ b/bindings/python/src/flat.cpp @@ -21,10 +21,10 @@ #include "svs/python/manager.h" // svs -#include "svs/quantization/lvq/lvq_concept.h" #include "svs/lib/datatype.h" #include "svs/lib/dispatcher.h" #include "svs/orchestrators/exhaustive.h" +#include "svs/quantization/lvq/lvq_concept.h" // stl #include diff --git a/examples/cpp/fallback.cpp b/examples/cpp/fallback.cpp index 52070121..6c3b57d3 100644 --- a/examples/cpp/fallback.cpp +++ b/examples/cpp/fallback.cpp @@ -18,9 +18,9 @@ //! [Includes] // SVS Dependencies -#include "svs/orchestrators/vamana.h" // bulk of the dependencies required. -#include "svs/core/recall.h" // Convenient k-recall@n computation. #include "svs/fallback/fallback.h" +#include "svs/core/recall.h" // Convenient k-recall@n computation. +#include "svs/orchestrators/vamana.h" // bulk of the dependencies required. // Alternative main definition #include "svsmain.h" @@ -198,9 +198,8 @@ int svs_main(std::vector args) { auto compressor_lean = svs::lib::Lazy([=](svs::threads::ThreadPool auto& threadpool) { auto data = svs::VectorDataLoader("example_data").load(); - return leanvec::LeanDataset, leanvec::UsingLVQ<8>, 64, 128>::reduce( - data, std::nullopt, threadpool, padding - ); + return leanvec::LeanDataset, leanvec::UsingLVQ<8>, 64, 128>:: + reduce(data, std::nullopt, threadpool, padding); }); index = svs::Vamana::assemble( "example_config", diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index 8fd7a002..91e13cce 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -17,8 +17,8 @@ #pragma once #include "svs/fallback/fallback_mode.h" -#include "svs/quantization/lvq/lvq_concept.h" #include "svs/leanvec/leanvec_concept.h" +#include "svs/quantization/lvq/lvq_concept.h" #ifdef USE_PROPRIETARY diff --git a/include/svs/fallback/fallback_mode.h b/include/svs/fallback/fallback_mode.h index 80d4ed5b..1043b953 100644 --- a/include/svs/fallback/fallback_mode.h +++ b/include/svs/fallback/fallback_mode.h @@ -30,11 +30,13 @@ inline FallbackMode get_mode() { return mode; } class UnsupportedHardwareError : public std::runtime_error { public: explicit UnsupportedHardwareError() - : std::runtime_error{"LVQ and Leanvec functionality of SVS is not supported on non-Intel hardware."} {} + : std::runtime_error{"LVQ and Leanvec functionality of SVS is not supported on " + "non-Intel hardware."} {} }; -constexpr const char* fallback_warning = "LVQ and Leanvec functionality of SVS is not supported on non-Intel hardware. " - "Using uncompressed data.\n"; +constexpr const char* fallback_warning = + "LVQ and Leanvec functionality of SVS is not supported on non-Intel hardware. " + "Using uncompressed data.\n"; -} -} +} // namespace fallback +} // namespace svs diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 2fdc0895..893f916f 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -598,8 +598,7 @@ SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( template SVS_FORCE_INLINE data::GetDatumAccessor svs_invoke( - svs::tag_t SVS_UNUSED(cpo), - const Data& SVS_UNUSED(dataset) + svs::tag_t SVS_UNUSED(cpo), const Data& SVS_UNUSED(dataset) ) { return data::GetDatumAccessor(); } diff --git a/include/svs/leanvec/leanvec_common.h b/include/svs/leanvec/leanvec_common.h index ba7cbec2..a32db637 100644 --- a/include/svs/leanvec/leanvec_common.h +++ b/include/svs/leanvec/leanvec_common.h @@ -34,11 +34,11 @@ namespace detail { template inline constexpr bool is_using_lvq_tag_v = false; template inline constexpr bool is_using_lvq_tag_v> = true; -} +} // namespace detail // Compatible type parameters for LeanDatasets template concept LeanCompatible = has_datatype_v || detail::is_using_lvq_tag_v; -} -} +} // namespace leanvec +} // namespace svs diff --git a/include/svs/leanvec/leanvec_concept.h b/include/svs/leanvec/leanvec_concept.h index 85e1a75b..044bba8c 100644 --- a/include/svs/leanvec/leanvec_concept.h +++ b/include/svs/leanvec/leanvec_concept.h @@ -94,7 +94,7 @@ template <> struct LeanVecPicker> { template inline constexpr LeanVecKind leanvec_kind_v = detail::LeanVecPicker::value; - + // LeanDataset Matcher struct Matcher { private: @@ -180,15 +180,13 @@ struct Matcher { .total_dims = secondary.dims, .primary_kind = primary.kind, .secondary_kind = secondary.kind}; - } - else if (schema == fallback_schema) { + } else if (schema == fallback_schema) { return Matcher{ .leanvec_dims = primary.dims, .total_dims = primary.dims, .primary_kind = primary.kind, .secondary_kind = LeanVecKind::float32}; - } - else { + } else { // TODO raise exception throw ANNEXCEPTION("Invalid schema!"); } diff --git a/include/svs/leanvec/leanvec_fallback.h b/include/svs/leanvec/leanvec_fallback.h index 6cf0594e..0032c619 100644 --- a/include/svs/leanvec/leanvec_fallback.h +++ b/include/svs/leanvec/leanvec_fallback.h @@ -16,9 +16,9 @@ #pragma once -#include "svs/quantization/lvq/lvq_fallback.h" #include "svs/fallback/fallback_mode.h" #include "svs/leanvec/leanvec_common.h" +#include "svs/quantization/lvq/lvq_fallback.h" // #include leanvec_common.h @@ -43,6 +43,7 @@ template struct LeanVecMatrices { throw ANNEXCEPTION("Mismatched data and query matrix dimensions!"); } } + private: leanvec_matrix_type data_matrix_; leanvec_matrix_type query_matrix_; @@ -63,7 +64,7 @@ template struct select_rebind_allocator { }; template using select_rebind_allocator_t = typename select_rebind_allocator::type; -} +} // namespace detail template < typename T1, @@ -74,17 +75,21 @@ template < class LeanDataset { public: using allocator_type = detail::select_rebind_allocator_t; + private: data::SimpleData primary_; + public: static constexpr bool is_resizeable = detail::is_blocked; using leanvec_matrices_type = LeanVecMatrices; - using const_value_type = typename data::SimpleData::const_value_type; + using const_value_type = + typename data::SimpleData::const_value_type; using element_type = float; using value_type = const_value_type; using primary_type = data::SimpleData; - LeanDataset(primary_type primary): primary_{std::move(primary)} { + LeanDataset(primary_type primary) + : primary_{std::move(primary)} { if (fallback::get_mode() == fallback::FallbackMode::Error) { throw fallback::UnsupportedHardwareError(); } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { @@ -145,7 +150,8 @@ class LeanDataset { lib::MaybeStatic SVS_UNUSED(leanvec_dims) = {}, const Alloc& allocator = {} ) { - primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; + primary_type primary = + primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; svs::data::copy(data, primary); return LeanDataset{primary}; } @@ -154,9 +160,7 @@ class LeanDataset { static constexpr std::string_view serialization_schema = fallback_schema; lib::SaveTable save(const lib::SaveContext& ctx) const { return lib::SaveTable( - serialization_schema, - save_version, - {SVS_LIST_SAVE_(primary, ctx)} + serialization_schema, save_version, {SVS_LIST_SAVE_(primary, ctx)} ); } diff --git a/include/svs/quantization/lvq/lvq_common.h b/include/svs/quantization/lvq/lvq_common.h index b1ca1554..fde2dd29 100644 --- a/include/svs/quantization/lvq/lvq_common.h +++ b/include/svs/quantization/lvq/lvq_common.h @@ -109,8 +109,7 @@ struct DatasetSummary { version == get_current_version(ScaledBiased)) { return true; } - if (schema == get_schema(Fallback) && - version == get_current_version(Fallback)) { + if (schema == get_schema(Fallback) && version == get_current_version(Fallback)) { return true; } return false; @@ -137,9 +136,9 @@ struct DatasetSummary { if (schema == get_schema(Fallback)) { return DatasetSummary{ .kind = Fallback, - .is_signed = false,//??? + .is_signed = false, .dims = lib::load_at(table, "dims"), - .bits = 32};//??? + .bits = 32}; } throw ANNEXCEPTION("Invalid table schema {}!", schema); } diff --git a/include/svs/quantization/lvq/lvq_fallback.h b/include/svs/quantization/lvq/lvq_fallback.h index 176bb823..c5724fa9 100644 --- a/include/svs/quantization/lvq/lvq_fallback.h +++ b/include/svs/quantization/lvq/lvq_fallback.h @@ -17,9 +17,9 @@ #pragma once #include "svs/core/data/simple.h" -#include "svs/lib/threads.h" -#include "svs/lib/saveload/save.h" #include "svs/fallback/fallback_mode.h" +#include "svs/lib/saveload/save.h" +#include "svs/lib/threads.h" #include "svs/quantization/lvq/lvq_common.h" namespace fallback = svs::fallback; @@ -63,7 +63,7 @@ template struct select_rebind_allocator { template using select_rebind_allocator_t = typename select_rebind_allocator::type; -} +} // namespace detail template concept LVQPackingStrategy = detail::is_lvq_packing_strategy_v; @@ -81,11 +81,14 @@ template < class LVQDataset { public: using allocator_type = detail::select_rebind_allocator_t; + private: data::SimpleData primary_; + public: static constexpr bool is_resizeable = detail::is_blocked; - using const_value_type = typename data::SimpleData::const_value_type; + using const_value_type = + typename data::SimpleData::const_value_type; using element_type = float; using value_type = const_value_type; using primary_type = data::SimpleData; @@ -102,7 +105,8 @@ class LVQDataset { } template - LVQDataset(Dataset primary): primary_{primary} { + LVQDataset(Dataset primary) + : primary_{primary} { if (fallback::get_mode() == fallback::FallbackMode::Error) { throw fallback::UnsupportedHardwareError(); } else if (fallback::get_mode() == fallback::FallbackMode::Warning) { @@ -116,7 +120,9 @@ class LVQDataset { void prefetch(size_t i) const { primary_.prefetch(i); } template - void set_datum(size_t i, std::span datum, size_t SVS_UNUSED(centroid_selector) = 0) { + void set_datum( + size_t i, std::span datum, size_t SVS_UNUSED(centroid_selector) = 0 + ) { primary_.set_datum(i, datum); } @@ -143,19 +149,17 @@ class LVQDataset { size_t SVS_UNUSED(alignment), const Alloc& allocator = {} ) { - primary_type primary = primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; + primary_type primary = + primary_type{data.size(), data.dimensions(), allocator_type{allocator}}; svs::data::copy(data, primary); return LVQDataset{primary}; } - static constexpr lib::Version save_version = fallback_save_version; static constexpr std::string_view serialization_schema = fallback_serialization_schema; lib::SaveTable save(const lib::SaveContext& ctx) const { return lib::SaveTable( - serialization_schema, - save_version, - {SVS_LIST_SAVE_(primary, ctx)} + serialization_schema, save_version, {SVS_LIST_SAVE_(primary, ctx)} ); } @@ -169,14 +173,12 @@ class LVQDataset { }; // No constraints on fallback for primary, residual, strategy -template +template inline bool check_primary_residual(size_t SVS_UNUSED(p), size_t SVS_UNUSED(r)) { return false; } -inline bool check_strategy_match(int64_t SVS_UNUSED(strategy_match)) { - return false; -} +inline bool check_strategy_match(int64_t SVS_UNUSED(strategy_match)) { return false; } namespace detail { diff --git a/tests/svs/fallback/fallback.cpp b/tests/svs/fallback/fallback.cpp index 2ae29f78..47e92763 100644 --- a/tests/svs/fallback/fallback.cpp +++ b/tests/svs/fallback/fallback.cpp @@ -15,9 +15,9 @@ */ // SVS +#include "svs/fallback/fallback.h" #include "svs/core/recall.h" #include "svs/lib/static.h" -#include "svs/fallback/fallback.h" #include "svs/orchestrators/dynamic_vamana.h" #include "svs/orchestrators/exhaustive.h" #include "svs/orchestrators/vamana.h" From 5288ca0f926f9941e11020e5315c6cdd54a68201 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Thu, 24 Apr 2025 11:21:16 -0700 Subject: [PATCH 18/23] update include statements --- include/svs/fallback/fallback_mode.h | 2 ++ include/svs/leanvec/leanvec_common.h | 12 ++++++++++++ include/svs/quantization/lvq/lvq_common.h | 16 ++++++++++++++++ include/svs/quantization/lvq/lvq_fallback.h | 2 -- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/include/svs/fallback/fallback_mode.h b/include/svs/fallback/fallback_mode.h index 1043b953..d698cb9e 100644 --- a/include/svs/fallback/fallback_mode.h +++ b/include/svs/fallback/fallback_mode.h @@ -16,6 +16,8 @@ #pragma once +#include + namespace svs { namespace fallback { diff --git a/include/svs/leanvec/leanvec_common.h b/include/svs/leanvec/leanvec_common.h index a32db637..a0d15094 100644 --- a/include/svs/leanvec/leanvec_common.h +++ b/include/svs/leanvec/leanvec_common.h @@ -16,6 +16,18 @@ #pragma once +// svs +#include "svs/core/data.h" + +// stl +#include +#include +#include +#include + +// third-party +#include "fmt/core.h" + namespace svs { namespace leanvec { diff --git a/include/svs/quantization/lvq/lvq_common.h b/include/svs/quantization/lvq/lvq_common.h index fde2dd29..62a66f54 100644 --- a/include/svs/quantization/lvq/lvq_common.h +++ b/include/svs/quantization/lvq/lvq_common.h @@ -16,6 +16,22 @@ #pragma once +// svs +#include "eve/algo.hpp" +#include "svs/core/data.h" +#include "svs/core/distance.h" +#include "svs/core/kmeans.h" +#include "svs/lib/dispatcher.h" +#include "svs/lib/meta.h" +#include "svs/lib/misc.h" +#include "svs/lib/saveload.h" + +// stl +#include +#include +#include +#include + namespace svs { namespace quantization { namespace lvq { diff --git a/include/svs/quantization/lvq/lvq_fallback.h b/include/svs/quantization/lvq/lvq_fallback.h index c5724fa9..1ca8566d 100644 --- a/include/svs/quantization/lvq/lvq_fallback.h +++ b/include/svs/quantization/lvq/lvq_fallback.h @@ -18,8 +18,6 @@ #include "svs/core/data/simple.h" #include "svs/fallback/fallback_mode.h" -#include "svs/lib/saveload/save.h" -#include "svs/lib/threads.h" #include "svs/quantization/lvq/lvq_common.h" namespace fallback = svs::fallback; From 90571fc2b0f77d376854d66958f97f2762f3cb17 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 28 Apr 2025 17:04:08 -0700 Subject: [PATCH 19/23] address a few more comments --- include/svs/fallback/fallback_mode.h | 2 +- include/svs/index/vamana/extensions.h | 1 - include/svs/leanvec/leanvec_concept.h | 34 --------------------------- 3 files changed, 1 insertion(+), 36 deletions(-) diff --git a/include/svs/fallback/fallback_mode.h b/include/svs/fallback/fallback_mode.h index d698cb9e..169c9fd1 100644 --- a/include/svs/fallback/fallback_mode.h +++ b/include/svs/fallback/fallback_mode.h @@ -36,7 +36,7 @@ class UnsupportedHardwareError : public std::runtime_error { "non-Intel hardware."} {} }; -constexpr const char* fallback_warning = +inline constexpr const char* fallback_warning = "LVQ and Leanvec functionality of SVS is not supported on non-Intel hardware. " "Using uncompressed data.\n"; diff --git a/include/svs/index/vamana/extensions.h b/include/svs/index/vamana/extensions.h index 893f916f..bf45c18b 100644 --- a/include/svs/index/vamana/extensions.h +++ b/include/svs/index/vamana/extensions.h @@ -583,7 +583,6 @@ struct Reconstruct { // Customization point for reconstructing vectors. inline constexpr Reconstruct reconstruct_accessor{}; -// TOOD: unify these #ifdef USE_PROPRIETARY template diff --git a/include/svs/leanvec/leanvec_concept.h b/include/svs/leanvec/leanvec_concept.h index 044bba8c..aa106a61 100644 --- a/include/svs/leanvec/leanvec_concept.h +++ b/include/svs/leanvec/leanvec_concept.h @@ -304,10 +304,6 @@ template > struct ProtoLeanVecLoader explicit ProtoLeanVecLoader( Reload reloader, - // size_t leanvec_dims, - // size_t dims, - // LeanVecKind primary_kind, - // LeanVecKind secondary_kind, size_t alignment = 0, const Alloc& allocator = {} ) @@ -471,36 +467,6 @@ struct lib::DispatchConverter< return overload_score( loader.primary_kind_, loader.leanvec_dims_, loader.secondary_kind_, loader.dims_ ); - // if (loader.primary_kind_ != leanvec::leanvec_kind_v) { - // return lib::invalid_match; - // } - - // // Check secondary kind - // if (loader.secondary_kind_ != leanvec::leanvec_kind_v) { - // return lib::invalid_match; - // } - - // // Check extent-tags. - // auto extent_match = lib::dispatch_match>( - // lib::ExtentArg{loader.dims_} - // ); - - // // If extents don't match, then we abort immediately. - // if (extent_match < 0) { - // return lib::invalid_match; - // } - - // // Check leanvec_dims-tags. - // auto leanvec_dims_match = - // lib::dispatch_match>(lib::ExtentArg{ loader.leanvec_dims_}); - // // If leanvec_dims don't match, then we abort immediately. - // if (leanvec_dims_match < 0) { - // return lib::invalid_match; - // } - - // return extent_match + leanvec_dims_match; } static leanvec::LeanVecLoader From 93b5a73d8884b616134c276208d552eaec1d9a1a Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 2 May 2025 12:37:19 -0700 Subject: [PATCH 20/23] clean up ugly paths within USE_PROPRIETARY --- bindings/python/include/svs/python/core.h | 2 +- include/svs/fallback/fallback.h | 2 +- include/svs/leanvec/leanvec_concept.h | 2 +- include/svs/quantization/lvq/lvq_concept.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bindings/python/include/svs/python/core.h b/bindings/python/include/svs/python/core.h index c2921ac7..c6bb3631 100644 --- a/bindings/python/include/svs/python/core.h +++ b/bindings/python/include/svs/python/core.h @@ -31,7 +31,7 @@ #ifdef USE_PROPRIETARY -#include "../../../../../../include/svs/fallback/fallback_python.h" +#include "svs/fallback/fallback_python.h" #endif // USE_PROPRIETARY diff --git a/include/svs/fallback/fallback.h b/include/svs/fallback/fallback.h index 91e13cce..cfa72444 100644 --- a/include/svs/fallback/fallback.h +++ b/include/svs/fallback/fallback.h @@ -22,6 +22,6 @@ #ifdef USE_PROPRIETARY -#include "../../../../include/svs/fallback/fallback_cpp.h" +#include "svs/fallback/fallback_cpp.h" #endif // USE_PROPRIETARY diff --git a/include/svs/leanvec/leanvec_concept.h b/include/svs/leanvec/leanvec_concept.h index aa106a61..85277528 100644 --- a/include/svs/leanvec/leanvec_concept.h +++ b/include/svs/leanvec/leanvec_concept.h @@ -24,7 +24,7 @@ #else // USE_PROPRIETARY -#include "../../../../include/svs/leanvec/leanvec.h" +#include "svs/leanvec/leanvec.h" #endif // USE_PROPRIETARY diff --git a/include/svs/quantization/lvq/lvq_concept.h b/include/svs/quantization/lvq/lvq_concept.h index cfe29632..aac4f021 100644 --- a/include/svs/quantization/lvq/lvq_concept.h +++ b/include/svs/quantization/lvq/lvq_concept.h @@ -22,7 +22,7 @@ #else // USE_PROPRIETARY -#include "../../../../../include/svs/quantization/lvq/lvq.h" +#include "svs/quantization/lvq/lvq.h" #endif // USE_PROPRIETARY From cd5cde046142103c97fb11ed6c8d2cf70e171a6c Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 5 May 2025 10:48:17 -0700 Subject: [PATCH 21/23] fallback example non-simple --- examples/cpp/CMakeLists.txt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 1bd2c438..998f2138 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -37,7 +37,6 @@ endfunction() create_simple_example(saveload test_saveload saveload.cpp) create_simple_example(types test_types types.cpp) create_simple_example(vamana_iterator test_vamana_iterator vamana_iterator.cpp) -create_simple_example(fallback test_fallback fallback.cpp) ## More complicated examples involving more extensive setup. @@ -75,6 +74,19 @@ add_test( groundtruth_euclidean.ivecs ) +# The fallback executable. +add_executable(fallback fallback.cpp) +target_include_directories(fallback PRIVATE ${CMAKE_CURRENT_LIST_DIR}) +target_link_libraries(fallback ${SVS_LIB} svs_compile_options svs_native_options) +add_test( + NAME test_fallback + COMMAND + fallback + data_f32.fvecs + queries_f32.fvecs + groundtruth_euclidean.ivecs +) + ##### ##### Dispatcher From 9e15d854a7fe324131f4dd5341c4d28435466e7c Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 5 May 2025 15:12:13 -0700 Subject: [PATCH 22/23] clean up constexpr std::string issue with clang --- include/svs/quantization/lvq/lvq_fallback.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/svs/quantization/lvq/lvq_fallback.h b/include/svs/quantization/lvq/lvq_fallback.h index 1ca8566d..45fdcdb7 100644 --- a/include/svs/quantization/lvq/lvq_fallback.h +++ b/include/svs/quantization/lvq/lvq_fallback.h @@ -31,7 +31,7 @@ struct Sequential { }; template struct Turbo { - static constexpr std::string name() { + static std::string name() { return fmt::format("turbo<{}x{}>", Lanes, ElementsPerLane); } }; From a57b27489344adf53b312f2d735616ab8b8344f1 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 5 May 2025 16:05:35 -0700 Subject: [PATCH 23/23] move eve to private --- include/svs/quantization/lvq/lvq_common.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/svs/quantization/lvq/lvq_common.h b/include/svs/quantization/lvq/lvq_common.h index 62a66f54..7968b81d 100644 --- a/include/svs/quantization/lvq/lvq_common.h +++ b/include/svs/quantization/lvq/lvq_common.h @@ -17,7 +17,6 @@ #pragma once // svs -#include "eve/algo.hpp" #include "svs/core/data.h" #include "svs/core/distance.h" #include "svs/core/kmeans.h"