Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a batch iterator for the Vamana Indexes #64

Merged
merged 2 commits into from
Dec 14, 2024
Merged

Add a batch iterator for the Vamana Indexes #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f5b3693
Move batch iterator code to public svs
ibhati Dec 13, 2024
210a4da
Merge branch 'main' into ib/batch-iterator
ibhati Dec 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions benchmark/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ set(SHARED_LIBRARY_FILES
src/vamana/search.cpp
src/vamana/build.cpp
src/vamana/test.cpp
src/vamana/iterator.cpp
# inverted
src/inverted/inverted.cpp
src/inverted/memory/common.cpp
Expand Down
3 changes: 1 addition & 2 deletions benchmark/include/svs-benchmark/benchmark.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,8 +222,7 @@ class ExecutableDispatcher {
// In general, index builds can take a long time and it may be beneficial to two things:
//
// (1) Regularly save checkpoints of results as they are generated so that if the
// application
// fails, we do not lose all of our data.
// application fails, we do not lose all of our data.
// (2) Provide results in as near real-time as we can so we can monitor currently running
// processes to determine as early as possible if something has gone wrong.
//
Expand Down
10 changes: 6 additions & 4 deletions benchmark/include/svs-benchmark/index_traits.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@ template <typename Index> struct IndexTraits; //{
//
///// Search Requirements
// template<svs::data::ImmutableMemoryDataset Queries>
// static auto search(index_type&, const Queries&, size_t num_neighbors, const
// config_type&);
// static auto search(
// index_type&, const Queries&, size_t num_neighbors, const config_type&
// );
//
// template<svs::data::ImmutableMemoryDataset Queries, typename GroundTruth>
// static config_type calibrate(
Expand All @@ -45,8 +46,9 @@ template <typename Index> struct IndexTraits; //{
///// Dynamic Build Requirements
//
// template<svs::data::ImmutableMemoryDataset Points>
// static void add_points(index_type&, const Points& points, const std::vector<size_t>&
// ids);
// static void add_points(
// index_type&, const Points& points, const std::vector<size_t>& ids
// );
//
// static void delete_points(index_type&, const std::vector<size_t>& ids);
//
Expand Down
20 changes: 17 additions & 3 deletions benchmark/include/svs-benchmark/search.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,25 @@ template <typename T, std::integral I> struct QuerySet {
}

QuerySet(
const dataset_type& queries,
const groundtruth_type& groundtruth,
const svs::data::SimpleData<T>& queries,
const svs::data::SimpleData<I>& groundtruth,
size_t number_of_training_elements
)
: QuerySet{queries.cview(), groundtruth.cview(), number_of_training_elements} {}

QuerySet(
const svs::data::ConstSimpleDataView<T>& queries,
const svs::data::ConstSimpleDataView<I>& groundtruth,
size_t number_of_training_elements
) {
assert(queries.size() == groundtruth.size());
size_t nqueries = queries.size();
if (nqueries != groundtruth.size()) {
throw ANNEXCEPTION(
"Query size ({}) and groundtruth size ({}) do not match!",
nqueries,
groundtruth.size()
);
}
if (number_of_training_elements >= queries.size()) {
throw ANNEXCEPTION(
"Number of elements to pull out into the training ({}) is greater than the "
Expand Down
Loading
Loading