Skip to content

Commit

Permalink
fix: some comment
Browse files Browse the repository at this point in the history
Signed-off-by: chyezh <[email protected]>
  • Loading branch information
chyezh committed Jan 13, 2025
1 parent 44007aa commit fd22729
Show file tree
Hide file tree
Showing 31 changed files with 135 additions and 152 deletions.
11 changes: 3 additions & 8 deletions internal/core/src/index/BitmapIndex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -286,20 +286,15 @@ BitmapIndex<T>::Serialize(const Config& config) {
}

template <typename T>
CreateIndexResultPtr
IndexStatsPtr
BitmapIndex<T>::Upload(const Config& config) {
auto binary_set = Serialize(config);

file_manager_->AddFile(binary_set);

auto remote_path_to_size = file_manager_->GetRemotePathsToFileSize();
std::vector<SerializedIndexFileInfo> index_files;
index_files.reserve(remote_path_to_size.size());
for (auto& file : remote_path_to_size) {
index_files.emplace_back(file.first, file.second);
}
return CreateIndexResult::New(file_manager_->GetAddedTotalMemSize(),
std::move(index_files));
return IndexStats::NewFromSizeMap(file_manager_->GetAddedTotalMemSize(),
remote_path_to_size);
}

template <typename T>
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/BitmapIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ class BitmapIndex : public ScalarIndex<T> {
return Count();
}

CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config = {}) override;

const bool
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/HybridScalarIndex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ HybridScalarIndex<T>::SerializeIndexType() {
}

template <typename T>
CreateIndexResultPtr
IndexStatsPtr
HybridScalarIndex<T>::Upload(const Config& config) {
auto internal_index = GetInternalIndex();
auto index_ret = internal_index->Upload(config);
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/HybridScalarIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ class HybridScalarIndex : public ScalarIndex<T> {
return internal_index_->HasRawData();
}

CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config = {}) override;

private:
Expand Down
4 changes: 2 additions & 2 deletions internal/core/src/index/Index.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
#include "common/Tracer.h"
#include "common/Types.h"
#include "index/Meta.h"
#include "index/CreateIndexResult.h"
#include "index/IndexStats.h"

namespace milvus::index {

Expand Down Expand Up @@ -58,7 +58,7 @@ class IndexBase {
virtual int64_t
Count() = 0;

virtual CreateIndexResultPtr
virtual IndexStatsPtr
Upload(const Config& config = {}) = 0;

virtual const bool
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,45 +9,54 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License

#include "index/CreateIndexResult.h"
#include "index/IndexStats.h"

namespace milvus::index {

CreateIndexResultPtr
CreateIndexResult::New(
int64_t mem_size,
std::vector<SerializedIndexFileInfo>&& serialized_index_infos) {
return std::unique_ptr<CreateIndexResult>(
new CreateIndexResult(mem_size, std::move(serialized_index_infos)));
IndexStatsPtr
IndexStats::NewFromSizeMap(int64_t mem_size,
std::map<std::string, int64_t>& index_size_map) {
std::vector<SerializedIndexFileInfo> serialized_index_infos;
serialized_index_infos.reserve(index_size_map.size());
for (auto& file : index_size_map) {
serialized_index_infos.emplace_back(file.first, file.second);
}
return IndexStats::New(mem_size, std::move(serialized_index_infos));
}

IndexStatsPtr
IndexStats::New(int64_t mem_size,
std::vector<SerializedIndexFileInfo>&& serialized_index_infos) {
return std::unique_ptr<IndexStats>(
new IndexStats(mem_size, std::move(serialized_index_infos)));
}

CreateIndexResult::CreateIndexResult(
IndexStats::IndexStats(
int64_t mem_size,
std::vector<SerializedIndexFileInfo>&& serialized_index_infos)
: mem_size_(mem_size), serialized_index_infos_(serialized_index_infos) {
}

void
CreateIndexResult::AppendSerializedIndexFileInfo(
SerializedIndexFileInfo&& info) {
IndexStats::AppendSerializedIndexFileInfo(SerializedIndexFileInfo&& info) {
serialized_index_infos_.push_back(std::move(info));
}

void
CreateIndexResult::SerializeAt(milvus::ProtoLayout* layout) {
milvus::proto::cgo::CreateIndexResult result;
IndexStats::SerializeAt(milvus::ProtoLayout* layout) {
milvus::proto::cgo::IndexStats result;
result.set_mem_size(mem_size_);
for (auto& info : serialized_index_infos_) {
auto serialized_info = result.add_serialized_index_infos();
serialized_info->set_file_name(info.file_name);
serialized_info->set_file_size(info.file_size);

Check warning on line 52 in internal/core/src/index/IndexStats.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/IndexStats.cpp#L46-L52

Added lines #L46 - L52 were not covered by tests
}
AssertInfo(layout->SerializeAndHoldProto(result),

Check warning on line 54 in internal/core/src/index/IndexStats.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/IndexStats.cpp#L54

Added line #L54 was not covered by tests
"marshal CreateIndexResult failed");
"marshal IndexStats failed");
}

Check warning on line 56 in internal/core/src/index/IndexStats.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/IndexStats.cpp#L56

Added line #L56 was not covered by tests

std::vector<std::string>
CreateIndexResult::GetIndexFiles() const {
IndexStats::GetIndexFiles() const {
std::vector<std::string> files;
for (auto& info : serialized_index_infos_) {
files.push_back(info.file_name);
Expand All @@ -56,12 +65,12 @@ CreateIndexResult::GetIndexFiles() const {
}

Check warning on line 65 in internal/core/src/index/IndexStats.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/IndexStats.cpp#L65

Added line #L65 was not covered by tests

int64_t
CreateIndexResult::GetMemSize() const {
IndexStats::GetMemSize() const {
return mem_size_;
}

int64_t
CreateIndexResult::GetSerializedSize() const {
IndexStats::GetSerializedSize() const {
int64_t size = 0;
for (auto& info : serialized_index_infos_) {
size += info.file_size;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,26 +28,30 @@ class SerializedIndexFileInfo {
int64_t file_size;
};

class CreateIndexResult;
class IndexStats;

using CreateIndexResultPtr = std::unique_ptr<CreateIndexResult>;
using IndexStatsPtr = std::unique_ptr<IndexStats>;

class CreateIndexResult {
class IndexStats {
public:
// Create a new CreateIndexResult instance.
static CreateIndexResultPtr
static IndexStatsPtr
NewFromSizeMap(int64_t mem_size,
std::map<std::string, int64_t>& index_size_map);

// Create a new IndexStats instance.
static IndexStatsPtr
New(int64_t mem_size,
std::vector<SerializedIndexFileInfo>&& serialized_index_infos);

CreateIndexResult(const CreateIndexResult&) = delete;
IndexStats(const IndexStats&) = delete;

CreateIndexResult(CreateIndexResult&&) = delete;
IndexStats(IndexStats&&) = delete;

CreateIndexResult&
operator=(const CreateIndexResult&) = delete;
IndexStats&
operator=(const IndexStats&) = delete;

CreateIndexResult&
operator=(CreateIndexResult&&) = delete;
IndexStats&
operator=(IndexStats&&) = delete;

// Append a new serialized index file info into the result.
void
Expand All @@ -67,9 +71,8 @@ class CreateIndexResult {
GetSerializedSize() const;

private:
CreateIndexResult(
int64_t mem_size,
std::vector<SerializedIndexFileInfo>&& serialized_index_infos);
IndexStats(int64_t mem_size,
std::vector<SerializedIndexFileInfo>&& serialized_index_infos);

int64_t mem_size_;
std::vector<SerializedIndexFileInfo> serialized_index_infos_;
Expand Down
9 changes: 4 additions & 5 deletions internal/core/src/index/InvertedIndexTantivy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ InvertedIndexTantivy<T>::Serialize(const Config& config) {
}

template <typename T>
CreateIndexResultPtr
IndexStatsPtr
InvertedIndexTantivy<T>::Upload(const Config& config) {
finish();

Expand Down Expand Up @@ -172,10 +172,9 @@ InvertedIndexTantivy<T>::Upload(const Config& config) {
for (auto& file : remote_mem_path_to_size) {
index_files.emplace_back(file.first, file.second);
}
return CreateIndexResult::New(
mem_file_manager_->GetAddedTotalMemSize() +
disk_file_manager_->GetAddedTotalFileSize(),
std::move(index_files));
return IndexStats::New(mem_file_manager_->GetAddedTotalMemSize() +
disk_file_manager_->GetAddedTotalFileSize(),
std::move(index_files));
}

template <typename T>
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/InvertedIndexTantivy.h
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class InvertedIndexTantivy : public ScalarIndex<T> {
BinarySet
Serialize(const Config& config) override;

CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config = {}) override;

/*
Expand Down
11 changes: 3 additions & 8 deletions internal/core/src/index/ScalarIndexSort.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -151,19 +151,14 @@ ScalarIndexSort<T>::Serialize(const Config& config) {
}

template <typename T>
CreateIndexResultPtr
IndexStatsPtr
ScalarIndexSort<T>::Upload(const Config& config) {
auto binary_set = Serialize(config);
file_manager_->AddFile(binary_set);

auto remote_paths_to_size = file_manager_->GetRemotePathsToFileSize();
std::vector<SerializedIndexFileInfo> index_files;
index_files.reserve(remote_paths_to_size.size());
for (auto& file : remote_paths_to_size) {
index_files.emplace_back(file.first, file.second);
}
return CreateIndexResult::New(file_manager_->GetAddedTotalMemSize(),
std::move(index_files));
return IndexStats::NewFromSizeMap(file_manager_->GetAddedTotalMemSize(),
remote_paths_to_size);

Check warning on line 161 in internal/core/src/index/ScalarIndexSort.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/ScalarIndexSort.cpp#L161

Added line #L161 was not covered by tests
}

template <typename T>
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/ScalarIndexSort.h
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ class ScalarIndexSort : public ScalarIndex<T> {
return (int64_t)data_.size();
}

CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config = {}) override;

const bool
Expand Down
11 changes: 3 additions & 8 deletions internal/core/src/index/StringIndexMarisa.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -174,19 +174,14 @@ StringIndexMarisa::Serialize(const Config& config) {
return res_set;
}

CreateIndexResultPtr
IndexStatsPtr
StringIndexMarisa::Upload(const Config& config) {
auto binary_set = Serialize(config);
file_manager_->AddFile(binary_set);

auto remote_paths_to_size = file_manager_->GetRemotePathsToFileSize();
std::vector<SerializedIndexFileInfo> index_files;
index_files.reserve(remote_paths_to_size.size());
for (auto& file : remote_paths_to_size) {
index_files.emplace_back(file.first, file.second);
}
return CreateIndexResult::New(file_manager_->GetAddedTotalMemSize(),
std::move(index_files));
return IndexStats::NewFromSizeMap(file_manager_->GetAddedTotalMemSize(),
remote_paths_to_size);

Check warning on line 184 in internal/core/src/index/StringIndexMarisa.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/StringIndexMarisa.cpp#L183-L184

Added lines #L183 - L184 were not covered by tests
}

void
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/StringIndexMarisa.h
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class StringIndexMarisa : public StringIndex {
std::optional<std::string>
Reverse_Lookup(size_t offset) const override;

CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config = {}) override;

const bool
Expand Down
10 changes: 5 additions & 5 deletions internal/core/src/index/TextMatchIndex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ TextMatchIndex::TextMatchIndex(const storage::FileManagerContext& ctx)
d_type_ = TantivyDataType::Text;
}

CreateIndexResultPtr
IndexStatsPtr
TextMatchIndex::Upload(const Config& config) {
finish();

Expand All @@ -104,6 +104,7 @@ TextMatchIndex::Upload(const Config& config) {
mem_file_manager_->AddFile(binary_set);
auto remote_mem_path_to_size =
mem_file_manager_->GetRemotePathsToFileSize();

std::vector<SerializedIndexFileInfo> index_files;
index_files.reserve(remote_paths_to_size.size() +

Check warning on line 109 in internal/core/src/index/TextMatchIndex.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/TextMatchIndex.cpp#L108-L109

Added lines #L108 - L109 were not covered by tests
remote_mem_path_to_size.size());
Expand All @@ -113,10 +114,9 @@ TextMatchIndex::Upload(const Config& config) {
for (auto& file : remote_mem_path_to_size) {
index_files.emplace_back(file.first, file.second);

Check warning on line 115 in internal/core/src/index/TextMatchIndex.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/TextMatchIndex.cpp#L115

Added line #L115 was not covered by tests
}
return CreateIndexResult::New(
mem_file_manager_->GetAddedTotalMemSize() +
disk_file_manager_->GetAddedTotalFileSize(),
std::move(index_files));
return IndexStats::New(mem_file_manager_->GetAddedTotalMemSize() +

Check warning on line 117 in internal/core/src/index/TextMatchIndex.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/TextMatchIndex.cpp#L117

Added line #L117 was not covered by tests
disk_file_manager_->GetAddedTotalFileSize(),
std::move(index_files));

Check warning on line 119 in internal/core/src/index/TextMatchIndex.cpp

View check run for this annotation

Codecov / codecov/patch

internal/core/src/index/TextMatchIndex.cpp#L119

Added line #L119 was not covered by tests
}

void
Expand Down
4 changes: 2 additions & 2 deletions internal/core/src/index/TextMatchIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#include <boost/filesystem.hpp>

#include "index/InvertedIndexTantivy.h"
#include "index/CreateIndexResult.h"
#include "index/IndexStats.h"

namespace milvus::index {

Expand All @@ -40,7 +40,7 @@ class TextMatchIndex : public InvertedIndexTantivy<std::string> {
explicit TextMatchIndex(const storage::FileManagerContext& ctx);

public:
CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config) override;

void
Expand Down
11 changes: 3 additions & 8 deletions internal/core/src/index/VectorDiskIndex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ VectorDiskAnnIndex<T>::Load(milvus::tracer::TraceContext ctx,
}

template <typename T>
CreateIndexResultPtr
IndexStatsPtr
VectorDiskAnnIndex<T>::Upload(const Config& config) {
BinarySet ret;
auto stat = index_.Serialize(ret);
Expand All @@ -124,13 +124,8 @@ VectorDiskAnnIndex<T>::Upload(const Config& config) {
"failed to serialize index, " + KnowhereStatusString(stat));
}
auto remote_paths_to_size = file_manager_->GetRemotePathsToFileSize();
std::vector<SerializedIndexFileInfo> index_files;
index_files.reserve(remote_paths_to_size.size());
for (auto& file : remote_paths_to_size) {
index_files.emplace_back(file.first, file.second);
}
return CreateIndexResult::New(file_manager_->GetAddedTotalFileSize(),
std::move(index_files));
return IndexStats::NewFromSizeMap(file_manager_->GetAddedTotalFileSize(),
remote_paths_to_size);
}

template <typename T>
Expand Down
2 changes: 1 addition & 1 deletion internal/core/src/index/VectorDiskIndex.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class VectorDiskAnnIndex : public VectorIndex {
return binary_set;
}

CreateIndexResultPtr
IndexStatsPtr
Upload(const Config& config = {}) override;

int64_t
Expand Down
Loading

0 comments on commit fd22729

Please sign in to comment.