introduce new eval tool: eval_performance (antgroup#234)

- the same as test_performance but support more
- add Monitor, EvalCase to extends the eval
- use argparse for command line

Signed-off-by: LHT129 <[email protected]>

tools/CMakeLists.txt

@@ -1,6 +1,8 @@
 
-add_library (eval_dataset OBJECT eval_dataset.cpp eval_dataset.h)
-add_dependencies (eval_dataset hdf5 spdlog)
+add_subdirectory (eval)
 
 add_executable (test_performance test_performance.cpp)
-target_link_libraries (test_performance PRIVATE vsag eval_dataset simd libhdf5_cpp.a libhdf5.a z)
+target_link_libraries (test_performance PRIVATE vsag eval_obj simd libhdf5_cpp.a libhdf5.a z)
+
+add_executable (eval_performance eval_performance.cpp)
+target_link_libraries (eval_performance PRIVATE vsag eval_obj argparse::argparse simd libhdf5_cpp.a libhdf5.a z)

tools/eval/CMakeLists.txt

@@ -0,0 +1,14 @@
+
+set (EVAL_SRC
+        eval_case.cpp
+        search_eval_case.cpp
+        build_eval_case.cpp
+        eval_dataset.cpp
+        eval_config.cpp
+        monitor/monitor.cpp
+        monitor/latency_monitor.cpp
+        monitor/recall_monitor.cpp
+        monitor/memory_peak_monitor.cpp
+        monitor/duration_monitor.cpp)
+add_library (eval_obj OBJECT ${EVAL_SRC})
+add_dependencies (eval_obj hdf5 spdlog)

tools/eval/build_eval_case.cpp

@@ -0,0 +1,95 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "build_eval_case.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "monitor/duration_monitor.h"
+#include "monitor/memory_peak_monitor.h"
+
+namespace vsag::eval {
+
+BuildEvalCase::BuildEvalCase(const std::string& dataset_path,
+                             const std::string& index_path,
+                             vsag::IndexPtr index,
+                             EvalConfig config)
+    : EvalCase(dataset_path, index_path, index), config_(std::move(config)) {
+    this->init_monitors();
+}
+
+void
+BuildEvalCase::init_monitors() {
+    if (config_.enable_memory) {
+        auto memory_peak_monitor = std::make_shared<MemoryPeakMonitor>();
+        this->monitors_.emplace_back(std::move(memory_peak_monitor));
+    }
+    if (config_.enable_tps) {
+        auto duration_monitor = std::make_shared<DurationMonitor>();
+        this->monitors_.emplace_back(std::move(duration_monitor));
+    }
+}
+
+void
+BuildEvalCase::Run() {
+    this->do_build();
+    this->serialize();
+    auto result = this->process_result();
+    PrintResult(result);
+}
+void
+BuildEvalCase::do_build() {
+    auto base = vsag::Dataset::Make();
+    int64_t total_base = this->dataset_ptr_->GetNumberOfBase();
+    std::vector<int64_t> ids(total_base);
+    std::iota(ids.begin(), ids.end(), 0);
+    base->NumElements(total_base)->Dim(this->dataset_ptr_->GetDim())->Ids(ids.data())->Owner(false);
+    if (this->dataset_ptr_->GetTrainDataType() == vsag::DATATYPE_FLOAT32) {
+        base->Float32Vectors((const float*)this->dataset_ptr_->GetTrain());
+    } else if (this->dataset_ptr_->GetTrainDataType() == vsag::DATATYPE_INT8) {
+        base->Int8Vectors((const int8_t*)this->dataset_ptr_->GetTrain());
+    }
+    for (auto& monitor : monitors_) {
+        monitor->Start();
+    }
+    auto build_index = index_->Build(base);
+    for (auto& monitor : monitors_) {
+        monitor->Record();
+        monitor->Stop();
+    }
+}
+void
+BuildEvalCase::serialize() {
+    std::ofstream outfile(this->index_path_, std::ios::binary);
+    this->index_->Serialize(outfile);
+}
+
+EvalCase::JsonType
+BuildEvalCase::process_result() {
+    JsonType result;
+    JsonType eval_result;
+    for (auto& monitor : this->monitors_) {
+        const auto& one_result = monitor->GetResult();
+        EvalCase::MergeJsonType(one_result, eval_result);
+    }
+    result = eval_result;
+    result["tps"] = double(this->dataset_ptr_->GetNumberOfBase()) / double(result["duration(s)"]);
+    EvalCase::MergeJsonType(this->basic_info_, result);
+    result["index_info"] = JsonType::parse(config_.build_param);
+    return result;
+}
+
+}  // namespace vsag::eval

tools/eval/build_eval_case.h

@@ -0,0 +1,52 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "eval_case.h"
+#include "monitor/monitor.h"
+namespace vsag::eval {
+
+class BuildEvalCase : public EvalCase {
+public:
+    BuildEvalCase(const std::string& dataset_path,
+                  const std::string& index_path,
+                  vsag::IndexPtr index,
+                  EvalConfig config);
+
+    ~BuildEvalCase() override = default;
+
+    void
+    Run() override;
+
+private:
+    void
+    init_monitors();
+
+    void
+    do_build();
+
+    void
+    serialize();
+
+    JsonType
+    process_result();
+
+private:
+    std::vector<MonitorPtr> monitors_{};
+
+    EvalConfig config_;
+};
+}  // namespace vsag::eval

tools/eval/eval_case.cpp

@@ -0,0 +1,53 @@
+
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "eval_case.h"
+
+#include <utility>
+
+#include "build_eval_case.h"
+#include "search_eval_case.h"
+#include "vsag/factory.h"
+#include "vsag/options.h"
+
+namespace vsag::eval {
+
+EvalCase::EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index)
+    : dataset_path_(std::move(dataset_path)), index_path_(std::move(index_path)), index_(index) {
+    this->dataset_ptr_ = EvalDataset::Load(dataset_path_);
+    this->logger_ = vsag::Options::Instance().logger();
+    this->basic_info_ = this->dataset_ptr_->GetInfo();
+}
+
+EvalCasePtr
+EvalCase::MakeInstance(const EvalConfig& config) {
+    auto dataset_path = config.dataset_path;
+    auto index_path = config.index_path;
+    auto index_name = config.index_name;
+    auto create_params = config.build_param;
+
+    auto index = vsag::Factory::CreateIndex(index_name, create_params);
+
+    auto type = config.action_type;
+    if (type == "build") {
+        return std::make_shared<BuildEvalCase>(dataset_path, index_path, index.value(), config);
+    } else if (type == "search") {
+        return std::make_shared<SearchEvalCase>(dataset_path, index_path, index.value(), config);
+    } else {
+        return nullptr;
+    }
+}
+}  // namespace vsag::eval

tools/eval/eval_case.h

@@ -0,0 +1,72 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "eval_config.h"
+#include "eval_dataset.h"
+#include "nlohmann/json.hpp"
+#include "vsag/index.h"
+#include "vsag/logger.h"
+
+namespace vsag::eval {
+
+class EvalCase;
+using EvalCasePtr = std::shared_ptr<EvalCase>;
+
+class EvalCase {
+public:
+    using JsonType = nlohmann::json;
+
+public:
+    static EvalCasePtr
+    MakeInstance(const EvalConfig& parser);
+
+    static void
+    MergeJsonType(const JsonType& input, JsonType& output) {
+        for (auto& [key, value] : input.items()) {
+            output[key] = value;
+        }
+    }
+
+    static void
+    PrintResult(const JsonType& result) {
+        std::cout << result.dump(4) << std::endl;
+    }
+
+public:
+    explicit EvalCase(std::string dataset_path, std::string index_path, vsag::IndexPtr index);
+
+    virtual ~EvalCase() = default;
+
+    virtual void
+    Run() = 0;
+
+    using Logger = vsag::Logger*;
+
+protected:
+    const std::string dataset_path_{};
+    const std::string index_path_{};
+
+    EvalDatasetPtr dataset_ptr_{nullptr};
+
+    vsag::IndexPtr index_{nullptr};
+
+    Logger logger_{nullptr};
+
+    JsonType basic_info_{};
+};
+
+}  // namespace vsag::eval

tools/eval/eval_config.cpp

@@ -0,0 +1,58 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "eval_config.h"
+
+namespace vsag::eval {
+EvalConfig
+EvalConfig::Load(argparse::ArgumentParser& parser) {
+    EvalConfig config;
+    config.dataset_path = parser.get("--datapath");
+    config.action_type = parser.get("--type");
+    config.build_param = parser.get("--create_params");
+    config.index_name = parser.get("--index_name");
+    config.index_path = parser.get("--index_path");
+
+    config.search_param = parser.get("--search_params");
+    config.search_mode = parser.get("--search_mode");
+
+    config.top_k = parser.get<int>("--topk");
+    config.radius = parser.get<float>("--range");
+
+    if (parser.get<bool>("--disable_recall")) {
+        config.enable_recall = false;
+    }
+    if (parser.get<bool>("--disable_percent_recall")) {
+        config.enable_percent_recall = false;
+    }
+    if (parser.get<bool>("--disable_memory")) {
+        config.enable_memory = false;
+    }
+    if (parser.get<bool>("--disable_latency")) {
+        config.enable_latency = false;
+    }
+    if (parser.get<bool>("--disable_qps")) {
+        config.enable_qps = false;
+    }
+    if (parser.get<bool>("--disable_tps")) {
+        config.enable_tps = false;
+    }
+    if (parser.get<bool>("--disable_percent_latency")) {
+        config.enable_percent_latency = false;
+    }
+
+    return config;
+}
+}  // namespace vsag::eval