Skip to content

Commit

Permalink
Update test_index_check (#817)
Browse files Browse the repository at this point in the history 
Signed-off-by: Cai Yudong <[email protected]>
  • Loading branch information
@cydrain
cydrain authored Sep 6, 2024
1 parent 60820e1 commit 3f82179
Show file tree
Hide file tree
Showing 3 changed files with 269 additions and 74 deletions.
26 changes: 13 additions & 13 deletions include/knowhere/comp/knowhere_check.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ IndexTypeAndDataTypeCheck(const std::string& index_name, VecType data_type) {
}

static bool
SuppportMmapIndexTypeCheck(const std::string& index_name) {
SupportMmapIndexTypeCheck(const std::string& index_name) {
auto& mmap_index_table = std::get<1>(IndexFactory::StaticIndexTableInstance());
if (mmap_index_table.find(index_name) != mmap_index_table.end()) {
return true;
Expand All @@ -44,18 +44,18 @@ SuppportMmapIndexTypeCheck(const std::string& index_name) {
}

inline bool
CheckBooleanInJson(const knowhere::Json& json, std::string key, bool target) {
CheckBooleanInJson(const knowhere::Json& json, std::string key) {
if (json.find(key) == json.end()) {
return false;
return true;
}
if (json[key].is_boolean()) {
return json[key] == target;
return json[key];
}
if (json[key].is_string()) {
if (target) {
return json[key] == "true";
if (json[key] == "true") {
return true;
} else {
return json[key] == "false";
return false;
}
}
return false;
Expand All @@ -74,16 +74,16 @@ IndexHasRawData(const knowhere::IndexType& indexType, const knowhere::MetricType
static std::set<knowhere::IndexType> no_raw_data_index_set = {
IndexEnum::INDEX_FAISS_IVFPQ, IndexEnum::INDEX_FAISS_IVFSQ8, IndexEnum::INDEX_HNSW_SQ8,
IndexEnum::INDEX_FAISS_GPU_IDMAP, IndexEnum::INDEX_FAISS_GPU_IVFFLAT, IndexEnum::INDEX_FAISS_GPU_IVFSQ8,
IndexEnum::INDEX_FAISS_GPU_IVFPQ, IndexEnum::INDEX_RAFT_CAGRA, IndexEnum::INDEX_GPU_CAGRA,
IndexEnum::INDEX_RAFT_IVFFLAT, IndexEnum::INDEX_GPU_IVFFLAT, IndexEnum::INDEX_RAFT_IVFPQ,
IndexEnum::INDEX_GPU_IVFPQ,
IndexEnum::INDEX_FAISS_GPU_IVFPQ, IndexEnum::INDEX_GPU_BRUTEFORCE, IndexEnum::INDEX_GPU_IVFFLAT,
IndexEnum::INDEX_GPU_IVFPQ, IndexEnum::INDEX_GPU_CAGRA, IndexEnum::INDEX_RAFT_BRUTEFORCE,
IndexEnum::INDEX_RAFT_IVFFLAT, IndexEnum::INDEX_RAFT_IVFPQ, IndexEnum::INDEX_RAFT_CAGRA,
};

static std::set<knowhere::IndexType> no_raw_data_index_alias_set = {"IVFPQ", "IVFSQ"};

static std::set<knowhere::IndexType> conditional_hold_raw_data_index_set = {
IndexEnum::INDEX_HNSW, IndexEnum::INDEX_DISKANN, IndexEnum::INDEX_FAISS_SCANN,
IndexEnum::INDEX_FAISS_IVFSQ_CC, IndexEnum::INDEX_FAISS_IDMAP,
IndexEnum::INDEX_FAISS_IDMAP, IndexEnum::INDEX_FAISS_SCANN, IndexEnum::INDEX_FAISS_IVFSQ_CC,
IndexEnum::INDEX_HNSW, IndexEnum::INDEX_DISKANN,
};

if (has_raw_data_index_set.find(indexType) != has_raw_data_index_set.end() ||
Expand All @@ -110,7 +110,7 @@ IndexHasRawData(const knowhere::IndexType& indexType, const knowhere::MetricType
return IsMetricType(metricType, metric::L2) || IsMetricType(metricType, metric::COSINE);
#endif
} else if (indexType == IndexEnum::INDEX_FAISS_SCANN) {
return !CheckBooleanInJson(params, indexparam::WITH_RAW_DATA, false);
return CheckBooleanInJson(params, indexparam::WITH_RAW_DATA);
// INDEX_FAISS_IVFSQ_CC is not online yet
} else if (indexType == IndexEnum::INDEX_FAISS_IVFSQ_CC) {
return params.find(indexparam::RAW_DATA_STORE_PREFIX) != params.end();
Expand Down
39 changes: 9 additions & 30 deletions include/knowhere/index/index_table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ static std::set<std::pair<std::string, VecType>> legal_knowhere_index = {
// binary ivf
{IndexEnum::INDEX_FAISS_BIN_IDMAP, VecType::VECTOR_BINARY},
{IndexEnum::INDEX_FAISS_BIN_IVFFLAT, VecType::VECTOR_BINARY},
// ivf

// faiss index
{IndexEnum::INDEX_FAISS_IDMAP, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_FAISS_IDMAP, VecType::VECTOR_FLOAT16},
{IndexEnum::INDEX_FAISS_IDMAP, VecType::VECTOR_BFLOAT16},
Expand Down Expand Up @@ -49,11 +50,13 @@ static std::set<std::pair<std::string, VecType>> legal_knowhere_index = {
{IndexEnum::INDEX_FAISS_IVFSQ_CC, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_FAISS_IVFSQ_CC, VecType::VECTOR_FLOAT16},
{IndexEnum::INDEX_FAISS_IVFSQ_CC, VecType::VECTOR_BFLOAT16},

// gpu index
{IndexEnum::INDEX_GPU_BRUTEFORCE, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_GPU_IVFFLAT, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_GPU_IVFPQ, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_GPU_CAGRA, VecType::VECTOR_FLOAT},

// hnsw
{IndexEnum::INDEX_HNSW, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_HNSW, VecType::VECTOR_FLOAT16},
Expand All @@ -66,6 +69,7 @@ static std::set<std::pair<std::string, VecType>> legal_knowhere_index = {
{IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_FLOAT16},
{IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_BFLOAT16},

// faiss hnsw
{IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_FLOAT16},
Expand All @@ -82,10 +86,12 @@ static std::set<std::pair<std::string, VecType>> legal_knowhere_index = {
{IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_FLOAT16},
{IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_BFLOAT16},

// diskann
{IndexEnum::INDEX_DISKANN, VecType::VECTOR_FLOAT},
{IndexEnum::INDEX_DISKANN, VecType::VECTOR_FLOAT16},
{IndexEnum::INDEX_DISKANN, VecType::VECTOR_BFLOAT16},

// sparse index
{IndexEnum::INDEX_SPARSE_INVERTED_INDEX, VecType::VECTOR_SPARSE_FLOAT},
{IndexEnum::INDEX_SPARSE_WAND, VecType::VECTOR_SPARSE_FLOAT},
Expand All @@ -95,51 +101,24 @@ static std::set<std::string> legal_support_mmap_knowhere_index = {
// binary ivf
IndexEnum::INDEX_FAISS_BIN_IDMAP,
IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
// ivf
IndexEnum::INDEX_FAISS_IDMAP,
IndexEnum::INDEX_FAISS_IDMAP,
IndexEnum::INDEX_FAISS_IDMAP,

// faiss index
IndexEnum::INDEX_FAISS_IDMAP,
IndexEnum::INDEX_FAISS_IVFFLAT,
IndexEnum::INDEX_FAISS_IVFFLAT,
IndexEnum::INDEX_FAISS_IVFFLAT,

IndexEnum::INDEX_FAISS_IVFPQ,
IndexEnum::INDEX_FAISS_IVFPQ,
IndexEnum::INDEX_FAISS_IVFPQ,

IndexEnum::INDEX_FAISS_SCANN,
IndexEnum::INDEX_FAISS_SCANN,
IndexEnum::INDEX_FAISS_SCANN,

IndexEnum::INDEX_FAISS_IVFSQ8,
IndexEnum::INDEX_FAISS_IVFSQ8,
IndexEnum::INDEX_FAISS_IVFSQ8,

IndexEnum::INDEX_FAISS_IVFSQ_CC,
IndexEnum::INDEX_FAISS_IVFSQ_CC,
IndexEnum::INDEX_FAISS_IVFSQ_CC,

// hnsw
IndexEnum::INDEX_HNSW,
IndexEnum::INDEX_HNSW,
IndexEnum::INDEX_HNSW,

IndexEnum::INDEX_HNSW_SQ8,
IndexEnum::INDEX_HNSW_SQ8,
IndexEnum::INDEX_HNSW_SQ8,

IndexEnum::INDEX_HNSW_SQ8_REFINE,
IndexEnum::INDEX_HNSW_SQ8_REFINE,
IndexEnum::INDEX_HNSW_SQ8_REFINE,

// faiss hnsw
IndexEnum::INDEX_FAISS_HNSW_FLAT,

IndexEnum::INDEX_FAISS_HNSW_SQ,

IndexEnum::INDEX_FAISS_HNSW_PQ,

IndexEnum::INDEX_FAISS_HNSW_PRQ,

// sparse index
Expand Down
Loading

0 comments on commit 3f82179

Please sign in to comment.