Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix some issues in API #2982

Merged
merged 3 commits into from
Oct 23, 2024
+206 −106
Merged

Fix some issues in API #2982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d3514b
Fix some issues in API
Oct 22, 2024
05701f5
Doc
Oct 23, 2024
b89fe14
Chat
Oct 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 32 additions & 27 deletions api/apps/sdk/chat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,21 @@
from api.db import StatusEnum
from api.db.services.dialog_service import DialogService
from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.llm_service import TenantLLMService
from api.db.services.llm_service import TenantLLMService
from api.db.services.user_service import TenantService
from api.utils import get_uuid
from api.utils.api_utils import get_error_data_result, token_required
from api.utils.api_utils import get_result



@manager.route('/chat', methods=['POST'])
@token_required
def create(tenant_id):
req=request.json
ids= req.get("knowledgebases")
ids= req.get("datasets")
if not ids:
return get_error_data_result(retmsg="`knowledgebases` is required")
return get_error_data_result(retmsg="`datasets` is required")
for kb_id in ids:
kbs = KnowledgebaseService.query(id=kb_id,tenant_id=tenant_id)
if not kbs:
Expand All @@ -45,6 +46,8 @@ def create(tenant_id):
if llm:
if "model_name" in llm:
req["llm_id"] = llm.pop("model_name")
if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
req["llm_setting"] = req.pop("llm")
e, tenant = TenantService.get_by_id(tenant_id)
if not e:
Expand Down Expand Up @@ -73,10 +76,10 @@ def create(tenant_id):
req["top_n"] = req.get("top_n", 6)
req["top_k"] = req.get("top_k", 1024)
req["rerank_id"] = req.get("rerank_id", "")
if req.get("llm_id"):
if not TenantLLMService.query(llm_name=req["llm_id"]):
return get_error_data_result(retmsg="the model_name does not exist.")
else:
if req.get("rerank_id"):
if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_id"),model_type="rerank"):
return get_error_data_result(f"`rerank_model` {req.get('rerank_id')} doesn't exist")
if not req.get("llm_id"):
req["llm_id"] = tenant.llm_id
if not req.get("name"):
return get_error_data_result(retmsg="`name` is required.")
Expand Down Expand Up @@ -135,7 +138,7 @@ def create(tenant_id):
res["llm"] = res.pop("llm_setting")
res["llm"]["model_name"] = res.pop("llm_id")
del res["kb_ids"]
res["knowledgebases"] = req["knowledgebases"]
res["datasets"] = req["datasets"]
res["avatar"] = res.pop("icon")
return get_result(data=res)

Expand All @@ -145,27 +148,32 @@ def update(tenant_id,chat_id):
if not DialogService.query(tenant_id=tenant_id, id=chat_id, status=StatusEnum.VALID.value):
return get_error_data_result(retmsg='You do not own the chat')
req =request.json
if "knowledgebases" in req:
if not req.get("knowledgebases"):
return get_error_data_result(retmsg="`knowledgebases` can't be empty value")
kb_list = []
for kb in req.get("knowledgebases"):
if not kb["id"]:
return get_error_data_result(retmsg="knowledgebase needs id")
if not KnowledgebaseService.query(id=kb["id"], tenant_id=tenant_id):
return get_error_data_result(retmsg="you do not own the knowledgebase")
# if not DocumentService.query(kb_id=kb["id"]):
# return get_error_data_result(retmsg="There is a invalid knowledgebase")
kb_list.append(kb["id"])
req["kb_ids"] = kb_list
ids = req.get("datasets")
if "datasets" in req:
if not ids:
return get_error_data_result("`datasets` can't be empty")
if ids:
for kb_id in ids:
kbs = KnowledgebaseService.query(id=kb_id, tenant_id=tenant_id)
if not kbs:
return get_error_data_result(f"You don't own the dataset {kb_id}")
kb = kbs[0]
if kb.chunk_num == 0:
return get_error_data_result(f"The dataset {kb_id} doesn't own parsed file")
req["kb_ids"] = ids
llm = req.get("llm")
if llm:
if "model_name" in llm:
req["llm_id"] = llm.pop("model_name")
if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req["llm_id"],model_type="chat"):
return get_error_data_result(f"`model_name` {req.get('llm_id')} doesn't exist")
req["llm_setting"] = req.pop("llm")
e, tenant = TenantService.get_by_id(tenant_id)
if not e:
return get_error_data_result(retmsg="Tenant not found!")
if req.get("rerank_model"):
if not TenantLLMService.query(tenant_id=tenant_id,llm_name=req.get("rerank_model"),model_type="rerank"):
return get_error_data_result(f"`rerank_model` {req.get('rerank_model')} doesn't exist")
# prompt
prompt = req.get("prompt")
key_mapping = {"parameters": "variables",
Expand All @@ -185,9 +193,6 @@ def update(tenant_id,chat_id):
req["prompt_config"] = req.pop("prompt")
e, res = DialogService.get_by_id(chat_id)
res = res.to_json()
if "llm_id" in req:
if not TenantLLMService.query(llm_name=req["llm_id"]):
return get_error_data_result(retmsg="The `model_name` does not exist.")
if "name" in req:
if not req.get("name"):
return get_error_data_result(retmsg="`name` is not empty.")
Expand All @@ -209,8 +214,8 @@ def update(tenant_id,chat_id):
# avatar
if "avatar" in req:
req["icon"] = req.pop("avatar")
if "knowledgebases" in req:
req.pop("knowledgebases")
if "datasets" in req:
req.pop("datasets")
if not DialogService.update_by_id(chat_id, req):
return get_error_data_result(retmsg="Chat not found!")
return get_result()
Expand Down Expand Up @@ -279,7 +284,7 @@ def list_chat(tenant_id):
return get_error_data_result(retmsg=f"Don't exist the kb {kb_id}")
kb_list.append(kb[0].to_json())
del res["kb_ids"]
res["knowledgebases"] = kb_list
res["datasets"] = kb_list
res["avatar"] = res.pop("icon")
list_assts.append(res)
return get_result(data=list_assts)
42 changes: 28 additions & 14 deletions api/apps/sdk/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@
#

from flask import request

from api.db import StatusEnum, FileSource
from api.db.db_models import File
from api.db.services.document_service import DocumentService
from api.db.services.file2document_service import File2DocumentService
from api.db.services.file_service import FileService
from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.llm_service import TenantLLMService
from api.db.services.user_service import TenantService
from api.settings import RetCode
from api.utils import get_uuid
from api.utils.api_utils import get_result, token_required, get_error_data_result, valid
from api.utils.api_utils import get_result, token_required, get_error_data_result, valid,get_parser_config


@manager.route('/dataset', methods=['POST'])
Expand All @@ -36,15 +36,17 @@ def create(tenant_id):
permission = req.get("permission")
language = req.get("language")
chunk_method = req.get("chunk_method")
valid_permission = ("me", "team")
valid_language =("Chinese", "English")
valid_chunk_method = ("naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email")
parser_config = req.get("parser_config")
valid_permission = {"me", "team"}
valid_language ={"Chinese", "English"}
valid_chunk_method = {"naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email"}
check_validation=valid(permission,valid_permission,language,valid_language,chunk_method,valid_chunk_method)
if check_validation:
return check_validation
if "tenant_id" in req or "embedding_model" in req:
req["parser_config"]=get_parser_config(chunk_method,parser_config)
if "tenant_id" in req:
return get_error_data_result(
retmsg="`tenant_id` or `embedding_model` must not be provided")
retmsg="`tenant_id` must not be provided")
chunk_count=req.get("chunk_count")
document_count=req.get("document_count")
if chunk_count or document_count:
Expand All @@ -59,9 +61,13 @@ def create(tenant_id):
retmsg="`name` is not empty string!")
if KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id, status=StatusEnum.VALID.value):
return get_error_data_result(
retmsg="Duplicated knowledgebase name in creating dataset.")
retmsg="Duplicated dataset name in creating dataset.")
req["tenant_id"] = req['created_by'] = tenant_id
req['embedding_model'] = t.embd_id
if not req.get("embedding_model"):
req['embedding_model'] = t.embd_id
else:
if not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model")):
return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
key_mapping = {
"chunk_num": "chunk_count",
"doc_num": "document_count",
Expand Down Expand Up @@ -116,10 +122,12 @@ def update(tenant_id,dataset_id):
permission = req.get("permission")
language = req.get("language")
chunk_method = req.get("chunk_method")
valid_permission = ("me", "team")
valid_language =("Chinese", "English")
valid_chunk_method = ("naive","manual","qa","table","paper","book","laws","presentation","picture","one","knowledge_graph","email")
check_validation=valid(permission,valid_permission,language,valid_language,chunk_method,valid_chunk_method)
parser_config = req.get("parser_config")
valid_permission = {"me", "team"}
valid_language = {"Chinese", "English"}
valid_chunk_method = {"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one",
"knowledge_graph", "email"}
check_validation = valid(permission, valid_permission, language, valid_language, chunk_method, valid_chunk_method)
if check_validation:
return check_validation
if "tenant_id" in req:
Expand All @@ -142,18 +150,24 @@ def update(tenant_id,dataset_id):
return get_error_data_result(
retmsg="If `chunk_count` is not 0, `chunk_method` is not changeable.")
req['parser_id'] = req.pop('chunk_method')
if req['parser_id'] != kb.parser_id:
req["parser_config"] = get_parser_config(chunk_method, parser_config)
if "embedding_model" in req:
if kb.chunk_num != 0 and req['embedding_model'] != kb.embd_id:
return get_error_data_result(
retmsg="If `chunk_count` is not 0, `embedding_method` is not changeable.")
if not req.get("embedding_model"):
return get_error_data_result("`embedding_model` can't be empty")
if not TenantLLMService.query(tenant_id=tenant_id,model_type="embedding", llm_name=req.get("embedding_model")):
return get_error_data_result(f"`embedding_model` {req.get('embedding_model')} doesn't exist")
req['embd_id'] = req.pop('embedding_model')
if "name" in req:
req["name"] = req["name"].strip()
if req["name"].lower() != kb.name.lower() \
and len(KnowledgebaseService.query(name=req["name"], tenant_id=tenant_id,
status=StatusEnum.VALID.value)) > 0:
return get_error_data_result(
retmsg="Duplicated knowledgebase name in updating dataset.")
retmsg="Duplicated dataset name in updating dataset.")
if not KnowledgebaseService.update_by_id(kb.id, req):
return get_error_data_result(retmsg="Update dataset error.(Database error)")
return get_result(retcode=RetCode.SUCCESS)
Expand Down
Loading