add knowledge config (#22)

1. 新增 Milvus 服务配置
2. 新增知识库对应embedding服务配置
3. 调整部署脚本，优化ci 配置
4. 更新bisheng-langchain 包，支持本地打包
5.  其他bug 修复

.github/workflows/ci.yml

@@ -36,6 +36,22 @@ jobs:
         run: |
           echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
 
+      # 构建 bisheng_langchain
+      - name: Set python version 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Build PyPi bisheng-langchain and push
+        id: pypi_build_bisheng_langchain
+        run: |
+          pip install Cython
+          pip install wheel
+          pip install twine
+          cd ./src/bisheng_langchain
+          python setup.py bdist_wheel
+          twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
+          
       # 构建 backend 并推送到 Docker hub
       - name: Build backend and push
         id: docker_build_backend
@@ -70,3 +86,4 @@ jobs:
           tags: |
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
+      

.github/workflows/release.yml

@@ -0,0 +1,89 @@
+
+name: release
+
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    branches:
+      - "release"
+
+env:
+  DOCKERHUB_REPO: dataelement/
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    #if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Get version
+        id: get_version
+        run: |
+          echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      # 构建 bisheng-langchain
+      - name: Set python version 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Build PyPi bisheng-langchain and push
+        id: pypi_build_bisheng_langchain
+        run: |
+          pip install Cython
+          pip install wheel
+          pip install twine
+          cd ./src/bisheng-langchain
+          python setup.py bdist_wheel
+          repo="http://110.16.193.170:50083/repository/pypi-hosted/"
+          twine upload --verbose -u ${{ secrets.NEXUS_USER }}  -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo  dist/*.whl
+
+      # 发布到
+      - name: Login
+        uses: docker/login-action@v2
+        with:
+          # GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
+          # DOCKERHUB_USERNAME 是 docker hub 账号名.
+          # DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      # 构建 backend 并推送到 Docker hub
+      - name: Build backend and push
+        id: docker_build_backend
+        uses: docker/build-push-action@v2
+        with:
+          # backend 的context目录
+          context: "./src/backend/"
+          # 是否 docker push
+          push: true
+          # docker build arg, 注入 APP_NAME/APP_VERSION
+          build-args: |
+            APP_NAME="bisheng-backend"
+            APP_VERSION="release"
+          # 生成两个 docker tag: ${APP_VERSION} 和 latest
+          tags: |
+            ${{ env.DOCKERHUB_REPO }}bisheng-backend:release
+      # 构建 Docker frontend 并推送到 Docker hub
+      - name: Build frontend and push
+        id: docker_build_frontend
+        uses: docker/build-push-action@v2
+        with:
+          # frontend 的context目录
+          context: "./src/frontend/"
+          # 是否 docker push
+          push: true
+          # docker build arg, 注入 APP_NAME/APP_VERSION
+          build-args: |
+            APP_NAME="bisheng-frontend"
+            APP_VERSION="release"
+          # 生成两个 docker tag: ${APP_VERSION} 和 latest
+          tags: |
+            ${{ env.DOCKERHUB_REPO }}bisheng-frontend:release
+
+     

.gitignore

@@ -131,6 +131,7 @@ build/
 ./third_party
 output/
 develop-eggs/
+config.dev.yaml
 dist/
 downloads/
 eggs/

docker/bisheng/config/config.yaml

@@ -1,17 +1,20 @@
 # 数据库配置
 database_url:
-  "mysql+pymysql://username:[email protected]:3306/bisheng"
+  "mysql+pymysql://root:1234@mysql:3306/bisheng"
 redis_url:
-  "192.168.106.116:6379"
+  "redis:6379"
 
 # 为知识库的embedding进行模型撇脂
-embedding_config:
-  text-embedding-ada-002:
-    base_url:
-      ""
-  multilingual-e5-large:
-    base_url:
-      ""
+knowledges:
+  embeddings:
+    text-embedding-ada-002:
+      openai_api_base: "https://api.openai.com/v1/"
+      openai_proxy: ""
+      openai_api_key: ""
+  vectorstores:
+    # Milvus 最低要求cpu 4C 8G 推荐4C 16G
+    Milvus: # 如果需要切换其他vectordb，确保其他服务已经启动，然后配置对应参数
+      connection_args = {'host': '110.16.193.170', 'port': '50032', 'user': '', 'password': '', 'secure': False}
 
 agents:
   ZeroShotAgent:

docker/docker-compose.yml

@@ -14,7 +14,7 @@ services:
   mysql:
     image: mysql:5.7.40
     environment:
-      - "MYSQL_ROOT_PASSWORD=E1SkG0PaDMEPTAxY"
+      - "MYSQL_ROOT_PASSWORD=1234"  # 数据库密码，建议修改，如果修改需要同步修改bisheng/congfig/config.yaml配置
       - "MYSQL_DATABASE=bisheng"
       - "TZ=Asia/Shanghai"
     ports:
@@ -24,18 +24,21 @@ services:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/mysql/data:/var/lib/mysql
 
   backend:
-    image: dataelement/bisheng-backend:0.0.1
+    image: dataelement/bisheng-backend:latest
     volumes:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/config/config.yaml:/app/bisheng/config.yaml
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/data/:/app/data/
     ports:
-      - "7861:7860"
+      - "7860:7860"
+    security_opt:
+      - seccomp:unconfined
+    command: bash -c "uvicorn bisheng.main:app --host 0.0.0.0 --port 7860 --workers 2" # --workers 表示使用几个进程，提高并发度
     restart: on-failure
 
   nginx:
-    image: dataelement/bisheng-frontend:0.0.1
+    image: dataelement/bisheng-frontend:latest
     ports:
-      - 3001:3001
+      - "3001:3001"
     volumes:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/nginx/nginx.conf:/etc/nginx/nginx.conf
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/nginx/conf.d:/etc/nginx/conf.d
-      - ${DOCKER_VOLUME_DIRECTORY:-.}/nginx/html:/usr/share/nginx/html
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/nginx/conf.d:/etc/nginx/conf.d

docker/nginx/conf.d/default.conf

@@ -33,6 +33,7 @@ server {
 		proxy_http_version 1.1;
 		proxy_set_header Upgrade $http_upgrade;
 		proxy_set_header Connection $connection_upgrade;
+		client_max_body_size 50m;
 	}
 
 }

src/backend/Dockerfile

@@ -14,9 +14,7 @@ COPY poetry.lock pyproject.toml ./
 COPY ./ ./
 
 RUN python -m pip install --upgrade pip && \
-    pip install gunicorn && \
-    pip install shapely==2.0.1 && \
-    pip install langchain_contrib==0.0.1
+    pip install shapely==2.0.1 
 
 # Install dependencies
 RUN poetry config virtualenvs.create false

src/backend/bisheng/api/v1/flows.py

@@ -5,8 +5,7 @@
 from bisheng.api.utils import build_flow_no_yield, remove_api_keys
 from bisheng.api.v1.schemas import FlowListCreate, FlowListRead
 from bisheng.database.base import get_session
-from bisheng.database.models.flow import (Flow, FlowCreate, FlowRead,
-                                          FlowReadWithStyle, FlowUpdate)
+from bisheng.database.models.flow import Flow, FlowCreate, FlowRead, FlowReadWithStyle, FlowUpdate
 from bisheng.database.models.template import Template
 from bisheng.database.models.user import User
 from bisheng.settings import settings
@@ -57,11 +56,12 @@ def read_flows(*,
             sql = sql.where(Flow.name.like(f'%{name}%'))
         if status:
             sql = sql.where(Flow.status == status)
+        # count = session.exec(sql.count())
+        # total = count.scalar()
 
         sql = sql.order_by(Flow.update_time.desc())
         if page_num and page_size:
             sql = sql.offset((page_num - 1) * page_size).limit(page_size)
-
         flows = session.exec(sql).all()
 
         res = [jsonable_encoder(flow) for flow in flows]
@@ -88,7 +88,11 @@ def read_flow(*, session: Session = Depends(get_session), flow_id: UUID):
 
 
 @router.patch('/{flow_id}', response_model=FlowRead, status_code=200)
-def update_flow(*, session: Session = Depends(get_session), flow_id: UUID, flow: FlowUpdate, Authorize: AuthJWT = Depends()):
+def update_flow(*,
+                session: Session = Depends(get_session),
+                flow_id: UUID,
+                flow: FlowUpdate,
+                Authorize: AuthJWT = Depends()):
     Authorize.jwt_required()
     payload = json.loads(Authorize.get_jwt_subject())
     """Update a flow."""
@@ -159,7 +163,10 @@ def create_flows(*, session: Session = Depends(get_session), flow_list: FlowList
 
 
 @router.post('/upload/', response_model=List[FlowRead], status_code=201)
-async def upload_file(*, session: Session = Depends(get_session), file: UploadFile = File(...), Authorize: AuthJWT = Depends()):
+async def upload_file(*,
+                      session: Session = Depends(get_session),
+                      file: UploadFile = File(...),
+                      Authorize: AuthJWT = Depends()):
     """Upload flows from a file."""
     contents = await file.read()
     data = json.loads(contents)

src/backend/bisheng/api/v1/knowledge.py

@@ -7,19 +7,21 @@
 from bisheng.api.v1.schemas import UploadFileResponse
 from bisheng.cache.utils import save_uploaded_file
 from bisheng.database.base import get_session
-from bisheng.database.models.knowledge import (Knowledge, KnowledgeCreate,
-                                               KnowledgeRead)
-from bisheng.database.models.knowledge_file import (KnowledgeFile,
-                                                    KnowledgeFileRead)
+from bisheng.database.models.knowledge import Knowledge, KnowledgeCreate, KnowledgeRead
+from bisheng.database.models.knowledge_file import KnowledgeFile, KnowledgeFileRead
 from bisheng.database.models.user import User
-from bisheng.interface.embeddings.custom import OpenAIProxyEmbedding
+from bisheng.interface.importing.utils import import_vectorstore
+from bisheng.interface.initialize.loading import instantiate_vectorstore
 from bisheng.settings import settings
 from bisheng.utils.logger import logger
+from bisheng_langchain.embeddings.host_embedding import HostEmbeddings
 from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
 from fastapi.encoders import jsonable_encoder
 from fastapi_jwt_auth import AuthJWT
+from langchain.embeddings.base import Embeddings
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.vectorstores import Milvus
+from langchain.vectorstores.base import VectorStore
 from sqlmodel import Session, select
 
 # build router
@@ -42,7 +44,7 @@ async def upload_file(*, file: UploadFile = File(...)):
 async def get_embedding():
     try:
         # 获取本地配置的名字
-        model_list = settings.embedding_config
+        model_list = settings.knowledges.get('embeddings')
         models = list(model_list.keys())
         return {'data': {'models': models}}
     except Exception as exc:
@@ -83,7 +85,11 @@ async def process_knowledge(*, session: Session = Depends(get_session), data: di
         file_paths.append(filepath)
         logger.info(f'fileName={file_name} col={collection_name}')
     asyncio.create_task(
-        addEmbedding(collection_name=collection_name, chunk_size=chunck_size, file_paths=file_paths, knowledge_files=files))
+        addEmbedding(collection_name=collection_name,
+                     model=knowledge.model,
+                     chunk_size=chunck_size,
+                     file_paths=file_paths,
+                     knowledge_files=files))
 
     knowledge.update_time = db_file.create_time
     session.add(knowledge)
@@ -92,14 +98,18 @@ async def process_knowledge(*, session: Session = Depends(get_session), data: di
 
 
 @router.post('/create', response_model=KnowledgeRead, status_code=201)
-def create_knowledge(*, session: Session = Depends(get_session), knowledge: KnowledgeCreate, Authorize: AuthJWT = Depends()):
+def create_knowledge(*,
+                     session: Session = Depends(get_session),
+                     knowledge: KnowledgeCreate,
+                     Authorize: AuthJWT = Depends()):
     Authorize.jwt_required()
     payload = json.loads(Authorize.get_jwt_subject())
     """创建知识库."""
     db_knowldge = Knowledge.from_orm(knowledge)
-    know = session.exec(select(Knowledge).where(Knowledge.name == knowledge.name)).all()
+    know = session.exec(
+        select(Knowledge).where(Knowledge.name == knowledge.name, knowledge.user_id == payload.get('user_id'))).all()
     if know:
-        raise HTTPException(status_code=500, detail='name 重复')
+        raise HTTPException(status_code=500, detail='知识库名称重复')
     if not db_knowldge.collection_name:
         # 默认collectionName
         db_knowldge.collection_name = f'col_{int(time.time())}_{str(uuid4())[:8]}'
@@ -175,32 +185,48 @@ def delete_knowledge_file(*, session: Session = Depends(get_session), file_id: i
         raise HTTPException(status_code=404, detail='没有权限执行操作')
     knowledge = session.get(Knowledge, knowledge_file.knowledge_id)
     # 处理vectordb
+
     collection_name = knowledge.collection_name
-    embeddings = OpenAIEmbeddings()
-    milvus = Milvus(embedding_function=embeddings, collection_name=collection_name, connection_args=connection_args)
-    pk = milvus.col.query(expr=f'file_id == {file_id}', output_fields=['pk'])
-    res = milvus.col.delete(f"pk in {[p['pk'] for p in pk]}")
+    embeddings = decide_embeddings(knowledge.model)
+    vectore_client = decide_vectorstores(collection_name, embeddings)
+    if isinstance(vectore_client, Milvus):
+        pk = vectore_client.col.query(expr=f'file_id == {file_id}', output_fields=['pk'])
+        res = vectore_client.col.delete(f"pk in {[p['pk'] for p in pk]}")
+
     logger.info(f'act=delete_vector file_id={file_id} res={res}')
     session.delete(knowledge_file)
     session.commit()
     return {'message': 'knowledge file deleted successfully'}
 
 
-connection_args = {'host': '192.168.106.116', 'port': '19530', 'user': '', 'password': '', 'secure': False}
+def decide_embeddings(model: str) -> Embeddings:
+    model_list = settings.knowledges.get('embeddings')
+    if model == 'text-embedding-ada-002':
+        return OpenAIEmbeddings(**model_list.get(model))
+    else:
+        return HostEmbeddings(**model_list.get(model))
+
+
+def decide_vectorstores(collection_name: str, embedding: Embeddings) -> VectorStore:
+    param = {'collection_name': collection_name, 'embedding': embedding}
+    vector_store = list(settings.knowledges.get('vectorstores').keys())[0]
+    vector_config = settings.knowledges.get('vectorstores').get(vector_store)
+    param.update(vector_config)
+    class_obj = import_vectorstore(vector_store)
+    return instantiate_vectorstore(class_object=class_obj, params=param)
+
 
+async def addEmbedding(collection_name, model: str, chunk_size: int, file_paths: List[str],
+                       knowledge_files: List[KnowledgeFile]):
 
-async def addEmbedding(collection_name, chunk_size: int, file_paths: List[str], knowledge_files: List[KnowledgeFile]):
-    embeddings = OpenAIProxyEmbedding()
+    embeddings = decide_embeddings(model)
+    vectore_client = decide_vectorstores(collection_name, embeddings)
     for index, path in enumerate(file_paths):
         knowledge_file = knowledge_files[index]
         try:
             texts, metadatas = _read_chunk_text(path, knowledge_file.file_name, chunk_size)
             [metadata.update({'file_id': knowledge_file.id}) for metadata in metadatas]
-            Milvus.from_texts(texts=texts,
-                              embedding=embeddings,
-                              metadatas=metadatas,
-                              collection_name=collection_name,
-                              connection_args=connection_args)
+            vectore_client.add_texts(texts=texts, metadatas=metadatas)
 
             session = next(get_session())
             db_file = session.get(KnowledgeFile, knowledge_file.id)