部署测试 (#17)

.github/workflows/ci.yml

@@ -36,6 +36,22 @@ jobs:
         run: |
           echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
 
+      # 构建 bisheng_langchain
+      - name: Set python version 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Build PyPi bisheng-langchain and push
+        id: pypi_build_bisheng_langchain
+        run: |
+          pip install Cython
+          pip install wheel
+          pip install twine
+          cd ./src/bisheng_langchain
+          python setup.py bdist_wheel
+          twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
+          
       # 构建 backend 并推送到 Docker hub
       - name: Build backend and push
         id: docker_build_backend
@@ -70,18 +86,4 @@ jobs:
           tags: |
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
-      # 构建 bisheng_langchain
-      - name: Set python version 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-
-      - name: Build PyPi bisheng-langchain and push
-        id: pypi_build_bisheng_langchain
-        run: |
-          pip install Cython
-          pip install wheel
-          pip install twine
-          cd ./src/bisheng_langchain
-          python setup.py bdist_wheel
-          twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
+      

.github/workflows/release.yml

@@ -27,6 +27,31 @@ jobs:
         run: |
           echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
 
+      # 构建 bisheng-langchain
+      - name: Set python version 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Build PyPi bisheng-langchain and push
+        id: pypi_build_bisheng_langchain
+        run: |
+          pip install Cython
+          pip install wheel
+          pip install twine
+          cd ./src/bisheng-langchain
+          python setup.py bdist_wheel
+          repo="http://110.16.193.170:50083/repository/pypi-hosted/"
+          twine upload --verbose -u ${{ secrets.NEXUS_USER }}  -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo  dist/*.whl
+          twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
+
+      # 发布到私服
+      - name: Login
+        uses: docker/login-action@v1
+        with:
+          registry: "http://110.16.193.170:50083/repository/docker-hosted/"
+          username: ${{ secrets.NEXUS_USER }}
+          password: ${{ secrets.NEXUS_PASSWORD }} 
       # 构建 backend 并推送到 Docker hub
       - name: Build backend and push
         id: docker_build_backend
@@ -35,7 +60,7 @@ jobs:
           # backend 的context目录
           context: "./src/backend/"
           # 是否 docker push
-          push: false
+          push: true
           # docker build arg, 注入 APP_NAME/APP_VERSION
           build-args: |
             APP_NAME="bisheng-backend"
@@ -52,7 +77,7 @@ jobs:
           # frontend 的context目录
           context: "./src/frontend/"
           # 是否 docker push
-          push: false
+          push: true
           # docker build arg, 注入 APP_NAME/APP_VERSION
           build-args: |
             APP_NAME="bisheng-frontend"
@@ -62,19 +87,4 @@ jobs:
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:0.1
 
-      # 构建 bisheng-langchain
-      - name: Set python version 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-
-      - name: Build PyPi bisheng-langchain and push
-        id: pypi_build_bisheng_langchain
-        run: |
-          pip install Cython
-          pip install wheel
-          pip install twine
-          cd ./src/bisheng-langchain
-          python setup.py bdist_wheel
-          repo="http://110.16.193.170:50083/repository/pypi-hosted/"
-          twine upload --verbose -u ${{ secrets.NEXUS_USER }}  -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo  dist/*.whl
+     

docker/bisheng/config/config.yaml

@@ -8,14 +8,13 @@ redis_url:
 knowledges:
   embeddings:
     text-embedding-ada-002:
-      base_url: ""
+      openai_api_base: "https://api.openai.com/v1/"
       openai_proxy: ""
       openai_api_key: ""
   vectorstores:
-    Chroma:
-      persist_directory: "/app/data/chroma_persist"  # In-memory chroma with saving/loading to disk
-    # Milvus: # 如果需要切换其他vectordb，确保其他服务已经启动，然后配置对应参数
-    #   connection_args = {'host': '127.0.0.1', 'port': '19530', 'user': '', 'password': '', 'secure': False}
+    # Milvus 最低要求cpu 4C 8G 推荐4C 16G
+    Milvus: # 如果需要切换其他vectordb，确保其他服务已经启动，然后配置对应参数
+      connection_args = {'host': '110.16.193.170', 'port': '50032', 'user': '', 'password': '', 'secure': False}
 
 agents:
   ZeroShotAgent:

src/backend/bisheng/api/v1/flows.py

@@ -5,8 +5,7 @@
 from bisheng.api.utils import build_flow_no_yield, remove_api_keys
 from bisheng.api.v1.schemas import FlowListCreate, FlowListRead
 from bisheng.database.base import get_session
-from bisheng.database.models.flow import (Flow, FlowCreate, FlowRead,
-                                          FlowReadWithStyle, FlowUpdate)
+from bisheng.database.models.flow import Flow, FlowCreate, FlowRead, FlowReadWithStyle, FlowUpdate
 from bisheng.database.models.template import Template
 from bisheng.database.models.user import User
 from bisheng.settings import settings
@@ -57,11 +56,12 @@ def read_flows(*,
             sql = sql.where(Flow.name.like(f'%{name}%'))
         if status:
             sql = sql.where(Flow.status == status)
+        # count = session.exec(sql.count())
+        # total = count.scalar()
 
         sql = sql.order_by(Flow.update_time.desc())
         if page_num and page_size:
             sql = sql.offset((page_num - 1) * page_size).limit(page_size)
-
         flows = session.exec(sql).all()
 
         res = [jsonable_encoder(flow) for flow in flows]
@@ -88,7 +88,11 @@ def read_flow(*, session: Session = Depends(get_session), flow_id: UUID):
 
 
 @router.patch('/{flow_id}', response_model=FlowRead, status_code=200)
-def update_flow(*, session: Session = Depends(get_session), flow_id: UUID, flow: FlowUpdate, Authorize: AuthJWT = Depends()):
+def update_flow(*,
+                session: Session = Depends(get_session),
+                flow_id: UUID,
+                flow: FlowUpdate,
+                Authorize: AuthJWT = Depends()):
     Authorize.jwt_required()
     payload = json.loads(Authorize.get_jwt_subject())
     """Update a flow."""
@@ -159,7 +163,10 @@ def create_flows(*, session: Session = Depends(get_session), flow_list: FlowList
 
 
 @router.post('/upload/', response_model=List[FlowRead], status_code=201)
-async def upload_file(*, session: Session = Depends(get_session), file: UploadFile = File(...), Authorize: AuthJWT = Depends()):
+async def upload_file(*,
+                      session: Session = Depends(get_session),
+                      file: UploadFile = File(...),
+                      Authorize: AuthJWT = Depends()):
     """Upload flows from a file."""
     contents = await file.read()
     data = json.loads(contents)

src/backend/bisheng/api/v1/knowledge.py

@@ -44,7 +44,7 @@ async def upload_file(*, file: UploadFile = File(...)):
 async def get_embedding():
     try:
         # 获取本地配置的名字
-        model_list = settings.embedding_config
+        model_list = settings.knowledges.get('embeddings')
         models = list(model_list.keys())
         return {'data': {'models': models}}
     except Exception as exc:
@@ -86,6 +86,7 @@ async def process_knowledge(*, session: Session = Depends(get_session), data: di
         logger.info(f'fileName={file_name} col={collection_name}')
     asyncio.create_task(
         addEmbedding(collection_name=collection_name,
+                     model=knowledge.model,
                      chunk_size=chunck_size,
                      file_paths=file_paths,
                      knowledge_files=files))
@@ -105,9 +106,10 @@ def create_knowledge(*,
     payload = json.loads(Authorize.get_jwt_subject())
     """创建知识库."""
     db_knowldge = Knowledge.from_orm(knowledge)
-    know = session.exec(select(Knowledge).where(Knowledge.name == knowledge.name)).all()
+    know = session.exec(
+        select(Knowledge).where(Knowledge.name == knowledge.name, knowledge.user_id == payload.get('user_id'))).all()
     if know:
-        raise HTTPException(status_code=500, detail='name 重复')
+        raise HTTPException(status_code=500, detail='知识库名称重复')
     if not db_knowldge.collection_name:
         # 默认collectionName
         db_knowldge.collection_name = f'col_{int(time.time())}_{str(uuid4())[:8]}'
@@ -200,13 +202,13 @@ def delete_knowledge_file(*, session: Session = Depends(get_session), file_id: i
 def decide_embeddings(model: str) -> Embeddings:
     model_list = settings.knowledges.get('embeddings')
     if model == 'text-embedding-ada-002':
-        return OpenAIEmbeddings(**model_list.get('model'))
+        return OpenAIEmbeddings(**model_list.get(model))
     else:
-        return HostEmbeddings(**model_list.get('model'))
+        return HostEmbeddings(**model_list.get(model))
 
 
 def decide_vectorstores(collection_name: str, embedding: Embeddings) -> VectorStore:
-    param = {'collection_name': collection_name, 'embedding_function': embedding}
+    param = {'collection_name': collection_name, 'embedding': embedding}
     vector_store = list(settings.knowledges.get('vectorstores').keys())[0]
     vector_config = settings.knowledges.get('vectorstores').get(vector_store)
     param.update(vector_config)

src/backend/bisheng/config.yaml

@@ -1,3 +1,21 @@
+# 数据库配置
+database_url:
+  "mysql+pymysql://root:1234@mysql/bisheng"
+redis_url:
+  "redis:6379"
+
+# 为知识库的embedding进行模型撇脂
+knowledges:
+  embeddings:
+    text-embedding-ada-002:
+      openai_api_base: "https://api.openai.com"
+      openai_proxy: ""
+      openai_api_key: ""
+  vectorstores:
+    # Milvus 最低要求cpu 4C 8G 推荐4C 16G
+    Milvus: # 如果需要切换其他vectordb，确保其他服务已经启动，然后配置对应参数
+      connection_args: {'host': '110.16.193.170', 'port': '50032', 'user': '', 'password': '', 'secure': False}
+
 agents:
   ZeroShotAgent:
     documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
@@ -329,15 +347,4 @@ input_output:
   InputFile:
     documentation: ""
 
-embedding_config:
-  text-embedding-ada-002:
-    base_url:
-      ""
-  multilingual-e5-large:
-    base_url:
-      ""
 
-database_url:
-  "mysql+pymysql://root:[email protected]:3306/langflow"
-redis_url:
-  "192.168.106.116:6379"

src/backend/bisheng/database/models/flow.py

@@ -61,6 +61,7 @@ class FlowRead(FlowBase):
 
 class FlowReadWithStyle(FlowRead):
     style: Optional['FlowStyleRead'] = None
+    total: Optional[int] = None
 
 
 class FlowUpdate(SQLModelSerializable):

src/backend/bisheng/interface/initialize/vector_store.py

@@ -2,6 +2,7 @@
 import os
 from typing import Any, Callable, Dict, Type
 
+from bisheng.settings import settings
 from bisheng_langchain.vectorstores import ElasticKeywordsSearch
 from langchain.vectorstores import (FAISS, Chroma, Milvus, MongoDBAtlasVectorSearch, Pinecone,
                                     Qdrant, SupabaseVectorStore, Weaviate)
@@ -202,7 +203,7 @@ def initialize_qdrant(class_object: Type[Qdrant], params: dict):
 
 def initial_milvus(class_object: Type[Milvus], params: dict):
     if 'connection_args' not in params:
-        connection_args = {'host': '192.168.106.12', 'port': '19530', 'user': '', 'secure': False, 'password': ''}
+        connection_args = settings.knowledges.get('vectorstores').get('Milvus')
         params['connection_args'] = connection_args
     elif isinstance(params.get('connection_args'), str):
         print(f"milvus before params={params} type={type(params['connection_args'])}")

src/backend/bisheng/template/frontend_node/vectorstores.py

@@ -5,6 +5,7 @@
 
 
 class VectorStoreFrontendNode(FrontendNode):
+
     def add_extra_fields(self) -> None:
         extra_fields: List[TemplateField] = []
         # Add search_kwargs field
@@ -295,4 +296,3 @@ def format_field(field: TemplateField, name: Optional[str] = None) -> None:
         elif field.name == 'connection_args':
             field.show = True
             field.advanced = True
-            field.value = {'host': '192.168.106.116', 'port': '19530', 'user': '', 'password': '', 'secure': False}