diff --git a/.github/workflows/docker_jupyter_base_image.yml b/.github/workflows/docker_jupyter_base_image.yml index 1f149e90..fcbfc1c6 100644 --- a/.github/workflows/docker_jupyter_base_image.yml +++ b/.github/workflows/docker_jupyter_base_image.yml @@ -33,7 +33,7 @@ jobs: echo "* Manual build wedpr-jupyter-image image, docker version: ${{ github.event.inputs.tags }}" DOCKER_TAG="fiscoorg/wedpr-jupyter-image:${{ github.event.inputs.tags }}" echo "* Begin to build ${DOCKER_TAG}" - cd docker-files/base + cd docker-files/jupyter docker build -t ${DOCKER_TAG} . echo "* Build ${DOCKER_TAG} success" docker push ${DOCKER_TAG} diff --git a/docker-files/jupyter/Dockerfile b/docker-files/jupyter/Dockerfile index 64b7f34a..6f29bf7c 100644 --- a/docker-files/jupyter/Dockerfile +++ b/docker-files/jupyter/Dockerfile @@ -1,4 +1,4 @@ -FROM ficoorg/wedpr-image:v3.0.0 as wedpr-jupyter-image +FROM fiscoorg/wedpr-image:v3.0.0 as wedpr-jupyter-image ENV LANG zh_CN.UTF-8 ENV LANGUAGE zh_CN.UTF-8 diff --git a/docker-files/site/Dockerfile b/docker-files/site/Dockerfile index 552faca1..1f0c4426 100644 --- a/docker-files/site/Dockerfile +++ b/docker-files/site/Dockerfile @@ -41,7 +41,7 @@ FROM fiscoorg/wedpr-image:v3.0.0 as wedpr-pir LABEL maintainer service@webank.com RUN mkdir -p /data/home/wedpr -COPY --from=builder /WeDPR/wedpr-pir/dist/ /data/app/wedpr-pir +COPY --from=builder /WeDPR/wedpr-pir/dist/ /data/home/wedpr/wedpr-pir ENTRYPOINT ["/bin/bash", "/data/home/wedpr/wedpr-pir/start.sh", "true"] # the wedpr-site diff --git a/wedpr-builder/wedpr_builder/common/constant.py b/wedpr-builder/wedpr_builder/common/constant.py index 1f2f0b7e..62d469c1 100644 --- a/wedpr-builder/wedpr_builder/common/constant.py +++ b/wedpr-builder/wedpr_builder/common/constant.py @@ -83,6 +83,7 @@ class ConfigInfo: wedpr_pir_docker_dir = "wedpr-pir" wedpr_site_docker_dir = "wedpr-site" wedpr_mpc_docker_dir = "wedpr-mpc-service" + wedpr_model_docker_dir = "wedpr-model" wedpr_gateway_service_dir = "wedpr-gateway-service" wedpr_node_service_dir = "wedpr-pro-node-service" @@ -110,6 +111,7 @@ class ConfigInfo: scripts_file_list = ["start.sh", "stop.sh"] # the default path for docker default_spdz_home = "/data/app/wedpr/scripts/wedpr-mpc-no-gateway" + cpp_component_cmd = " -c config.ini " @staticmethod def get_docker_path(file_path: str): @@ -177,6 +179,7 @@ class ConfigProperities: WEDPR_LOG_DIR = "WEDPR_LOG_DIR" # specify the mounted docker log path DOCKER_LOG_PATH = "DOCKER_LOG_PATH" + DOCKER_CMD = "DOCKER_CMD" # specify the exposed docker port list WEDPR_DOCKER_EXPORSE_PORT_LIST = "WEDPR_DOCKER_EXPORSE_PORT_LIST" # the created docker name diff --git a/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py b/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py index f30f2f58..9806d6d3 100644 --- a/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py +++ b/wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py @@ -867,10 +867,12 @@ def to_properties(self) -> {}: {constant.ConfigProperities.WEDPR_API_TOKEN: self.wedpr_api_token}) # EXTENDED_MOUNT_CONF default is empty string props.update({constant.ConfigProperities.EXTENDED_MOUNT_CONF: ""}) + props.update({constant.ConfigProperities.DOCKER_CMD: ""}) return props def get_wedpr_model_properties(self, deploy_ip: str, node_index: int) -> {}: props = self.to_properties() + prefix_path = constant.ConfigInfo.wedpr_model_docker_dir # the zone config props.update(self.env_config.to_properties()) # the sql config @@ -884,21 +886,25 @@ def get_wedpr_model_properties(self, deploy_ip: str, node_index: int) -> {}: props.update( {constant.ConfigProperities.WEDPR_CONFIG_DIR: "application.yml"}) props.update( - {constant.ConfigProperities.DOCKER_CONF_PATH: constant.ConfigInfo.get_docker_path("model/application.yml")}) + {constant.ConfigProperities.DOCKER_CONF_PATH: constant.ConfigInfo.get_docker_path(f"{prefix_path}/application.yml")}) # the extended mount info local_path = "${SHELL_FOLDER}/logging.conf" - docker_path = constant.ConfigInfo.get_docker_path("model/logging.conf") + docker_path = constant.ConfigInfo.get_docker_path(os.path.join( + prefix_path, "logging.conf")) extended_mount_info = f" -v {local_path}:{docker_path} " local_path = "${SHELL_FOLDER}/wedpr_sdk_log_config.ini" docker_path = constant.ConfigInfo.get_docker_path( - "model/wedpr_sdk_log_config.ini") + os.path.join(prefix_path, "wedpr_sdk_log_config.ini")) extended_mount_info = f"{extended_mount_info} -v {local_path}:{docker_path} " + # set the working directory + working_dir = constant.ConfigInfo.get_docker_path(prefix_path) + extended_mount_info = f"{extended_mount_info} -w {working_dir}" props.update( {constant.ConfigProperities.EXTENDED_MOUNT_CONF: extended_mount_info}) # set the log mount information props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "logs"}) props.update({constant.ConfigProperities.DOCKER_LOG_PATH: - constant.ConfigInfo.get_docker_path("model/logs")}) + constant.ConfigInfo.get_docker_path(f"{constant.ConfigInfo.wedpr_model_docker_dir}/logs")}) return props @staticmethod @@ -907,26 +913,20 @@ def generate_cpp_component_docker_properties( prefix_path, zone_name: str, service_type: str, env_config, exposed_port_list: str, node_index: int): props = {} - # the config mount info + # the config mount info: mount the whole directory props.update( - {constant.ConfigProperities.WEDPR_CONFIG_DIR: "config.ini"}) - path = constant.ConfigInfo.get_docker_path(f"{prefix_path}/config.ini") + {constant.ConfigProperities.WEDPR_CONFIG_DIR: ""}) + path = constant.ConfigInfo.get_docker_path(f"{prefix_path}/") props.update( {constant.ConfigProperities.DOCKER_CONF_PATH: path}) - # set the extended mont config - local_mount_dir = '${SHELL_FOLDER}/conf' - remote_mount_dir = constant.ConfigInfo.get_docker_path( - f"{prefix_path}/conf") - extended_mount_conf = f" -v {local_mount_dir}:{remote_mount_dir} " - # nodes.json for gateway service - if service_type == constant.ServiceInfo.gateway_service_type: - node_connection_file = "nodes.json" - local_mount_dir = '${SHELL_FOLDER}/%s' % node_connection_file - remote_mount_dir = constant.ConfigInfo.get_docker_path( - f"{prefix_path}/{node_connection_file}") - extended_mount_conf = f"{extended_mount_conf} -v {local_mount_dir}:{remote_mount_dir} " + # set the working directory + working_dir = constant.ConfigInfo.get_docker_path(f"{prefix_path}") + extended_mount_conf = f" -w {working_dir}" props.update( {constant.ConfigProperities.EXTENDED_MOUNT_CONF: extended_mount_conf}) + # specify the cmd + props.update( + {constant.ConfigProperities.DOCKER_CMD: f"{constant.ConfigInfo.cpp_component_cmd}"}) # specify the log path to mount props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "log"}) props.update({constant.ConfigProperities.DOCKER_LOG_PATH: @@ -1032,9 +1032,9 @@ def get_mpc_properties(self, deploy_ip: str, node_index: int): # the config mount info docker_prefix_path = constant.ConfigInfo.wedpr_mpc_docker_dir props.update( - {constant.ConfigProperities.WEDPR_CONFIG_DIR: "config.ini"}) + {constant.ConfigProperities.WEDPR_CONFIG_DIR: ""}) path = constant.ConfigInfo.get_docker_path( - f"{docker_prefix_path}/config.ini") + f"{docker_prefix_path}") props.update( {constant.ConfigProperities.DOCKER_CONF_PATH: path}) @@ -1042,6 +1042,14 @@ def get_mpc_properties(self, deploy_ip: str, node_index: int): props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "log"}) props.update({constant.ConfigProperities.DOCKER_LOG_PATH: constant.ConfigInfo.get_docker_path(f"{docker_prefix_path}/log")}) + # specify the extended mount info + working_directory = constant.ConfigInfo.get_docker_path( + docker_prefix_path) + props.update( + {constant.ConfigProperities.EXTENDED_MOUNT_CONF: f" -w {working_directory}"}) + # specify the docker command + props.update( + {constant.ConfigProperities.DOCKER_CMD: constant.ConfigInfo.cpp_component_cmd}) return props def __update_dml__(self, sql, dml_file_path, use_double_quote=False): diff --git a/wedpr-builder/wedpr_builder/generator/wedpr_service_generator.py b/wedpr-builder/wedpr_builder/generator/wedpr_service_generator.py index e52da858..8df9133a 100644 --- a/wedpr-builder/wedpr_builder/generator/wedpr_service_generator.py +++ b/wedpr-builder/wedpr_builder/generator/wedpr_service_generator.py @@ -118,7 +118,8 @@ def __generate_single_node_config__( utilities.mkdir(node_path) # copy configuration into the dest path command = f"cp -r {service_config.tpl_config_file_path} {node_path}" - if service_config.service_type == constant.ServiceInfo.wedpr_model_service: + if service_config.service_type == constant.ServiceInfo.wedpr_model_service \ + or service_config.service_type == constant.ServiceInfo.wedpr_mpc_service: command = f"cp {service_config.tpl_config_file_path}/* {node_path}" (ret, output) = utilities.execute_command_and_getoutput(command) if ret is False: diff --git a/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh b/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh index c74183e3..215998b2 100644 --- a/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh +++ b/wedpr-builder/wedpr_builder/tpl/docker/create_docker.sh @@ -15,7 +15,7 @@ if [[ "${confirm}" == "Y" || "${confirm}" == "y" ]]; then LOG_INFO "* Pull image ${WEDPR_IMAGE_DESC}" docker pull ${WEDPR_IMAGE_DESC} LOG_INFO "* Pull image ${WEDPR_IMAGE_DESC} success, begin to create docker" - docker run -d ${WEDPR_DOCKER_EXPORSE_PORT_LIST} -v ${SHELL_FOLDER}/${WEDPR_CONFIG_DIR}:${DOCKER_CONF_PATH} -v ${SHELL_FOLDER}/${WEDPR_LOG_DIR}:${DOCKER_LOG_PATH} ${EXTENDED_MOUNT_CONF} --name ${WEDPR_DOCKER_NAME} ${WEDPR_IMAGE_DESC} + docker run -d ${WEDPR_DOCKER_EXPORSE_PORT_LIST} -v ${SHELL_FOLDER}/${WEDPR_CONFIG_DIR}:${DOCKER_CONF_PATH} -v ${SHELL_FOLDER}/${WEDPR_LOG_DIR}:${DOCKER_LOG_PATH} ${EXTENDED_MOUNT_CONF} --name ${WEDPR_DOCKER_NAME} ${WEDPR_IMAGE_DESC} ${DOCKER_CMD} LOG_INFO "Create docker: ${WEDPR_DOCKER_NAME} success" else LOG_INFO "Exit without create docker ${WEDPR_DOCKER_NAME}" diff --git a/wedpr-builder/wedpr_builder/tpl/model/conf/application.yml b/wedpr-builder/wedpr_builder/tpl/model/conf/application.yml index 50cb86f3..574d1da5 100644 --- a/wedpr-builder/wedpr_builder/tpl/model/conf/application.yml +++ b/wedpr-builder/wedpr_builder/tpl/model/conf/application.yml @@ -35,7 +35,7 @@ transport_threadpool_size: 4 transport_node_id: "${WEDPR_NODE_ID}" transport_gateway_targets: "${GATEWAY_TARGET}" transport_host_ip: "${WEDPR_TRANSPORT_HOST_IP}" -transport_listen_port: ${WEDPR_SERVER_LISTEN_PORT} +transport_listen_port: ${WEDPR_TRANSPORT_LISTEN_PORT} # the log retriever config LOG_PATH: "logs/wedpr-model.log" \ No newline at end of file