node and gateway support docker

Author: cyjseagull

wedpr-builder/conf/config-example.toml

@@ -14,7 +14,9 @@ wedpr_model_source_path = "/data/home/wedpr/WeDPR-Component/python/"
 docker_mode = true
 
 #### define the docker images desc ###
-wedpr_cpp_component_image_desc = "wedpr-cpp-component:latest"
+wedpr_gateway_service_image_desc = "wedpr-gateway-service:latest"
+wedpr_node_service_image_desc = "wedpr-pro-node-service:latest"
+wedpr_mpc_service_image_desc = "wedpr-mpc-service:latest"
 wedpr_jupyter_worker_image_desc = "wedpr-jupyter-worker:latest"
 wedpr_model_image_desc = "wedpr-model-service:latest"
 wedpr_site_image_desc = "wedpr-site:latest"

wedpr-builder/wedpr_builder/common/constant.py

@@ -18,11 +18,14 @@ class ServiceInfo:
     gateway_service_type = "wedpr-gateway"
     wedpr_site_service = "wedpr-site"
     wedpr_pir_service = "wedpr-pir"
+    wedpr_mpc_service = "wedpr-mpc-service"
     wedpr_model_service = "wedpr-model"
     wedpr_jupyter_worker_service = "wedpr-jupyter-worker"
     supported_service_type = [node_service_type, gateway_service_type,
                               wedpr_site_service, wedpr_pir_service,
-                              wedpr_jupyter_worker_service, wedpr_model_service]
+                              wedpr_jupyter_worker_service,
+                              wedpr_model_service,
+                              wedpr_mpc_service]
 
 
 def get_abs_path(file_path, tpl_abs_path="wedpr_builder/tpl/"):
@@ -68,6 +71,10 @@ class ConfigInfo:
     wedpr_worker_docker_dir = "wedpr-worker"
     wedpr_pir_docker_dir = "wedpr-pir"
     wedpr_site_docker_dir = "wedpr-site"
+
+    wedpr_gateway_service_dir = "wedpr-gateway-service"
+    wedpr_node_service_dir = "wedpr-pro-node-service"
+    wedpr_mpc_service_dir = "wedpr-mpc-service"
     docker_file_list = ["create_docker.sh",
                         "start_docker.sh", "stop_docker.sh"]
 
@@ -132,3 +139,4 @@ class ConfigProperities:
     WEDPR_DOCKER_EXPORSE_PORT_LIST = "WEDPR_DOCKER_EXPORSE_PORT_LIST"
     # the created docker name
     WEDPR_DOCKER_NAME = "WEDPR_DOCKER_NAME"
+    EXTENDED_MOUNT_CONF = "EXTENDED_MOUNT_CONF"

wedpr-builder/wedpr_builder/config/wedpr_deploy_config.py

@@ -13,13 +13,15 @@ def __init__(self, node_must_exists: bool = False,
                  site_must_exists: bool = False,
                  pir_must_exists: bool = False,
                  jupyter_must_exists: bool = False,
-                 model_must_exists: bool = False):
+                 model_must_exists: bool = False,
+                 mpc_service_must_exists: bool = False):
         self.node_must_exists = node_must_exists
         self.gateway_must_exists = gateway_must_exists
         self.site_must_exists = site_must_exists
         self.pir_must_exists = pir_must_exists
         self.jupyter_must_exists = jupyter_must_exists
         self.model_must_exists = model_must_exists
+        self.mpc_service_must_exists = mpc_service_must_exists
 
 
 class PeerInfo:
@@ -70,18 +72,31 @@ def __init__(self, config, section_name: str, component_switch: ComponentSwitch)
             self.config, self.section_name, "wedpr_jupyter_worker_image_desc",
             None, self.component_switch.jupyter_must_exists)
         # the cpp component image desc
-        self.wedpr_cpp_component_image_desc = utilities.get_value(
-            self.config, self.section_name, "wedpr_cpp_component_image_desc", None,
-            (self.component_switch.gateway_must_exists or self.component_switch.node_must_exists))
+        self.wedpr_gateway_service_image_desc = utilities.get_value(
+            self.config, self.section_name, "wedpr_gateway_service_image_desc", None,
+            self.component_switch.gateway_must_exists and self.docker_mode is True)
+        # the node service image desc
+        self.wedpr_node_service_image_desc = utilities.get_value(
+            self.config, self.section_name,
+            "wedpr_node_service_image_desc", None,
+            self.component_switch.node_must_exists and self.docker_mode is True)
+        # the mpc service image desc
+        self.wedpr_mpc_service_image_desc = utilities.get_value(
+            self.config, self.section_name,
+            "wedpr_mpc_service_image_desc", None,
+            self.docker_mode is True and self.component_switch.mpc_service_must_exists)
         # the model image desc
         self.wedpr_model_image_desc = utilities.get_value(
-            self.config, self.section_name, "wedpr_model_image_desc", None, self.component_switch.model_must_exists)
+            self.config, self.section_name, "wedpr_model_image_desc", None,
+            self.component_switch.model_must_exists)
         # the pir image
         self.wedpr_pir_image_desc = utilities.get_value(
-            self.config, self.section_name, "wedpr_pir_image_desc", None, self.component_switch.pir_must_exists)
+            self.config, self.section_name, "wedpr_pir_image_desc", None,
+            self.component_switch.pir_must_exists)
         # the site image
         self.wedpr_site_image_desc = utilities.get_value(
-            self.config, self.section_name, "wedpr_site_image_desc", None, self.component_switch.site_must_exists)
+            self.config, self.section_name, "wedpr_site_image_desc", None,
+            self.component_switch.site_must_exists)
 
     # Note: jupyter only use docker mode
     def get_dist_path_by_service_type(self, service_type: str) -> str:
@@ -102,9 +117,12 @@ def get_image_desc_by_service_name(self, service_type: str) -> str:
             return self.wedpr_pir_image_desc
         if service_type == constant.ServiceInfo.wedpr_jupyter_worker_service:
             return self.wedpr_jupyter_worker_image_desc
-        if service_type == constant.ServiceInfo.gateway_service_type or \
-                service_type == constant.ServiceInfo.node_service_type:
-            return self.wedpr_cpp_component_image_desc
+        if service_type == constant.ServiceInfo.gateway_service_type:
+            return self.wedpr_gateway_service_image_desc
+        if service_type == constant.ServiceInfo.node_service_type:
+            return self.wedpr_node_service_image_desc
+        if service_type == constant.ServiceInfo.wedpr_mpc_service:
+            return self.wedpr_mpc_service_image_desc
         return None
 
     def __repr__(self):
@@ -162,9 +180,13 @@ class GatewayConfig:
     the gateway config
     """
 
-    def __init__(self, agency_name: str, holding_msg_minutes: int,
+    def __init__(self, agency_name: str,
+                 env_config: EnvConfig,
+                 holding_msg_minutes: int,
                  config, config_section: str, must_exist: bool):
         self.config = config
+        self.service_type = constant.ServiceInfo.gateway_service_type
+        self.env_config = env_config
         self.config_section = config_section
         self.holding_msg_minutes = holding_msg_minutes
         self.agency_name = agency_name
@@ -494,10 +516,14 @@ class NodeConfig:
     the ppc-node config
     """
 
-    def __init__(self, agency_name: str, holding_msg_minutes: int, gateway_targets,
+    def __init__(self, agency_name: str,
+                 env_config: EnvConfig,
+                 holding_msg_minutes: int, gateway_targets,
                  hdfs_storage_config: HDFSStorageConfig, config, must_exist: bool):
         self.config = config
         self.section_name = "[[agency.node]]."
+        self.service_type = constant.ServiceInfo.node_service_type
+        self.env_config = env_config
         self.holding_msg_minutes = holding_msg_minutes
         # the hdfs config
         self.hdfs_storage_config = hdfs_storage_config
@@ -599,7 +625,8 @@ def __init__(self, config, env_config: EnvConfig,
         self.gateway_config = None
         if gateway_config_object is not None:
             self.gateway_config = GatewayConfig(
-                self.agency_name, self.holding_msg_minutes, gateway_config_object,
+                self.agency_name, self.env_config,
+                self.holding_msg_minutes, gateway_config_object,
                 gateway_config_section_name, self.component_switch.gateway_must_exists)
         utilities.log_debug("load the gateway config success")
 
@@ -657,6 +684,7 @@ def __init__(self, config, env_config: EnvConfig,
         for node_object in node_config_list:
             node_config = NodeConfig(
                 agency_name=self.agency_name,
+                env_config=self.env_config,
                 holding_msg_minutes=self.holding_msg_minutes,
                 hdfs_storage_config=self.hdfs_storage_config,
                 config=node_object,
@@ -711,6 +739,8 @@ def to_properties(self) -> {}:
             {constant.ConfigProperities.WEDPR_AGENCY: self.agency_name})
         props.update(
             {constant.ConfigProperities.PSI_API_TOKEN: self.psi_api_token})
+        # EXTENDED_MOUNT_CONF default is empty string
+        props.update({constant.ConfigProperities.EXTENDED_MOUNT_CONF: ""})
         return props
 
     def get_wedpr_model_properties(self, deploy_ip: str, node_index: int) -> {}:
@@ -734,6 +764,42 @@ def get_wedpr_model_properties(self, deploy_ip: str, node_index: int) -> {}:
                      constant.ConfigInfo.get_docker_path("model/logs")})
         return props
 
+    @staticmethod
+    def generate_cpp_component_docker_properties(
+            prefix_path, service_type: str, env_config,
+            exposed_port_list: str, node_index: int):
+        props = {}
+        # the config mount info
+        props.update(
+            {constant.ConfigProperities.WEDPR_CONFIG_DIR: "config.ini"})
+        path = constant.ConfigInfo.get_docker_path(f"{prefix_path}/config.ini")
+        props.update(
+            {constant.ConfigProperities.DOCKER_CONF_PATH: path})
+        # set the extended mont config
+        local_mount_dir = '${SHELL_FOLDER}/conf'
+        remote_mount_dir = constant.ConfigInfo.get_docker_path(
+            f"{prefix_path}/conf")
+        extended_mount_conf = f"-v {local_mount_dir}:{remote_mount_dir} "
+        props.update(
+            {constant.ConfigProperities.EXTENDED_MOUNT_CONF: extended_mount_conf})
+        # specify the conf path to mount
+        props.update({constant.ConfigProperities.WEDPR_LOG_DIR: "logs"})
+        props.update({constant.ConfigProperities.DOCKER_LOG_PATH:
+                      constant.ConfigInfo.get_docker_path(f"{prefix_path}/logs")})
+        # set the image desc for docker mode
+        image_desc = env_config.get_image_desc_by_service_name(service_type)
+        if image_desc is not None:
+            props.update(
+                {constant.ConfigProperities.WEDPR_IMAGE_DESC: image_desc})
+        # set the exposed port
+        props.update(
+            {constant.ConfigProperities.WEDPR_DOCKER_EXPORSE_PORT_LIST: exposed_port_list})
+        # set the docker name
+        docker_name = f"{service_type}-node{node_index}"
+        props.update(
+            {constant.ConfigProperities.WEDPR_DOCKER_NAME: docker_name})
+        return props
+
     def __generate_java_service_docker_properties__(self, prefix_path) -> {}:
         props = {}
         # the config mount info

wedpr-builder/wedpr_builder/generator/shell_script_generator.py

@@ -9,7 +9,10 @@ class ShellScriptGenerator:
     """
     generate the shell-scripts
     """
-    def generate_ip_shell_scripts(script_output_dir, start_shell_script_name, stop_shell_script_name):
+    def generate_ip_shell_scripts(
+            script_output_dir: str,
+            start_shell_script_name,
+            stop_shell_script_name):
         start_all_path = os.path.join(
             script_output_dir, start_shell_script_name)
         utilities.mkdir(script_output_dir)

wedpr-builder/wedpr_builder/generator/wedpr_gateway_config_generator.py

@@ -8,6 +8,8 @@
 from wedpr_builder.generator.binary_generator import BinaryGenerator
 from wedpr_builder.generator.cert_generator import CertGenerator
 from wedpr_builder.generator.shell_script_generator import ShellScriptGenerator
+from wedpr_builder.config.wedpr_deploy_config import AgencyConfig
+from wedpr_builder.config.wedpr_deploy_config import GatewayConfig
 
 
 class WeDPRGatewayConfigGenerator:
@@ -47,15 +49,70 @@ def generate_gateway_config(self):
             "* generate gateway config success, deploy_dir: %s" % self.config.env_config.deploy_dir)
         return True
 
-    def __generate_single_gateway_config__(self, gateway_config, connection_config, connection_config_path_list):
+    def __copy_binary__(self, agency_name, ip):
+        if self.config.env_config.docker_mode is True:
+            utilities.log_info(
+                "* No need to copy binary for enable docker mode")
+            return True
+        # generate the binary
+        binary_path = os.path.join(
+            self.config.env_config.binary_path, self.binary_name)
+        dst_binary_path = os.path.join(
+            self.__generate_ip_shell_scripts_output_path__(
+                agency_name, ip),
+            self.binary_name)
+        return BinaryGenerator.generate_binary(
+            binary_path, dst_binary_path)
+
+    def __generate_shell_scripts__(self, agency_name, ip, node_name):
+        if self.config.env_config.docker_mode is True:
+            utilities.log_info(
+                "* No need to copy the shell scripts for enable docker mode")
+            return True
+        return ShellScriptGenerator.generate_node_shell_scripts(
+            self.__generate_node_path__(
+                agency_name, ip, node_name), self.binary_name)
+
+    def __generate_docker_config__(
+            self, gateway_config: GatewayConfig,
+            node_path, exposed_port_list, node_index):
+        if self.config.env_config.docker_mode is False:
+            return True
+        # copy the docker files
+        command = f"cp {constant.ConfigInfo.docker_tpl_path}/*.sh {node_path}"
+        (ret, output) = utilities.execute_command_and_getoutput(command)
+        if ret is False:
+            raise Exception(f"Copy docker tpl file  from "
+                            f"{constant.ConfigInfo.docker_tpl_path} to {node_path} "
+                            f"failed, reason: {output}")
+        props = AgencyConfig.generate_cpp_component_docker_properties(
+            constant.ConfigInfo.wedpr_gateway_service_dir,
+            gateway_config.service_type, self.config.env_config,
+            exposed_port_list, node_index)
+        # substitute
+        for file in constant.ConfigInfo.docker_file_list:
+            utilities.substitute_configurations(
+                props, os.path.join(node_path, file))
+        return True
+
+    def __generate_ip_shell_scripts__(self, agency_name, ip):
+        if self.config.env_config.docker_mode is True:
+            return True
+        return ShellScriptGenerator.generate_ip_shell_scripts(
+            self.__generate_ip_shell_scripts_output_path__(
+                agency_name, ip), "start_all.sh", "stop_all.sh")
+
+    def __generate_single_gateway_config__(
+            self, gateway_config, connection_config,
+            connection_config_path_list):
         # load the config from tpl_config_path
         utilities.log_info("* generate config for ppc-gateway")
         for ip_str in gateway_config.deploy_ip:
             ip_array = ip_str.split(":")
             ip = ip_array[0]
             # generate the shell scripts for the given ip
-            ret = ShellScriptGenerator.generate_ip_shell_scripts(
-                self.__generate_ip_shell_scripts_output_path__(gateway_config.agency_name, ip), "start_all.sh", "stop_all.sh")
+            ret = self.__generate_ip_shell_scripts__(
+                gateway_config.agency_name, ip)
             if ret is False:
                 return False
             node_count = 1
@@ -78,19 +135,14 @@ def __generate_single_gateway_config__(self, gateway_config, connection_config,
                 self.__generate_gateway_config_content__(
                     gateway_config, config_content, listen_port, grpc_listen_port)
                 # generate the binary
-                binary_path = os.path.join(
-                    self.config.env_config.binary_path, self.binary_name)
-                dst_binary_path = os.path.join(
-                    self.__generate_ip_shell_scripts_output_path__(
-                        gateway_config.agency_name, ip),
-                    self.binary_name)
-                ret = BinaryGenerator.generate_binary(
-                    binary_path, dst_binary_path)
+                ret = self.__copy_binary__(gateway_config.agency_name, ip)
                 if ret is False:
                     return False
                 # generate the ini config
+                node_path = self.__generate_node_path__(
+                    gateway_config.agency_name, ip, node_name)
                 ini_config_output_path = os.path.join(
-                    self.__generate_node_path__(gateway_config.agency_name, ip, node_name), constant.ConfigInfo.config_ini_file)
+                    node_path, constant.ConfigInfo.config_ini_file)
                 ret = utilities.store_config(
                     config_content, "ini", ini_config_output_path, "config.ini")
                 if ret is False:
@@ -104,10 +156,14 @@ def __generate_single_gateway_config__(self, gateway_config, connection_config,
                     utilities.log_error(
                         "* generate config for ppc-gateway failed for generate the node config failed")
                     return False
-                ret = ShellScriptGenerator.generate_node_shell_scripts(self.__generate_node_path__(
-                    gateway_config.agency_name, ip, node_name), self.binary_name)
+                ret = self.__generate_shell_scripts__(
+                    gateway_config.agency_name, ip, node_name)
                 if ret is False:
                     return False
+                # try to generate docker config in the docker mode
+                exposed_port = f"-p {listen_port}:{listen_port} -p {grpc_listen_port}:{grpc_listen_port}"
+                self.__generate_docker_config__(
+                    gateway_config, node_path, exposed_port, node_index)
                 utilities.print_badge("* generate config for ppc-gateway%s success" %
                                       (node_name))
         utilities.log_info("* generate config for ppc-gateway success")
@@ -127,7 +183,7 @@ def __generate_connection_info__(self, connection_config_path_list, connection_c
     def __generate_ca_cert_path__(self):
         return os.path.join(self.output_dir, self.service_type, "ca")
 
-    def __generate_node_path__(self, agency_name, ip, node_name):
+    def __generate_node_path__(self, agency_name: str, ip, node_name):
         return os.path.join(self.output_dir, agency_name, ip, self.service_type, node_name)
 
     def __generate_conf_output_path__(self, agency_name, ip, node_name):