add jupyterClient implement

db/wedpr_dml.sql

@@ -1,12 +1,16 @@
 -- the Wizard algorithm template
 insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_algorithm_templates", '{"version":"1.0","templates":[{"name":"PSI","title":"数据对齐","detail":"","version":"1.0"},{"name":"XGB_TRAINING","title":"SecureLGBM训练","detail":"","version":"1.0"},{"name":"XGB_PREDICTING","title":"SecureLGBM预测","detail":"","version":"1.0"}]}');
 
+
+-- the jupyter related host settings
+-- insert into `wedpr_config_table`(`config_key`, `config_value`) values("jupyter_entrypoints", '{"hostSettings":[{"entryPoint":"192.168.0.238","maxJupyterCount":5,"jupyterStartPort":14000}]}');
+
 -- the  jupyter related code template
 -- command to create user
-insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_create_user", 'useradd -m ${user_name} -p ${user_pass}');
+insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_create_user", 'useradd -m ${user_name}');
 -- command to start jupyter
-insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_start_jupyter", 'su ${user_name} && mkdir -p  ~/project && jupyter-lab --config=/home/share/.jupyter/jupyter_lab_config.py --ip ${listen_ip} --port ${listen_port} --notebook-dir ~/project');
-insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_stop_jupyter", 'ps aux | grep -i jupyter-lab | grep -i ${user_name} | grep -v grep | awk -F\' \' \'{print $2}\' | xargs kill');
+insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_start_jupyter", 'su ${user_name} && mkdir -p  ${jupyter_project_path} && nohup ${jupyter_binary} --config=${jupyter_config_path} --ip ${listen_ip} --port ${listen_port} --notebook-dir ${jupyter_project_path} >> ${user_name}.jupyter.out 2>&1 &');
+insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_get_jupyter_pid", 'ps aux | grep -i ${jupyter_binary} | grep -i ${jupyter_project_path} | grep -v grep | awk -F\' \' \'{print $2}\'');
 insert into `wedpr_config_table`(`config_key`, `config_value`) values("wedpr_delete_user", 'userdel -m ${user_name}');
 
 

wedpr-adm/src/main/java/com/webank/wedpr/adm/controller/JupyterController.java

@@ -162,7 +162,11 @@ public WeDPRResponse destroy(@RequestParam String id, HttpServletRequest request
         try {
             UserToken userToken = TokenUtils.getLoginUser(request);
             boolean success =
-                    this.jupyterService.destroy(userToken.isAdmin(), userToken.getUsername(), id);
+                    this.jupyterService.destroy(
+                            userToken.isAdmin(),
+                            userToken.getUsername(),
+                            WeDPRCommonConfig.getAgency(),
+                            id);
             WeDPRResponse response =
                     new WeDPRResponse(Constant.WEDPR_SUCCESS, Constant.WEDPR_SUCCESS_MSG);
             response.setData(success);

wedpr-components/env-integration/jupyter/build.gradle

@@ -6,5 +6,6 @@ dependencies{
     compile project(":wedpr-components-mybatis")
     compile project(":wedpr-components-uuid")
     compile project(":wedpr-components-sys-config")
+    compile project(":wedpr-components-http-client")
     compile project(":wedpr-components-hook")
 }

wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/client/JupyterClient.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2017-2025  [webank-wedpr]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+package com.webank.wedpr.components.integration.jupyter.client;
+
+import com.webank.wedpr.components.integration.jupyter.core.JupyterConfig;
+import com.webank.wedpr.components.integration.jupyter.dao.JupyterInfoDO;
+import com.webank.wedpr.core.config.WeDPRCommonConfig;
+import com.webank.wedpr.core.protocol.task.TaskResponse;
+import java.util.HashMap;
+import java.util.Map;
+
+public interface JupyterClient {
+
+    TaskResponse create(JupyterInfoDO jupyterInfo) throws Exception;
+
+    TaskResponse start(JupyterInfoDO jupyterInfo) throws Exception;
+
+    TaskResponse stop(JupyterInfoDO jupyterInfo) throws Exception;
+
+    static Map<String, String> generateParamMap(JupyterInfoDO jupyterInfoDO) {
+        Map<String, String> paramMap = new HashMap<>();
+        // the username
+        paramMap.put(WeDPRCommonConfig.getParamKeyUserName(), jupyterInfoDO.getOwner());
+        // the listen ip
+        paramMap.put(
+                WeDPRCommonConfig.getParamKeyListenIp(),
+                jupyterInfoDO.getJupyterSetting().getListenIp());
+        // the listen port
+        paramMap.put(
+                WeDPRCommonConfig.getParamKeyListenPort(),
+                String.valueOf(jupyterInfoDO.getJupyterSetting().getListenPort()));
+        // the jupyter binary
+        paramMap.put(
+                JupyterConfig.getParamKeyJupyterBinary(),
+                jupyterInfoDO.getJupyterSetting().getJupyterBinary());
+        // the jupyter config path
+        paramMap.put(
+                JupyterConfig.getParamKeyJupyterConfigPath(),
+                jupyterInfoDO.getJupyterSetting().getConfigPath());
+        // the notebook project directory
+        paramMap.put(
+                JupyterConfig.getParamKeyJupyterProjectPath(),
+                jupyterInfoDO.getJupyterSetting().getNoteBookPath());
+        return paramMap;
+    }
+}

wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/client/impl/JupyterClientImpl.java

@@ -0,0 +1,166 @@
+/*
+ * Copyright 2017-2025  [webank-wedpr]
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+package com.webank.wedpr.components.integration.jupyter.client.impl;
+
+import com.webank.wedpr.components.http.client.HttpClientImpl;
+import com.webank.wedpr.components.http.client.HttpClientPool;
+import com.webank.wedpr.components.integration.jupyter.client.JupyterClient;
+import com.webank.wedpr.components.integration.jupyter.core.JupyterConfig;
+import com.webank.wedpr.components.integration.jupyter.dao.JupyterInfoDO;
+import com.webank.wedpr.components.meta.sys.config.dao.SysConfigDO;
+import com.webank.wedpr.components.meta.sys.config.dao.SysConfigMapper;
+import com.webank.wedpr.components.uuid.generator.WeDPRUuidGenerator;
+import com.webank.wedpr.core.config.WeDPRCommonConfig;
+import com.webank.wedpr.core.protocol.task.ShellParameters;
+import com.webank.wedpr.core.protocol.task.TaskExecutionContext;
+import com.webank.wedpr.core.protocol.task.TaskResponse;
+import com.webank.wedpr.core.protocol.task.TaskType;
+import com.webank.wedpr.core.utils.BaseResponseFactory;
+import com.webank.wedpr.core.utils.ShellConstant;
+import com.webank.wedpr.core.utils.WeDPRException;
+import com.webank.wedpr.core.utils.WeDPRResponseFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JupyterClientImpl implements JupyterClient {
+    private static final Logger logger = LoggerFactory.getLogger(JupyterClientImpl.class);
+    private final SysConfigMapper sysConfigMapper;
+    private final BaseResponseFactory responseFactory = new WeDPRResponseFactory();
+
+    public JupyterClientImpl(SysConfigMapper sysConfigMapper) {
+        this.sysConfigMapper = sysConfigMapper;
+    }
+
+    protected String getCodeTemplate(String key, boolean mustExist) throws WeDPRException {
+        SysConfigDO result = this.sysConfigMapper.queryConfig(key);
+        if (result != null) {
+            return result.getConfigValue();
+        }
+        if (mustExist) {
+            throw new WeDPRException(
+                    "The code template for " + key + " not configure! Please configure firstly");
+        }
+        return null;
+    }
+
+    protected HttpClientImpl generateHttpClient(String url) {
+        String apiUri =
+                String.format(
+                        "%s/%s/%s",
+                        url,
+                        WeDPRCommonConfig.getWedprWorkerApiPath(),
+                        WeDPRCommonConfig.getWedprWorkerSubmitTaskMethod());
+        return new HttpClientImpl(
+                HttpClientPool.getUrl(apiUri),
+                JupyterConfig.getMaxTotalConnection(),
+                JupyterConfig.buildConfig(),
+                responseFactory);
+    }
+
+    protected TaskResponse submitTask(String method, JupyterInfoDO jupyterInfoDO, String code)
+            throws Exception {
+        logger.info("Submit code to: {}", jupyterInfoDO.getAccessEntry());
+        HttpClientImpl httpClient = generateHttpClient(jupyterInfoDO.getAccessEntry());
+        TaskExecutionContext taskRequest = new TaskExecutionContext();
+        taskRequest.setTaskID(WeDPRUuidGenerator.generateID());
+        taskRequest.setTaskType(TaskType.SHELL.getType());
+        ShellParameters shellParameters = new ShellParameters(code);
+        taskRequest.setTaskParameters(shellParameters.serialize());
+        taskRequest.setParameterMap(JupyterClient.generateParamMap(jupyterInfoDO));
+        TaskResponse response = (TaskResponse) httpClient.executePost(taskRequest);
+        if (response != null && response.statusOk()) {
+            logger.info(
+                    "submitTask for method {} success, taskID: {}, jupyterInfo: {}, code: {}, response: {}",
+                    method,
+                    taskRequest.getTaskID(),
+                    jupyterInfoDO.toString(),
+                    code,
+                    response);
+            return response;
+        }
+        logger.error(
+                "submitTask for method {} failed, taskID: {}, jupyterInfo: {}, code: {}, response: {}",
+                method,
+                taskRequest.getTaskID(),
+                jupyterInfoDO.toString(),
+                code,
+                (response == null ? "null" : response.toString()));
+        throw new WeDPRException(
+                "submitTask for "
+                        + method
+                        + " failed, taskID: "
+                        + taskRequest.getTaskID()
+                        + ", jupyterInfo: "
+                        + jupyterInfoDO.toString()
+                        + ", response: "
+                        + (response == null ? "null" : response.toString()));
+    }
+
+    @Override
+    public TaskResponse create(JupyterInfoDO jupyterInfo) throws Exception {
+        // create user and start jupyter
+        String code =
+                getCodeTemplate(WeDPRCommonConfig.getCodeTemplateKeyCreateUser(), true)
+                        + WeDPRCommonConfig.getShellCodeConnector()
+                        + getCodeTemplate(JupyterConfig.getCodeTemplateKeyStartJupyter(), true);
+        logger.info("create jupyter, info: {}, code: {}", jupyterInfo.toString(), code);
+        return submitTask("createJupyter", jupyterInfo, code);
+    }
+
+    @Override
+    public TaskResponse start(JupyterInfoDO jupyterInfo) throws Exception {
+        // Note: should check the existence firstly
+        List<String> commands = new ArrayList<>();
+        commands.add(ShellConstant.BASH_HEADER);
+        commands.add(ShellConstant.BASE_DIR_CMD);
+        // check the pid existence
+        String getJupyterPidCode =
+                getCodeTemplate(JupyterConfig.getCodeTemplateKeyGetJupyterPid(), true);
+        commands.add(String.format("jupyter_pid=$(%s)", getJupyterPidCode));
+        commands.add("if [ ! -z ${jupyter_pid} ];then ");
+        commands.add("echo \"${jupyter_pid}\"");
+        commands.add("else");
+        commands.add(getCodeTemplate(JupyterConfig.getCodeTemplateKeyStartJupyter(), true));
+        commands.add("sleep 1.5");
+        commands.add(String.format("echo $(%s)", getJupyterPidCode));
+        commands.add("fi");
+        String code = commands.stream().collect(Collectors.joining(System.lineSeparator()));
+        logger.info("start jupyter, info: {}, code: {}", jupyterInfo.toString(), code);
+        return submitTask("startJupyter", jupyterInfo, code);
+    }
+
+    @Override
+    public TaskResponse stop(JupyterInfoDO jupyterInfo) throws Exception {
+        List<String> commands = new ArrayList<>();
+        commands.add(ShellConstant.BASH_HEADER);
+        commands.add(ShellConstant.BASE_DIR_CMD);
+        // check the pid existence
+        String getJupyterPidCode =
+                getCodeTemplate(JupyterConfig.getCodeTemplateKeyGetJupyterPid(), true);
+        commands.add(String.format("jupyter_pid=$(%s)", getJupyterPidCode));
+        commands.add("[ ! -z ${jupyter_pid} ] && kill ${jupyter_pid} > /dev/null");
+        commands.add("[ -z ${jupyter_pid} ] && exit 0");
+        commands.add("sleep 2");
+        commands.add(String.format("jupyter_pid=$(%s)", getJupyterPidCode));
+        commands.add("[ ! -z ${jupyter_pid} ] && exit 1");
+        commands.add("[ -z ${jupyter_pid} ] && exit 0");
+        String code = commands.stream().collect(Collectors.joining(System.lineSeparator()));
+        logger.info("stop jupyter, info: {}, code: {}", jupyterInfo.toString(), code);
+        return submitTask("stopJupyter", jupyterInfo, code);
+    }
+}

wedpr-components/env-integration/jupyter/src/main/java/com/webank/wedpr/components/integration/jupyter/core/JupyterConfig.java

@@ -16,18 +16,62 @@
 package com.webank.wedpr.components.integration.jupyter.core;
 
 import com.webank.wedpr.core.config.WeDPRConfig;
+import com.webank.wedpr.core.utils.Common;
+import org.apache.http.client.config.RequestConfig;
 
 public class JupyterConfig {
 
     private static Integer MAX_JUPYTER_PER_HOST =
             WeDPRConfig.apply("wedpr.jupyter.max_count_per_host", 3);
-    private static String JUPYTER_HOST_CONFIGUATINON_KEY =
+    private static String JUPYTER_HOST_CONFIGURATION_KEY =
             WeDPRConfig.apply("wedpr.jupyter.host_configuration_key", "jupyter_entrypoints");
     private static String JUPYTER_ENTRYPOINT_SPLITTER = ";";
     private static String JUPYTER_MODULE = "jupyter-integration";
+    private static String DEFAULT_JUPYTER_BINARY =
+            WeDPRConfig.apply("wedpr.jupyter.binary", "jupyter-lab");
+    private static String DEFAULT_JUPYTER_CONFIG_PATH =
+            WeDPRConfig.apply(
+                    "wedpr.juypter.config.path", "/home/share/.jupyter/jupyter_lab_config.py");
+    private static String DEFAULT_JUPYTER_LISTEN_IP =
+            WeDPRConfig.apply("wedpr.jupyter.listen.ip", "0.0.0.0");
+    private static String DEFAULT_JUPYTER_PROJECT_DIR =
+            WeDPRConfig.apply("wedpr.jupyter.project.dir", "project");
+    private static Integer DEFAULT_JUPYTER_START_PORT =
+            WeDPRConfig.apply("wedpr.jupyter.jupyter.default.startPort", 14001);
 
-    public static String getJupyterHostConfiguatinonKey() {
-        return JUPYTER_HOST_CONFIGUATINON_KEY;
+    ///// the parameter map key configuration for jupyter
+    private static String PARAM_KEY_JUPYTER_CONFIG_PATH =
+            WeDPRConfig.apply("wedpr.jupyter.param.key.jupyter_config_path", "jupyter_config_path");
+    private static String PARAM_KEY_JUPYTER_PROJECT_PATH =
+            WeDPRConfig.apply(
+                    "wedpr.jupyter.param.key.jupyter_project_path", "jupyter_project_path");
+    private static String PARAM_KEY_JUPYTER_BINARY =
+            WeDPRConfig.apply("wedpr.jupyter.param.key.jupyter_binary", "jupyter_binary");
+    ///// the parameter map key configuration for jupyter
+
+    //// the key to store the jupyter code template
+    private static String CODE_TEMPLATE_KEY_START_JUPYTER =
+            WeDPRConfig.apply("wedpr.code.template.key.start_jupyter", "wedpr_start_jupyter");
+    private static String CODE_TEMPLATE_KEY_GET_JUPYTER_PID =
+            WeDPRConfig.apply(
+                    "wedpr.code.template.key.wedpr_get_jupyter_pid", "wedpr_get_jupyter_pid");
+    //// the key to store the jupyter code template
+
+    /// the jupyter worker client config ///
+    private static final Integer CONNECTION_REQUEST_TIME_OUT =
+            WeDPRConfig.apply("wedpr.jupyter.connect.request.timeout.ms", 10000);
+    private static final Integer CONNECTION_TIME_OUT =
+            WeDPRConfig.apply("wedpr.jupyter.connect.timeout.ms", 5000);
+    private static final Integer REQUEST_TIMEOUT =
+            WeDPRConfig.apply("wedpr.jupyter.request.timeout.ms", 60000);
+    private static final Integer MAX_TOTAL_CONNECTION =
+            WeDPRConfig.apply("wedpr.jupyter.max.total.connection", 5);
+    /// the jupyter worker client config ///
+
+    private static String DEFAULT_HOME_DIR = "/home";
+
+    public static String getJupyterHostConfigurationKey() {
+        return JUPYTER_HOST_CONFIGURATION_KEY;
     }
 
     public static Integer getMaxJupyterPerHost() {
@@ -41,4 +85,61 @@ public static String getJupyterEntrypointSplitter() {
     public static String getJupyterModule() {
         return JUPYTER_MODULE;
     }
+
+    public static String getDefaultJupyterBinary() {
+        return DEFAULT_JUPYTER_BINARY;
+    }
+
+    public static String getDefaultJupyterConfigPath() {
+        return DEFAULT_JUPYTER_CONFIG_PATH;
+    }
+
+    public static String getDefaultJupyterListenIp() {
+        return DEFAULT_JUPYTER_LISTEN_IP;
+    }
+
+    public static String getDefaultJupyterProjectDir() {
+        return DEFAULT_JUPYTER_PROJECT_DIR;
+    }
+
+    public static String getUserJupyterProjectPath(String userName) {
+        return Common.joinPath(
+                Common.joinPath(DEFAULT_HOME_DIR, userName), getDefaultJupyterProjectDir());
+    }
+
+    public static Integer getDefaultJupyterStartPort() {
+        return DEFAULT_JUPYTER_START_PORT;
+    }
+
+    public static String getParamKeyJupyterConfigPath() {
+        return PARAM_KEY_JUPYTER_CONFIG_PATH;
+    }
+
+    public static String getParamKeyJupyterProjectPath() {
+        return PARAM_KEY_JUPYTER_PROJECT_PATH;
+    }
+
+    public static String getParamKeyJupyterBinary() {
+        return PARAM_KEY_JUPYTER_BINARY;
+    }
+
+    public static String getCodeTemplateKeyStartJupyter() {
+        return CODE_TEMPLATE_KEY_START_JUPYTER;
+    }
+
+    public static String getCodeTemplateKeyGetJupyterPid() {
+        return CODE_TEMPLATE_KEY_GET_JUPYTER_PID;
+    }
+
+    public static RequestConfig buildConfig() {
+        return RequestConfig.custom()
+                .setConnectionRequestTimeout(CONNECTION_REQUEST_TIME_OUT)
+                .setConnectTimeout(CONNECTION_TIME_OUT)
+                .setSocketTimeout(REQUEST_TIMEOUT)
+                .build();
+    }
+
+    public static Integer getMaxTotalConnection() {
+        return MAX_TOTAL_CONNECTION;
+    }
 }