From 7bed6ab4bee0bbc49d8d660de190c6ecd1062277 Mon Sep 17 00:00:00 2001 From: lifeng68 Date: Mon, 25 May 2020 11:24:27 +0800 Subject: [PATCH] sync with openeuler stable 2.0.0 Signed-off-by: lifeng68 --- CMakeLists.txt | 2 +- iSulad.spec | 2 +- src/cmd/isula/base/create.c | 68 +--- src/cmd/isula/stream/exec.c | 86 +++-- src/constants.h | 2 + src/cutils/utils.c | 81 +++++ src/cutils/utils.h | 5 + src/cutils/utils_file.c | 177 +++++++++- src/cutils/utils_file.h | 4 + src/libisula.c | 31 +- src/runtime/isula/isula_rt_ops.c | 6 + src/services/cri/cni_network_plugin.cc | 320 ++++++++++-------- src/services/cri/cni_network_plugin.h | 65 ++-- src/services/cri/cri_helpers.cc | 4 +- src/services/cri/cri_helpers.h | 4 + src/services/cri/cri_runtime_service.cc | 7 +- src/services/cri/cri_runtime_service.h | 2 +- src/services/cri/cri_sandbox.cc | 35 +- src/services/cri/network_plugin.cc | 22 +- src/services/cri/network_plugin.h | 21 +- src/services/execution/execute/execution.c | 100 ++++++ src/services/execution/execute/execution.h | 1 + .../execution/manager/container_unix.c | 15 +- src/services/execution/manager/restore.c | 7 + src/services/execution/spec/specs.c | 2 +- src/services/execution/spec/specs_mount.c | 8 +- src/services/execution/spec/verify.c | 31 ++ src/websocket/service/exec_serve.cc | 4 +- .../execution/spec/selinux_label_mock_llt.cc | 2 +- test/test.sh | 2 +- 30 files changed, 781 insertions(+), 335 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d71ced1..f2fc473 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ include(cmake/set_build_flags.cmake) #set(CMAKE_C_COMPILER "gcc" CACHE PATH "c compiler") -set(GIT_COMMIT_HASH "d03048c03e5f83d8de4e6f164f354a9cb8316362") +set(GIT_COMMIT_HASH "237e7cb8572b23cde98864077c5ea89ea19bfec1") message("-- commit id: " ${GIT_COMMIT_HASH}) add_definitions(-DISULAD_GIT_COMMIT="${GIT_COMMIT_HASH}") diff --git a/iSulad.spec b/iSulad.spec index d8e7dde..1fcd245 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.0.0 -%global _release 20200406.224614.gitd03048c0 +%global _release 20200428.173159.git237e7cb8 %global is_systemd 1 %global debug_package %{nil} diff --git a/src/cmd/isula/base/create.c b/src/cmd/isula/base/create.c index 43951b8..5f11012 100644 --- a/src/cmd/isula/base/create.c +++ b/src/cmd/isula/base/create.c @@ -231,18 +231,27 @@ static int request_pack_custom_env(struct client_arguments *args, isula_containe { int ret = 0; char *pe = NULL; + char *new_env = NULL; if (args->custom_conf.env != NULL) { size_t i; for (i = 0; i < util_array_len((const char **)(args->custom_conf.env)); i++) { - if (util_array_append(&conf->env, args->custom_conf.env[i]) != 0) { - COMMAND_ERROR("Failed to append custom config env list"); + if (util_validate_env(args->custom_conf.env[i], &new_env) != 0) { + COMMAND_ERROR("Invalid environment %s", args->custom_conf.env[i]); ret = -1; goto out; } + if (new_env == NULL) { + continue; + } + if (util_array_append(&conf->env, new_env) != 0) { + COMMAND_ERROR("Failed to append custom config env list %s", new_env); + ret = -1; + goto out; + } + free(new_env); + new_env = NULL; } - util_free_array(args->custom_conf.env); - args->custom_conf.env = conf->env; /* make sure args->custom_conf.env point to valid memory. */ conf->env_len = util_array_len((const char **)(conf->env)); } @@ -255,9 +264,7 @@ static int request_pack_custom_env(struct client_arguments *args, isula_containe goto out; } } - args->custom_conf.env = conf->env; /* make sure args->custom_conf.env point to valid memory. */ conf->env_len = util_array_len((const char **)(conf->env)); - conf->accel = args->custom_conf.accel; conf->accel_len = util_array_len((const char **)(args->custom_conf.accel)); if (util_env_set_isulad_enable_plugins(&conf->env, &conf->env_len, ISULAD_ISULA_ADAPTER)) { @@ -269,52 +276,7 @@ static int request_pack_custom_env(struct client_arguments *args, isula_containe out: free(pe); - return ret; -} - -static int validate_env(const char *env, char **dst) -{ - int ret = 0; - char *value = NULL; - char **arr = util_string_split_multi(env, '='); - if (arr == NULL) { - ERROR("Failed to split env string"); - return -1; - } - if (strlen(arr[0]) == 0) { - ERROR("Invalid environment variable: %s", env); - ret = -1; - goto out; - } - - if (util_array_len((const char **)arr) > 1) { - *dst = util_strdup_s(env); - goto out; - } - - value = getenv(env); - if (value == NULL) { - *dst = util_strdup_s(env); - goto out; - } else { - int sret; - size_t len = strlen(env) + 1 + strlen(value) + 1; - *dst = (char *)util_common_calloc_s(len); - if (*dst == NULL) { - ERROR("Out of memory"); - ret = -1; - goto out; - } - sret = snprintf(*dst, len, "%s=%s", env, value); - if (sret < 0 || (size_t)sret >= len) { - ERROR("Failed to compose env string"); - ret = -1; - goto out; - } - } - -out: - util_free_array(arr); + free(new_env); return ret; } @@ -345,7 +307,7 @@ static int read_env_from_file(const char *path, size_t file_size, isula_containe continue; } buf[len - 1] = '\0'; - if (validate_env(buf, &new_env) != 0) { + if (util_validate_env(buf, &new_env) != 0) { ret = -1; goto out; } diff --git a/src/cmd/isula/stream/exec.c b/src/cmd/isula/stream/exec.c index 5494caf..65aa4f2 100644 --- a/src/cmd/isula/stream/exec.c +++ b/src/cmd/isula/stream/exec.c @@ -38,11 +38,64 @@ sem_t g_command_waitexit_sem; struct client_arguments g_cmd_exec_args = {}; +static int fill_exec_request(const struct client_arguments *args, const struct command_fifo_config *fifos, + struct isula_exec_request *request) +{ + int ret = 0; + size_t i = 0; + char *new_env = NULL; + + request->name = util_strdup_s(args->name); + request->suffix = util_strdup_s(args->exec_suffix); + request->tty = args->custom_conf.tty; + request->open_stdin = args->custom_conf.open_stdin; + request->attach_stdin = args->custom_conf.attach_stdin; + request->attach_stdout = args->custom_conf.attach_stdout; + request->attach_stderr = args->custom_conf.attach_stderr; + if (fifos != NULL) { + request->stdin = util_strdup_s(fifos->stdin_name); + request->stdout = util_strdup_s(fifos->stdout_name); + request->stderr = util_strdup_s(fifos->stderr_name); + } + + request->user = util_strdup_s(args->custom_conf.user); + + if (dup_array_of_strings((const char **)args->argv, args->argc, &(request->argv), (size_t *) & (request->argc)) != 0) { + ERROR("Failed to dup args"); + ret = -1; + goto out; + } + + /* environment variables */ + for (i = 0; i < util_array_len((const char **)(args->extra_env)); i++) { + if (util_validate_env(args->extra_env[i], &new_env) != 0) { + ERROR("Invalid environment %s", args->extra_env[i]); + ret = -1; + goto out; + } + if (new_env == NULL) { + continue; + } + if (util_array_append(&request->env, new_env) != 0) { + ERROR("Failed to append custom config env list %s", new_env); + ret = -1; + goto out; + } + request->env_len++; + free(new_env); + new_env = NULL; + } + +out: + free(new_env); + return ret; +} + static int client_exec(const struct client_arguments *args, const struct command_fifo_config *fifos, uint32_t *exit_code) { isula_connect_ops *ops = NULL; - struct isula_exec_request request = { 0 }; + struct isula_exec_request *request = NULL; struct isula_exec_response *response = NULL; client_connect_config_t config = { 0 }; int ret = 0; @@ -53,26 +106,18 @@ static int client_exec(const struct client_arguments *args, const struct command return ECOMMON; } - request.name = args->name; - request.suffix = args->exec_suffix; - request.tty = args->custom_conf.tty; - request.open_stdin = args->custom_conf.open_stdin; - request.attach_stdin = args->custom_conf.attach_stdin; - request.attach_stdout = args->custom_conf.attach_stdout; - request.attach_stderr = args->custom_conf.attach_stderr; - if (fifos != NULL) { - request.stdin = fifos->stdin_name; - request.stdout = fifos->stdout_name; - request.stderr = fifos->stderr_name; + request = util_common_calloc_s(sizeof(struct isula_exec_request)); + if (request == NULL) { + ERROR("Out of memory"); + ret = ECOMMON; + goto out; } - request.user = args->custom_conf.user; - request.argc = args->argc; - request.argv = (char **)args->argv; - - /* environment variables */ - request.env_len = util_array_len((const char **)(args->extra_env)); - request.env = args->extra_env; + if (fill_exec_request(args, fifos, request) != 0) { + ERROR("Failed to fill exec request"); + ret = ECOMMON; + goto out; + } ops = get_connect_client_ops(); if (ops == NULL || !ops->container.exec) { @@ -82,7 +127,7 @@ static int client_exec(const struct client_arguments *args, const struct command } config = get_connect_config(args); - ret = ops->container.exec(&request, response, &config); + ret = ops->container.exec(request, response, &config); if (ret) { client_print_error(response->cc, response->server_errono, response->errmsg); ret = ECOMMON; @@ -94,6 +139,7 @@ out: } isula_exec_response_free(response); + isula_exec_request_free(request); return ret; } diff --git a/src/constants.h b/src/constants.h index c3930f1..922f7aa 100644 --- a/src/constants.h +++ b/src/constants.h @@ -48,6 +48,8 @@ #define NETWORK_MOUNT_FILE_MODE 0644 +#define ETC_FILE_MODE 0755 + #define ISULAD_CONFIG "/etc/isulad" #define ISULAD_DAEMON_JSON_CONF_FILE ISULAD_CONFIG "/daemon.json" diff --git a/src/cutils/utils.c b/src/cutils/utils.c index 671b2bb..b5565cd 100644 --- a/src/cutils/utils.c +++ b/src/cutils/utils.c @@ -1294,6 +1294,41 @@ out: return bret; } +bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *pattern) +{ + FILE *fp = NULL; + char *line = NULL; + char *mountpoint = NULL; + size_t length = 0; + bool bret = true; + int nret = 0; + + fp = util_fopen("/proc/self/mountinfo", "r"); + if (fp == NULL) { + ERROR("Failed opening /proc/self/mountinfo"); + return false; + } + + while (getline(&line, &length, fp) != -1) { + mountpoint = get_mtpoint(line); + if (mountpoint == NULL) { + INFO("Error reading mountinfo: bad line '%s'", line); + continue; + } + nret = cb(mountpoint, pattern); + free(mountpoint); + if (nret != 0) { + bret = false; + goto out; + } + } + +out: + fclose(fp); + free(line); + return bret; +} + static int set_echo_back(bool echo_back) { struct termios old, new; @@ -1474,3 +1509,49 @@ void add_array_kv(char **array, size_t total, size_t *pos, const char *k, const add_array_elem(array, total, pos, v); } +int util_validate_env(const char *env, char **dst) +{ + int ret = 0; + char *value = NULL; + + char **arr = util_string_split_multi(env, '='); + if (arr == NULL) { + ERROR("Failed to split env string"); + return -1; + } + if (strlen(arr[0]) == 0) { + ERROR("Invalid environment variable: %s", env); + ret = -1; + goto out; + } + + if (util_array_len((const char **)arr) > 1) { + *dst = util_strdup_s(env); + goto out; + } + + value = getenv(env); + if (value == NULL) { + *dst = NULL; + goto out; + } else { + int sret; + size_t len = strlen(env) + 1 + strlen(value) + 1; + *dst = (char *)util_common_calloc_s(len); + if (*dst == NULL) { + ERROR("Out of memory"); + ret = -1; + goto out; + } + sret = snprintf(*dst, len, "%s=%s", env, value); + if (sret < 0 || (size_t)sret >= len) { + ERROR("Failed to compose env string"); + ret = -1; + goto out; + } + } + +out: + util_free_array(arr); + return ret; +} diff --git a/src/cutils/utils.h b/src/cutils/utils.h index 12e1b8e..580ece5 100644 --- a/src/cutils/utils.h +++ b/src/cutils/utils.h @@ -428,6 +428,11 @@ void add_array_elem(char **array, size_t total, size_t *pos, const char *elem); void add_array_kv(char **array, size_t total, size_t *pos, const char *k, const char *v); +typedef int (*mount_info_call_back_t)(const char *, const char *); +bool util_deal_with_mount_info(mount_info_call_back_t cb, const char *); + +int util_validate_env(const char *env, char **dst); + #ifdef __cplusplus } #endif diff --git a/src/cutils/utils_file.c b/src/cutils/utils_file.c index bd066ee..484d2ee 100644 --- a/src/cutils/utils_file.c +++ b/src/cutils/utils_file.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #include @@ -182,6 +184,39 @@ static bool check_dir_valid(const char *dirpath, int recursive_depth, int *failu return true; } +static int mark_file_mutable(const char *fname) +{ + int ret = 0; + int fd = -EBADF; + int attributes = 0; + + fd = open(fname, O_RDONLY | O_CLOEXEC | O_NONBLOCK); + if (fd < 0) { + ERROR("Failed to open file to modify flags:%s, %s", fname, strerror(errno)); + return -1; + } + + if (ioctl(fd, FS_IOC_GETFLAGS, &attributes) < 0) { + ERROR("Failed to retrieve file flags"); + ret = -1; + goto out; + } + + attributes &= ~FS_IMMUTABLE_FL; + + if (ioctl(fd, FS_IOC_SETFLAGS, &attributes) < 0) { + ERROR("Failed to set file flags"); + ret = -1; + goto out; + } + +out: + if (fd >= 0) { + close(fd); + } + return ret; +} + static int recursive_rmdir_next_depth(struct stat fstat, const char *fname, int recursive_depth, int *saved_errno, int failure) { @@ -191,11 +226,19 @@ static int recursive_rmdir_next_depth(struct stat fstat, const char *fname, int } } else { if (unlink(fname) < 0) { - ERROR("Failed to delete %s: %s", fname, strerror(errno)); + ERROR("Failed to delete \"%s\": %s", fname, strerror(errno)); if (*saved_errno == 0) { *saved_errno = errno; } - failure = 1; + + if (mark_file_mutable(fname) != 0) { + ERROR("Failed to mark file mutable"); + } + + if (unlink(fname) < 0) { + ERROR("Failed to delete \"%s\": %s", fname, strerror(errno)); + failure = 1; + } } } @@ -934,3 +977,133 @@ free_out: return ret; } +char *util_path_base(const char *path) +{ + char *dir = NULL; + int len = 0; + int i = 0; + + if (path == NULL) { + ERROR("invalid NULL param"); + return NULL; + } + + len = (int)strlen(path); + if (len == 0) { + return util_strdup_s("."); + } + + dir = util_strdup_s(path); + + // strip last slashes + for (i = len - 1; i >= 0; i--) { + if (dir[i] != '/') { + break; + } + dir[i] = '\0'; + } + + len = (int)strlen(dir); + if (len == 0) { + free(dir); + return util_strdup_s("/"); + } + + for (i = len - 1; i >= 0; i--) { + if (dir[i] == '/') { + break; + } + } + + if (i < 0) { + return dir; + } + + char *result = util_strdup_s(&dir[i + 1]); + free(dir); + return result; +} + +static char *get_random_tmp_file(const char *fname) +{ +#define RANDOM_TMP_PATH 10 + int nret = 0; + char *result = NULL; + char *base = NULL; + char *dir = NULL; + char rpath[PATH_MAX] = { 0x00 }; + char random_tmp[RANDOM_TMP_PATH + 1] = { 0x00 }; + + base = util_path_base(fname); + if (base == NULL) { + ERROR("Failed to get base of %s", fname); + goto out; + } + + dir = util_path_dir(fname); + if (dir == NULL) { + ERROR("Failed to get dir of %s", fname); + goto out; + } + + if (util_generate_random_str(random_tmp, (size_t)RANDOM_TMP_PATH)) { + ERROR("Failed to generate random str for random path"); + goto out; + } + + nret = snprintf(rpath, PATH_MAX, ".tmp-%s-%s", base, random_tmp); + if (nret < 0 || nret >= PATH_MAX) { + ERROR("Failed to generate tmp base file"); + goto out; + } + + result = util_path_join(dir, rpath); + +out: + free(base); + free(dir); + return result; +} + +int util_atomic_write_file(const char *fname, const char *content, size_t content_len, mode_t mode) +{ + int ret = 0; + char *tmp_file = NULL; + char rpath[PATH_MAX] = { 0x00 }; + + if (fname == NULL) { + return -1; + } + if (content == NULL || content_len == 0) { + return 0; + } + + if (cleanpath(fname, rpath, sizeof(rpath)) == NULL) { + return -1; + } + + tmp_file = get_random_tmp_file(fname); + if (tmp_file == NULL) { + ERROR("Failed to get tmp file for %s", fname); + ret = -1; + goto free_out; + } + + ret = util_write_file(tmp_file, content, content_len, mode); + if (ret != 0) { + ERROR("Failed to write content to tmp file for %s", tmp_file); + ret = -1; + goto free_out; + } + + ret = rename(tmp_file, rpath); + if (ret != 0) { + ERROR("Failed to rename old file %s to target %s", tmp_file, rpath); + ret = -1; + goto free_out; + } + +free_out: + free(tmp_file); + return ret; +} diff --git a/src/cutils/utils_file.h b/src/cutils/utils_file.h index a109dc5..12faf6e 100644 --- a/src/cutils/utils_file.h +++ b/src/cutils/utils_file.h @@ -75,6 +75,10 @@ char *verify_file_and_get_real_path(const char *file); int util_copy_file(const char *src_file, const char *dst_file, mode_t mode); +char *util_path_base(const char *path); + +int util_atomic_write_file(const char *fname, const char *content, size_t content_len, mode_t mode); + #ifdef __cplusplus } #endif diff --git a/src/libisula.c b/src/libisula.c index d13aab8..1f23ae3 100644 --- a/src/libisula.c +++ b/src/libisula.c @@ -667,26 +667,17 @@ void isula_exec_request_free(struct isula_exec_request *request) free(request->stderr); request->stderr = NULL; - if (request->argc && request->argv != NULL) { - int i; - for (i = 0; i < request->argc; i++) { - free(request->argv[i]); - request->argv[i] = NULL; - } - free(request->argv); - request->argv = NULL; - request->argc = 0; - } - if (request->env_len && request->env != NULL) { - size_t j; - for (j = 0; j < request->env_len; j++) { - free(request->env[j]); - request->env[j] = NULL; - } - free(request->env); - request->env = NULL; - request->env_len = 0; - } + free(request->user); + request->user = NULL; + + util_free_array_by_len(request->argv, request->argc); + request->argv = NULL; + request->argc = 0; + + util_free_array_by_len(request->env, request->env_len); + request->env = NULL; + request->env_len = 0; + free(request); } diff --git a/src/runtime/isula/isula_rt_ops.c b/src/runtime/isula/isula_rt_ops.c index 44bc0dc..510482b 100644 --- a/src/runtime/isula/isula_rt_ops.c +++ b/src/runtime/isula/isula_rt_ops.c @@ -606,6 +606,12 @@ static int shim_create(bool fg, const char *id, const char *workdir, goto realexec; } + // clear NOTIFY_SOCKET from the env to adapt runc create + if (unsetenv("NOTIFY_SOCKET") != 0) { + (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno)); + exit(EXIT_FAILURE); + } + pid = fork(); if (pid < 0) { (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno)); diff --git a/src/services/cri/cni_network_plugin.cc b/src/services/cri/cni_network_plugin.cc index ea90eaf..e8611de 100644 --- a/src/services/cri/cni_network_plugin.cc +++ b/src/services/cri/cni_network_plugin.cc @@ -19,21 +19,18 @@ #include #include #include +#include +#include "cxxutils.h" #include "log.h" #include "utils.h" #include "cri_helpers.h" namespace Network { -static std::string VendorCNIDir(const std::string &prefix, const std::string &pluginType) -{ - return prefix + "/opt/" + pluginType + "/bin"; -} - -static std::unique_ptr GetLoNetwork(const std::string &binDir, const std::string &vendorDirPrefix) +static std::unique_ptr GetLoNetwork(std::vector binDirs, const std::string &vendorDirPrefix) { const std::string loNetConfListJson { "{\"cniVersion\": \"0.3.0\", \"name\": \"cni-loopback\"," - "\"plugins\":[{\"type\": \"loopback\" }]}" }; + "\"plugins\":[{\"type\": \"loopback\" }]}" }; char *cerr { nullptr }; struct cni_network_list_conf *loConf { @@ -62,8 +59,7 @@ static std::unique_ptr GetLoNetwork(const std::string &binDir, const return nullptr; } - result->InsertPath(VendorCNIDir(vendorDirPrefix, "loopback")); - result->InsertPath(binDir); + result->SetPaths(binDirs); return result; } @@ -92,8 +88,9 @@ void ProbeNetworkPlugins(const std::string &pluginDir, const std::string &binDir std::vector> *plugins) { const std::string useBinDir = binDir.empty() ? DEFAULT_CNI_DIR : binDir; - auto plugin = std::make_shared(useBinDir, pluginDir); - plugin->SetLoNetwork(GetLoNetwork(useBinDir, "")); + std::vector binDirs = CXXUtils::Split(useBinDir, ','); + auto plugin = std::make_shared(binDirs, pluginDir); + plugin->SetLoNetwork(GetLoNetwork(binDirs, "")); plugins->push_back(plugin); } @@ -104,17 +101,43 @@ void CniNetworkPlugin::SetLoNetwork(std::unique_ptr lo) } } -CniNetworkPlugin::CniNetworkPlugin(const std::string &binDir, const std::string &pluginDir, - const std::string &vendorCNIDirPrefix) - : m_pluginDir(pluginDir) - , m_vendorCNIDirPrefix(vendorCNIDirPrefix) - , m_binDir(binDir) +void CniNetworkPlugin::SetDefaultNetwork(std::unique_ptr network, + std::vector &binDirs, Errors &err) +{ + if (network == nullptr) { + return; + } + WLockNetworkMap(err); + if (err.NotEmpty()) { + ERROR("%s", err.GetCMessage()); + return; + } + m_defaultNetwork = std::move(network); + m_defaultNetwork->SetPaths(binDirs); + + DEBUG("Update new cni network: \"%s\"", m_defaultNetwork->GetName().c_str()); + + UnlockNetworkMap(err); + if (err.NotEmpty()) { + ERROR("%s", err.GetCMessage()); + } +} + +CniNetworkPlugin::CniNetworkPlugin(std::vector &binDirs, const std::string &confDir, + const std::string &podCidr) + : m_confDir(confDir) + , m_binDirs(binDirs) + , m_podCidr(podCidr) + , m_needFinish(false) { } CniNetworkPlugin::~CniNetworkPlugin() { - m_networks.clear(); + m_needFinish = true; + if (m_syncThread.joinable()) { + m_syncThread.join(); + } } void CniNetworkPlugin::PlatformInit(Errors &error) @@ -232,65 +255,16 @@ out: return ret; } -int CniNetworkPlugin::InsertNewNetwork(struct cni_network_list_conf *n_list, - std::map> &newNets, - const std::string &binDir, const std::string &vendorCNIDirPrefix, Errors &err) -{ - std::string confType { "" }; - - if (n_list == nullptr) { - err.Errorf("Invalid arguments"); - return -1; - } - if (n_list->first_plugin_type != nullptr) { - confType = n_list->first_plugin_type; - } - - std::string tpath = VendorCNIDir(vendorCNIDirPrefix, confType); - if (tpath.empty()) { - free_cni_network_list_conf(n_list); - err.SetError("Out of memory"); - return -1; - } - std::unique_ptr network(new (std::nothrow) CNINetwork(n_list->name, n_list)); - if (network == nullptr) { - free_cni_network_list_conf(n_list); - err.SetError("Out of memory"); - return -1; - } - network->InsertPath(tpath); - network->InsertPath(binDir); - - std::string n_key(network->GetName()); - newNets.insert(std::pair>(n_key, std::move(network))); - DEBUG("---parse cni network: %s finish ----", n_key.c_str()); - - return 0; -} - -void CniNetworkPlugin::ResetCNINetwork(std::map> &newNets, Errors &err) -{ - std::string pluginNames { "map[" }; - - m_networks.clear(); - for (auto iter = newNets.begin(); iter != newNets.end(); iter++) { - m_networks[iter->first] = std::move(iter->second); - pluginNames += (iter->first + " "); - } - INFO("Loaded cni plugins successfully, all plugins: %s]", pluginNames.substr(0, pluginNames.length() - 1).c_str()); -} - -void CniNetworkPlugin::GetCNINetwork(const std::string &pluginDir, const std::string &binDir, - const std::string &vendorCNIDirPrefix, Errors &err) +void CniNetworkPlugin::GetDefaultCNINetwork(const std::string &confDir, std::vector &binDirs, Errors &err) { std::vector files; - std::set allPanes; - std::map> newNets; + bool found = false; - if (GetCNIConfFiles(pluginDir, files, err) != 0) { + if (GetCNIConfFiles(confDir, files, err) != 0) { goto free_out; } + sort(files.begin(), files.end()); for (auto elem : files) { struct cni_network_list_conf *n_list = nullptr; @@ -305,23 +279,15 @@ void CniNetworkPlugin::GetCNINetwork(const std::string &pluginDir, const std::st continue; } - if (InsertConfNameToAllPanes(n_list, allPanes, err) != 0) { - goto free_out; - } - - if (InsertNewNetwork(n_list, newNets, binDir, vendorCNIDirPrefix, err) != 0) { - goto free_out; - } + SetDefaultNetwork(std::unique_ptr(new (std::nothrow) CNINetwork(n_list->name, n_list)), binDirs, err); + found = true; + break; } - - if (newNets.size() == 0) { - err.Errorf("No valid networks found in %s", pluginDir.c_str()); - goto free_out; + if (!found) { + err.Errorf("No valid networks found in %s", confDir.c_str()); } - ResetCNINetwork(newNets, err); free_out: - newNets.clear(); return; } @@ -332,9 +298,9 @@ void CniNetworkPlugin::CheckInitialized(Errors &err) ERROR("%s", err.GetCMessage()); return; } - size_t len = m_networks.size(); + bool inited = (m_defaultNetwork != nullptr); UnlockNetworkMap(err); - if (len == 0) { + if (!inited) { err.AppendError("cni config uninitialized"); } } @@ -342,19 +308,10 @@ void CniNetworkPlugin::CheckInitialized(Errors &err) void CniNetworkPlugin::SyncNetworkConfig() { Errors err; - WLockNetworkMap(err); - if (err.NotEmpty()) { - ERROR("%s", err.GetCMessage()); - return; - } - GetCNINetwork(m_pluginDir, m_binDir, m_vendorCNIDirPrefix, err); + GetDefaultCNINetwork(m_confDir, m_binDirs, err); if (err.NotEmpty()) { WARN("Unable to update cni config: %s", err.GetCMessage()); } - UnlockNetworkMap(err); - if (err.NotEmpty()) { - ERROR("%s", err.GetCMessage()); - } } void CniNetworkPlugin::Init(CRIRuntimeServiceImpl *criImpl, const std::string &hairpinMode, @@ -375,6 +332,10 @@ void CniNetworkPlugin::Init(CRIRuntimeServiceImpl *criImpl, const std::string &h m_criImpl = criImpl; SyncNetworkConfig(); + // start a thread to sync network config from confDir periodically to detect network config updates in every 5 seconds + m_syncThread = std::thread([&]() { + UpdateDefaultNetwork(); + }); return; } @@ -385,14 +346,13 @@ const std::string &CniNetworkPlugin::Name() const void CniNetworkPlugin::Status(Errors &err) { - SyncNetworkConfig(); - CheckInitialized(err); } -void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, +void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &id, - const std::map &annotations, Errors &err) + const std::map &annotations, + const std::map &options, Errors &err) { CheckInitialized(err); if (err.NotEmpty()) { @@ -403,16 +363,10 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, ERROR("CNI failed to retrieve network namespace path: %s", err.GetCMessage()); return; } - auto iter = annotations.find(CRIHelpers::Constants::POD_CHECKPOINT_KEY); - std::string jsonCheckpoint { "" }; - if (iter != annotations.end()) { - jsonCheckpoint = iter->second; - } - DEBUG("add checkpoint: ", jsonCheckpoint.c_str()); struct result *preResult = nullptr; if (m_loNetwork != nullptr) { - AddToNetwork(m_loNetwork.get(), jsonCheckpoint, name, ns, interfaceName, id, netnsPath, &preResult, err); + AddToNetwork(m_loNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, options, &preResult, err); free_result(preResult); preResult = nullptr; if (err.NotEmpty()) { @@ -426,26 +380,20 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, ERROR("%s", err.GetCMessage()); return; } - auto netIter = m_networks.find(networkPlane); - if (netIter == m_networks.end()) { - ERROR("Can't find cni plugin for network plane %s.", networkPlane.c_str()); - err.Errorf("Can't find cni plugin for network plane %s.", networkPlane.c_str()); - goto unlock_out; - } - AddToNetwork((netIter->second).get(), jsonCheckpoint, name, ns, interfaceName, id, netnsPath, &preResult, err); + AddToNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, options, &preResult, err); + free_result(preResult); preResult = nullptr; if (err.NotEmpty()) { ERROR("Error while adding to cni network: %s", err.GetCMessage()); } -unlock_out: UnlockNetworkMap(err); } -void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &id, +void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, + const std::string &id, const std::map &annotations, Errors &err) { CheckInitialized(err); @@ -459,27 +407,14 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam err.Clear(); } - auto iter = annotations.find(CRIHelpers::Constants::POD_CHECKPOINT_KEY); - std::string jsonCheckpoint = ""; - if (iter != annotations.end()) { - jsonCheckpoint = iter->second; - } - DEBUG("delete checkpoint: ", jsonCheckpoint.c_str()); RLockNetworkMap(err); if (err.NotEmpty()) { ERROR("%s", err.GetCMessage()); return; } - auto netIter = m_networks.find(networkPlane); - if (netIter == m_networks.end()) { - ERROR("Can't find cni plugin for network plane %s.", networkPlane.c_str()); - err.Errorf("Can't find cni plugin for network plane %s.", networkPlane.c_str()); - goto unlock_out; - } - DeleteFromNetwork((netIter->second).get(), jsonCheckpoint, name, ns, interfaceName, id, netnsPath, err); + DeleteFromNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, err); -unlock_out: UnlockNetworkMap(err); } @@ -488,9 +423,40 @@ std::map *CniNetworkPlugin::Capabilities() return m_noop.Capabilities(); } +void CniNetworkPlugin::SetPodCidr(const std::string &podCidr) +{ + Errors err; + + WLockNetworkMap(err); + if (err.NotEmpty()) { + ERROR("%s", err.GetCMessage()); + return; + } + + if (!m_podCidr.empty()) { + WARN("Ignoring subsequent pod CIDR update to %s", podCidr.c_str()); + goto unlock_out; + } + + m_podCidr = podCidr; + +unlock_out: + UnlockNetworkMap(err); +} + void CniNetworkPlugin::Event(const std::string &name, std::map &details) { - m_noop.Event(name, details); + if (name != CRIHelpers::Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE) { + return; + } + + auto iter = details.find(CRIHelpers::Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR); + if (iter == details.end()) { + WARN("%s event didn't contain pod CIDR", CRIHelpers::Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE.c_str()); + return; + } + + SetPodCidr(iter->second); } void CniNetworkPlugin::GetPodNetworkStatus(const std::string &ns, const std::string &name, @@ -526,9 +492,11 @@ out: INFO("get_pod_network_status: %s", podSandboxID.c_str()); } -void CniNetworkPlugin::AddToNetwork(CNINetwork *snetwork, const std::string &jsonCheckpoint, const std::string &podName, +void CniNetworkPlugin::AddToNetwork(CNINetwork *snetwork, const std::string &podName, const std::string &podNamespace, const std::string &interfaceName, const std::string &podSandboxID, const std::string &podNetnsPath, + const std::map &annotations, + const std::map &options, struct result **presult, Errors &err) { struct runtime_conf *rc { @@ -540,7 +508,8 @@ void CniNetworkPlugin::AddToNetwork(CNINetwork *snetwork, const std::string &jso ERROR("Invalid arguments"); return; } - BuildCNIRuntimeConf(podName, jsonCheckpoint, podNamespace, interfaceName, podSandboxID, podNetnsPath, &rc, err); + + BuildCNIRuntimeConf(podName, podNamespace, interfaceName, podSandboxID, podNetnsPath, annotations, options, &rc, err); if (err.NotEmpty()) { ERROR("Error adding network when building cni runtime conf: %s", err.GetCMessage()); return; @@ -566,10 +535,12 @@ void CniNetworkPlugin::AddToNetwork(CNINetwork *snetwork, const std::string &jso free(serr); } -void CniNetworkPlugin::DeleteFromNetwork(CNINetwork *network, const std::string &jsonCheckpoint, +void CniNetworkPlugin::DeleteFromNetwork(CNINetwork *network, const std::string &podName, const std::string &podNamespace, const std::string &interfaceName, const std::string &podSandboxID, - const std::string &podNetnsPath, Errors &err) + const std::string &podNetnsPath, + const std::map &annotations, + Errors &err) { struct runtime_conf *rc { nullptr @@ -580,7 +551,8 @@ void CniNetworkPlugin::DeleteFromNetwork(CNINetwork *network, const std::string ERROR("Invalid arguments"); return; } - BuildCNIRuntimeConf(podName, jsonCheckpoint, podNamespace, interfaceName, podSandboxID, podNetnsPath, &rc, err); + std::map options; + BuildCNIRuntimeConf(podName, podNamespace, interfaceName, podSandboxID, podNetnsPath, annotations, options, &rc, err); if (err.NotEmpty()) { ERROR("Error deleting network when building cni runtime conf: %s", err.GetCMessage()); return; @@ -606,20 +578,25 @@ void CniNetworkPlugin::DeleteFromNetwork(CNINetwork *network, const std::string free(serr); } -void CniNetworkPlugin::BuildCNIRuntimeConf(const std::string &podName, const std::string &jsonCheckpoint, - const std::string &podNs, const std::string &interfaceName, - const std::string &podSandboxID, const std::string &podNetnsPath, - struct runtime_conf **cni_rc, Errors &err) +static void PrepareRuntimeConf(const std::string &podName, + const std::string &podNs, const std::string &interfaceName, + const std::string &podSandboxID, const std::string &podNetnsPath, + const std::map &options, + struct runtime_conf **cni_rc, Errors &err) { - std::vector portMappings; - INFO("Got netns path %s", podNetnsPath.c_str()); - INFO("Using podns path %s", podNs.c_str()); - + const size_t defaultLen = 5; if (cni_rc == nullptr) { err.Errorf("Invalid arguments"); ERROR("Invalid arguments"); return; } + + auto iter = options.find("UID"); + std::string podUID {""}; + if (iter != options.end()) { + podUID = iter->second; + } + struct runtime_conf *rt = (struct runtime_conf *)util_common_calloc_s(sizeof(struct runtime_conf)); if (rt == nullptr) { ERROR("Out of memory"); @@ -631,13 +608,13 @@ void CniNetworkPlugin::BuildCNIRuntimeConf(const std::string &podName, const std rt->netns = util_strdup_s(podNetnsPath.c_str()); rt->ifname = util_strdup_s(interfaceName.c_str()); - rt->args = (char *(*)[2])util_common_calloc_s(sizeof(char *) * 2 * 4); + rt->args = (char *(*)[2])util_common_calloc_s(sizeof(char *) * 2 * defaultLen); if (rt->args == nullptr) { ERROR("Out of memory"); err.SetError("Out of memory"); goto free_out; } - rt->args_len = 4; + rt->args_len = defaultLen; rt->args[0][0] = util_strdup_s("IgnoreUnknown"); rt->args[0][1] = util_strdup_s("1"); rt->args[1][0] = util_strdup_s("K8S_POD_NAMESPACE"); @@ -646,6 +623,39 @@ void CniNetworkPlugin::BuildCNIRuntimeConf(const std::string &podName, const std rt->args[2][1] = util_strdup_s(podName.c_str()); rt->args[3][0] = util_strdup_s("K8S_POD_INFRA_CONTAINER_ID"); rt->args[3][1] = util_strdup_s(podSandboxID.c_str()); + rt->args[4][0] = util_strdup_s("K8S_POD_UID"); + rt->args[4][1] = util_strdup_s(podUID.c_str()); + + *cni_rc = rt; + return; +free_out: + free_runtime_conf(rt); +} + +void CniNetworkPlugin::BuildCNIRuntimeConf(const std::string &podName, + const std::string &podNs, const std::string &interfaceName, + const std::string &podSandboxID, const std::string &podNetnsPath, + const std::map &annotations, + const std::map &options, + struct runtime_conf **cni_rc, Errors &err) +{ + PrepareRuntimeConf(podName, podNs, interfaceName, podSandboxID, podNetnsPath, options, cni_rc, err); + if (err.NotEmpty()) { + return; + } + struct runtime_conf *rt = *cni_rc; + *cni_rc = nullptr; + + auto iter = annotations.find(CRIHelpers::Constants::POD_CHECKPOINT_KEY); + std::string jsonCheckpoint { "" }; + if (iter != annotations.end()) { + jsonCheckpoint = iter->second; + } + DEBUG("add checkpoint: ", jsonCheckpoint.c_str()); + + std::vector portMappings; + INFO("Got netns path %s", podNetnsPath.c_str()); + INFO("Using podns path %s", podNs.c_str()); if (!jsonCheckpoint.empty()) { cri::PodSandboxCheckpoint checkpoint; @@ -723,4 +733,22 @@ void CniNetworkPlugin::UnlockNetworkMap(Errors &error) } } +void CniNetworkPlugin::UpdateDefaultNetwork() +{ + const int defaultSyncConfigCnt = 5; + const int defaultSyncConfigPeriod = 1000; + + pthread_setname_np(pthread_self(), "CNIUpdater"); + + while (true) { + for (int i = 0; i < defaultSyncConfigCnt; i++) { + std::this_thread::sleep_for(std::chrono::milliseconds(defaultSyncConfigPeriod)); + if (m_needFinish) { + return; + } + } + SyncNetworkConfig(); + } +} + } // namespace Network diff --git a/src/services/cri/cni_network_plugin.h b/src/services/cri/cni_network_plugin.h index 601e04e..8be6d5a 100644 --- a/src/services/cri/cni_network_plugin.h +++ b/src/services/cri/cni_network_plugin.h @@ -20,6 +20,8 @@ #include #include #include +#include + #include #include "network_plugin.h" @@ -48,9 +50,9 @@ public: { m_name = name; } - void InsertPath(const std::string &path) + void SetPaths(std::vector &binDirs) { - m_path.push_back(path); + m_path = binDirs; } std::string GetNetworkConfigJsonStr() { @@ -64,6 +66,13 @@ public: { return m_networkConfig->first_plugin_name ? m_networkConfig->first_plugin_name : ""; } + struct cni_network_list_conf *UpdateCNIConfList(struct cni_network_list_conf *newConf) + { + struct cni_network_list_conf *result = m_networkConfig; + m_networkConfig = newConf; + return result; + } + char **GetPaths(Errors &err); private: @@ -76,8 +85,8 @@ private: class CniNetworkPlugin : public NetworkPlugin { public: - CniNetworkPlugin(const std::string &binDir, const std::string &pluginDir, - const std::string &vendorCNIDirPrefix = ""); + CniNetworkPlugin(std::vector &binDirs, const std::string &confDir, + const std::string &podCidr = ""); virtual ~CniNetworkPlugin(); @@ -90,12 +99,13 @@ public: std::map *Capabilities() override; - void SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, + void SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, - const std::map &annotations, Errors &error) override; + const std::map &annotations, + const std::map &options, Errors &error) override; - void TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &podSandboxID, + void TearDownPod(const std::string &ns, const std::string &name, + const std::string &networkPlane, const std::string &podSandboxID, const std::map &annotations, Errors &error) override; void GetPodNetworkStatus(const std::string &ns, const std::string &name, const std::string &interfaceName, @@ -109,47 +119,56 @@ private: virtual void PlatformInit(Errors &error); virtual void SyncNetworkConfig(); - virtual void GetCNINetwork(const std::string &pluginDir, const std::string &binDir, - const std::string &vendorCNIDirPrefix, Errors &error); + virtual void GetDefaultCNINetwork(const std::string &pluginDir, std::vector &binDirs, Errors &error); virtual void CheckInitialized(Errors &error); - virtual void AddToNetwork(CNINetwork *network, const std::string &jsonCheckpoint, const std::string &podName, + virtual void AddToNetwork(CNINetwork *network, const std::string &podName, const std::string &podNamespace, const std::string &interfaceName, - const std::string &podSandboxID, const std::string &podNetnsPath, struct result **presult, - Errors &error); + const std::string &podSandboxID, const std::string &podNetnsPath, + const std::map &annotations, + const std::map &options, + struct result **presult, Errors &error); - virtual void DeleteFromNetwork(CNINetwork *network, const std::string &jsonCheckpoint, const std::string &podName, + virtual void DeleteFromNetwork(CNINetwork *network, const std::string &podName, const std::string &podNamespace, const std::string &interfaceName, - const std::string &podSandboxID, const std::string &podNetnsPath, Errors &error); + const std::string &podSandboxID, const std::string &podNetnsPath, + const std::map &annotations, + Errors &error); - virtual void BuildCNIRuntimeConf(const std::string &podName, const std::string &jsonCheckpoint, + virtual void BuildCNIRuntimeConf(const std::string &podName, const std::string &podNs, const std::string &interfaceName, const std::string &podSandboxID, const std::string &podNetnsPath, + const std::map &annotations, + const std::map &options, struct runtime_conf **cni_rc, Errors &error); private: void RLockNetworkMap(Errors &error); void WLockNetworkMap(Errors &error); void UnlockNetworkMap(Errors &error); + void SetDefaultNetwork(std::unique_ptr network, std::vector &binDirs, Errors &err); + void SetPodCidr(const std::string &podCidr); int GetCNIConfFiles(const std::string &pluginDir, std::vector &vect_files, Errors &err); int LoadCNIConfigFileList(const std::string &elem, struct cni_network_list_conf **n_list); int InsertConfNameToAllPanes(struct cni_network_list_conf *n_list, std::set &allPanes, Errors &err); - int InsertNewNetwork(struct cni_network_list_conf *n_list, - std::map> &newNets, const std::string &binDir, - const std::string &vendorCNIDirPrefix, Errors &err); void ResetCNINetwork(std::map> &newNets, Errors &err); + void UpdateDefaultNetwork(); NoopNetworkPlugin m_noop; std::unique_ptr m_loNetwork { nullptr }; + + std::unique_ptr m_defaultNetwork { nullptr }; CRIRuntimeServiceImpl *m_criImpl { nullptr }; std::string m_nsenterPath; - std::string m_pluginDir; - std::string m_vendorCNIDirPrefix; - std::string m_binDir; + std::string m_confDir; + std::vector m_binDirs; + std::string m_podCidr; pthread_rwlock_t m_netsLock = PTHREAD_RWLOCK_INITIALIZER; - std::map> m_networks; + + std::thread m_syncThread; + bool m_needFinish; }; } // namespace Network diff --git a/src/services/cri/cri_helpers.cc b/src/services/cri/cri_helpers.cc index ef33383..0717747 100644 --- a/src/services/cri/cri_helpers.cc +++ b/src/services/cri/cri_helpers.cc @@ -49,6 +49,8 @@ const std::string Constants::CONTAINER_TYPE_ANNOTATION_KEY { "io.kubernetes.cri. const std::string Constants::CONTAINER_TYPE_ANNOTATION_CONTAINER { "container" }; const std::string Constants::CONTAINER_TYPE_ANNOTATION_SANDBOX { "sandbox" }; const std::string Constants::SANDBOX_ID_ANNOTATION_KEY { "io.kubernetes.cri.sandbox-id" }; +const std::string Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE { "pod-cidr-change" }; +const std::string Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR { "pod-cidr" }; const char *InternalLabelKeys[] = { CRIHelpers::Constants::CONTAINER_TYPE_LABEL_KEY.c_str(), @@ -565,7 +567,7 @@ std::vector GetSeccompiSuladOpts(const std::string &seccompProfile, E if (seccompProfile.empty() || seccompProfile == "unconfined") { return std::vector { { "seccomp", "unconfined", "" } }; } - if (seccompProfile == "iSulad/default" || seccompProfile == "docker/default") { + if (seccompProfile == "iSulad/default" || seccompProfile == "docker/default" || seccompProfile == "runtime/default") { // return nil so docker will load the default seccomp profile return std::vector {}; } diff --git a/src/services/cri/cri_helpers.h b/src/services/cri/cri_helpers.h index 2f8561d..73c08d1 100644 --- a/src/services/cri/cri_helpers.h +++ b/src/services/cri/cri_helpers.h @@ -53,7 +53,11 @@ public: static const std::string CONTAINER_TYPE_ANNOTATION_CONTAINER; static const std::string CONTAINER_TYPE_ANNOTATION_SANDBOX; static const std::string SANDBOX_ID_ANNOTATION_KEY; + + static const std::string NET_PLUGIN_EVENT_POD_CIDR_CHANGE; + static const std::string NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR; }; + std::string GetDefaultSandboxImage(Errors &err); json_map_string_string *MakeLabels(const google::protobuf::Map &mapLabels, Errors &error); diff --git a/src/services/cri/cri_runtime_service.cc b/src/services/cri/cri_runtime_service.cc index 807ce01..0333341 100644 --- a/src/services/cri/cri_runtime_service.cc +++ b/src/services/cri/cri_runtime_service.cc @@ -109,14 +109,11 @@ cleanup: void CRIRuntimeServiceImpl::UpdateRuntimeConfig(const runtime::v1alpha2::RuntimeConfig &config, Errors &error) { - const std::string NET_PLUGIN_EVENT_POD_CIDR_CHANGE { "pod-cidr-change" }; - const std::string NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR { "pod-cidr" }; - INFO("iSulad cri received runtime config: %s", config.network_config().pod_cidr().c_str()); if (m_pluginManager != nullptr && config.has_network_config() && !(config.network_config().pod_cidr().empty())) { std::map events; - events[NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = config.network_config().pod_cidr(); - m_pluginManager->Event(NET_PLUGIN_EVENT_POD_CIDR_CHANGE, events); + events[CRIHelpers::Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = config.network_config().pod_cidr(); + m_pluginManager->Event(CRIHelpers::Constants::NET_PLUGIN_EVENT_POD_CIDR_CHANGE, events); } return; } diff --git a/src/services/cri/cri_runtime_service.h b/src/services/cri/cri_runtime_service.h index 29c0d23..e5a1679 100644 --- a/src/services/cri/cri_runtime_service.h +++ b/src/services/cri/cri_runtime_service.h @@ -247,7 +247,7 @@ private: const std::string &jsonCheckpoint, Errors &error); void SetupUserDefinedNetworkPlane(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id, container_inspect *inspect_data, std::map &stdAnnos, - Errors &error); + std::map &options, Errors &error); void StartSandboxContainer(const std::string &response_id, Errors &error); std::string CreateSandboxContainer(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &image, std::string &jsonCheckpoint, const std::string &runtimeHandler, Errors &error); diff --git a/src/services/cri/cri_sandbox.cc b/src/services/cri/cri_sandbox.cc index 671a70a..805c617 100644 --- a/src/services/cri/cri_sandbox.cc +++ b/src/services/cri/cri_sandbox.cc @@ -469,7 +469,8 @@ void CRIRuntimeServiceImpl::StartSandboxContainer(const std::string &response_id void CRIRuntimeServiceImpl::SetupUserDefinedNetworkPlane(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id, container_inspect *inspect_data, - std::map &stdAnnos, Errors &error) + std::map &stdAnnos, + std::map &options, Errors &error) { google::protobuf::Map annotations; CRIHelpers::ExtractAnnotations(inspect_data->config->annotations, annotations); @@ -485,8 +486,8 @@ void CRIRuntimeServiceImpl::SetupUserDefinedNetworkPlane(const runtime::v1alpha2 if (networks[i] && networks[i]->name && networks[i]->interface && strcmp(networks[i]->name, Network::DEFAULT_NETWORK_PLANE_NAME.c_str()) != 0) { INFO("SetupPod net: %s", networks[i]->name); - m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), networks[i]->name, - networks[i]->interface, response_id, stdAnnos, error); + m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), networks[i]->interface, response_id, + stdAnnos, options, error); if (error.Empty()) { continue; } @@ -508,6 +509,8 @@ void CRIRuntimeServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::PodSand Errors &error) { std::map stdAnnos; + std::map networkOptions; + container_inspect *inspect_data = InspectContainer(response_id, error); if (error.NotEmpty()) { return; @@ -530,22 +533,16 @@ void CRIRuntimeServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::PodSand // Setup networking for the sandbox. CRIHelpers::ProtobufAnnoMapToStd(config.annotations(), stdAnnos); stdAnnos[CRIHelpers::Constants::POD_CHECKPOINT_KEY] = jsonCheckpoint; + networkOptions["UID"] = config.metadata().uid(); + m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), - Network::DEFAULT_NETWORK_PLANE_NAME, Network::DEFAULT_NETWORK_INTERFACE_NAME, response_id, - stdAnnos, error); + Network::DEFAULT_NETWORK_INTERFACE_NAME, response_id, stdAnnos, networkOptions, error); if (error.NotEmpty()) { ERROR("SetupPod failed: %s", error.GetCMessage()); StopContainerHelper(response_id, error); goto cleanup; } - // Multi network plane featrue - // Set up user defined network plane - SetupUserDefinedNetworkPlane(config, response_id, inspect_data, stdAnnos, error); - if (error.NotEmpty()) { - ERROR("failed to user defined network plane"); - goto cleanup; - } cleanup: free_container_inspect(inspect_data); } @@ -723,8 +720,7 @@ int CRIRuntimeServiceImpl::TearDownPodCniNetwork(const std::string &realSandboxI if (networks[i] && networks[i]->name && networks[i]->interface && strcmp(networks[i]->name, Network::DEFAULT_NETWORK_PLANE_NAME.c_str()) != 0) { Errors tmpErr; - m_pluginManager->TearDownPod(ns, name, networks[i]->name, networks[i]->interface, inspect_data->id, - stdAnnos, tmpErr); + m_pluginManager->TearDownPod(ns, name, networks[i]->interface, inspect_data->id, stdAnnos, tmpErr); if (tmpErr.NotEmpty()) { WARN("TearDownPod cni network failed: %s", tmpErr.GetCMessage()); errlist.push_back(tmpErr.GetMessage()); @@ -746,22 +742,17 @@ int CRIRuntimeServiceImpl::ClearCniNetwork(const std::string &realSandboxID, boo bool ready = GetNetworkReady(realSandboxID, networkErr); if (!hostNetwork && (ready || networkErr.NotEmpty())) { Errors pluginErr; - m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_PLANE_NAME, - Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, pluginErr); + m_pluginManager->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, realSandboxID, stdAnnos, pluginErr); if (pluginErr.NotEmpty()) { - WARN("TearDownPod cni network: %s failed: %s", Network::DEFAULT_NETWORK_PLANE_NAME.c_str(), - pluginErr.GetCMessage()); + WARN("TearDownPod cni network failed: %s", pluginErr.GetCMessage()); errlist.push_back(pluginErr.GetMessage()); } else { - INFO("TearDownPod cni network: %s success", Network::DEFAULT_NETWORK_PLANE_NAME.c_str()); + INFO("TearDownPod cni network: success"); SetNetworkReady(realSandboxID, false, pluginErr); if (pluginErr.NotEmpty()) { WARN("set network ready: %s", pluginErr.GetCMessage()); } } - if (TearDownPodCniNetwork(realSandboxID, errlist, stdAnnos, ns, name, error)) { - return -1; - } } return 0; } diff --git a/src/services/cri/network_plugin.cc b/src/services/cri/network_plugin.cc index bd7074e..8b899d9 100644 --- a/src/services/cri/network_plugin.cc +++ b/src/services/cri/network_plugin.cc @@ -401,9 +401,10 @@ void PluginManager::GetPodNetworkStatus(const std::string &ns, const std::string Unlock(fullName, error); } -void PluginManager::SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, +void PluginManager::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, - std::map &annotations, Errors &error) + std::map &annotations, + const std::map &options, Errors &error) { if (m_plugin == nullptr) { return; @@ -417,7 +418,7 @@ void PluginManager::SetUpPod(const std::string &ns, const std::string &name, con INFO("Calling network plugin %s to set up pod %s", m_plugin->Name().c_str(), fullName.c_str()); Errors tmpErr; - m_plugin->SetUpPod(ns, name, networkPlane, interfaceName, podSandboxID, annotations, tmpErr); + m_plugin->SetUpPod(ns, name, interfaceName, podSandboxID, annotations, options, tmpErr); if (tmpErr.NotEmpty()) { error.Errorf("NetworkPlugin %s failed to set up pod %s network: %s", m_plugin->Name().c_str(), fullName.c_str(), tmpErr.GetCMessage()); @@ -425,8 +426,8 @@ void PluginManager::SetUpPod(const std::string &ns, const std::string &name, con Unlock(fullName, error); } -void PluginManager::TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &podSandboxID, +void PluginManager::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, + const std::string &podSandboxID, std::map &annotations, Errors &error) { Errors tmpErr; @@ -440,7 +441,7 @@ void PluginManager::TearDownPod(const std::string &ns, const std::string &name, } INFO("Calling network plugin %s to tear down pod %s", m_plugin->Name().c_str(), fullName.c_str()); - m_plugin->TearDownPod(ns, name, networkPlane, interfaceName, podSandboxID, annotations, tmpErr); + m_plugin->TearDownPod(ns, name, Network::DEFAULT_NETWORK_INTERFACE_NAME, podSandboxID, annotations, tmpErr); if (tmpErr.NotEmpty()) { error.Errorf("NetworkPlugin %s failed to teardown pod %s network: %s", m_plugin->Name().c_str(), fullName.c_str(), tmpErr.GetCMessage()); @@ -499,15 +500,16 @@ std::map *NoopNetworkPlugin::Capabilities() return ret; } -void NoopNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, +void NoopNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, - const std::map &annotations, Errors &error) + const std::map &annotations, + const std::map &options, Errors &error) { return; } -void NoopNetworkPlugin::TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &podSandboxID, +void NoopNetworkPlugin::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, + const std::string &podSandboxID, const std::map &annotations, Errors &error) { return; diff --git a/src/services/cri/network_plugin.h b/src/services/cri/network_plugin.h index eff5875..7e6e1e8 100644 --- a/src/services/cri/network_plugin.h +++ b/src/services/cri/network_plugin.h @@ -102,12 +102,13 @@ public: virtual std::map *Capabilities() = 0; - virtual void SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, + virtual void SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, - const std::map &annotations, Errors &error) = 0; + const std::map &annotations, + const std::map &options, Errors &error) = 0; virtual void TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &podSandboxID, + const std::string &podSandboxID, const std::map &annotations, Errors &error) = 0; virtual void GetPodNetworkStatus(const std::string &ns, const std::string &name, const std::string &interfaceName, @@ -136,12 +137,13 @@ public: std::map *Capabilities() override; - void SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, + void SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, - const std::map &annotations, Errors &error) override; + const std::map &annotations, + const std::map &options, Errors &error) override; void TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &podSandboxID, + const std::string &podSandboxID, const std::map &annotations, Errors &error) override; void GetPodNetworkStatus(const std::string &ns, const std::string &name, const std::string &interfaceName, @@ -201,11 +203,12 @@ public: void Status(Errors &error); void GetPodNetworkStatus(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, PodNetworkStatus &status, Errors &error); - void SetUpPod(const std::string &ns, const std::string &name, const std::string &networkPlane, + void SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &podSandboxID, - std::map &annotations, Errors &error); + std::map &annotations, + const std::map &options, Errors &error); void TearDownPod(const std::string &ns, const std::string &name, const std::string &networkPlane, - const std::string &interfaceName, const std::string &podSandboxID, + const std::string &podSandboxID, std::map &annotations, Errors &error); private: diff --git a/src/services/execution/execute/execution.c b/src/services/execution/execute/execution.c index d0c1f8d..b7b40c3 100644 --- a/src/services/execution/execute/execution.c +++ b/src/services/execution/execute/execution.c @@ -792,6 +792,63 @@ out: return ret; } +static int create_mtab_link(const oci_runtime_spec *oci_spec) +{ + char *pathname = "/proc/mounts"; + char *slink = NULL; + char *dir = NULL; + int ret = 0; + + if (oci_spec->root == NULL || oci_spec->root->path == NULL) { + ERROR("Root path is NULL, can not create link /etc/mtab for target /proc/mounts"); + return -1; + } + + slink = util_path_join(oci_spec->root->path, "/etc/mtab"); + if (slink == NULL) { + ERROR("Failed to join path:%s with /etc/mtab", oci_spec->root->path); + ret = -1; + goto out; + } + + dir = util_path_dir(slink); + if (dir == NULL) { + ERROR("Failed to get dir %s", slink); + ret = -1; + goto out; + } + + if (!util_dir_exists(dir)) { + ret = util_mkdir_p(dir, ETC_FILE_MODE); + if (ret != 0) { + ERROR("Unable to create mtab directory %s.", dir); + goto out; + } + } + + if (util_file_exists(slink)) { + goto out; + } + + ret = symlink(pathname, slink); + if (ret < 0 && errno != EEXIST) { + if (errno == EROFS) { + WARN("Failed to create link %s for target %s. Read-only filesystem", slink, pathname); + } else { + SYSERROR("Failed to create \"%s\"", slink); + ret = -1; + goto out; + } + } + + ret = 0; + +out: + free(slink); + free(dir); + return ret; +} + static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, container_pid_t *pid_info) { @@ -867,6 +924,13 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo goto close_exit_fd; } + nret = create_mtab_link(oci_spec); + if (nret != 0) { + ERROR("Failed to create link /etc/mtab for target /proc/mounts"); + ret = -1; + goto close_exit_fd; + } + if (renew_oci_config(cont, oci_spec) != 0) { ret = -1; goto close_exit_fd; @@ -1745,6 +1809,39 @@ out: return ret; } +int umount_residual_shm(const char *mount_info, const char *target) +{ + if (strncmp(mount_info, target, strlen(target)) != 0) { + return 0; + } + + DEBUG("Try to umount: %s", mount_info); + if (umount2(mount_info, MNT_DETACH)) { + SYSERROR("Failed to umount residual mount: %s", mount_info); + } + + return 0; +} + +int cleanup_mounts_by_id(const char *id, const char *engine_root_path) +{ + char target[PATH_MAX] = {0}; + int nret = 0; + + nret = snprintf(target, PATH_MAX, "%s/%s", engine_root_path, id); + if (nret < 0 || nret >= PATH_MAX) { + ERROR("Sprintf failed"); + return -1; + } + + if (!util_deal_with_mount_info(umount_residual_shm, target)) { + ERROR("Cleanup mounts failed"); + return -1; + } + + return 0; +} + static int do_cleanup_container_resources(container_t *cont) { int ret = 0; @@ -1804,6 +1901,9 @@ static int do_cleanup_container_resources(container_t *cont) umount_host_channel(cont->hostconfig->host_channel); + // clean residual mount points + cleanup_mounts_by_id(id, rootpath); + if (do_runtime_rm_helper(id, runtime, rootpath) != 0) { ret = -1; goto out; diff --git a/src/services/execution/execute/execution.h b/src/services/execution/execute/execution.h index 44a42ad..af6a4b3 100644 --- a/src/services/execution/execute/execution.h +++ b/src/services/execution/execute/execution.h @@ -35,6 +35,7 @@ int stop_container(container_t *cont, int timeout, bool force, bool restart); int set_container_to_removal(const container_t *cont); +int cleanup_mounts_by_id(const char *id, const char *engine_root_path); #ifdef __cplusplus } diff --git a/src/services/execution/manager/container_unix.c b/src/services/execution/manager/container_unix.c index 3646eca..b7aac72 100644 --- a/src/services/execution/manager/container_unix.c +++ b/src/services/execution/manager/container_unix.c @@ -454,8 +454,6 @@ static int save_json_config_file(const char *id, const char *rootpath, { int ret = 0; int nret; - int fd = -1; - ssize_t len = 0; char filename[PATH_MAX] = { 0 }; if (json_data == NULL || strlen(json_data) == 0) { @@ -468,21 +466,12 @@ static int save_json_config_file(const char *id, const char *rootpath, goto out; } - fd = util_open(filename, O_CREAT | O_TRUNC | O_CLOEXEC | O_WRONLY, CONFIG_FILE_MODE); - if (fd == -1) { - ERROR("Create file %s failed: %s", filename, strerror(errno)); - isulad_set_error_message("Create file '%s' failed: %s", filename, strerror(errno)); - ret = -1; - goto out; - } - - len = util_write_nointr(fd, json_data, strlen(json_data)); - if (len < 0 || ((size_t)len) != strlen(json_data)) { + nret = util_atomic_write_file(filename, json_data, strlen(json_data), CONFIG_FILE_MODE); + if (nret != 0) { ERROR("Write file %s failed: %s", filename, strerror(errno)); isulad_set_error_message("Write file '%s' failed: %s", filename, strerror(errno)); ret = -1; } - close(fd); out: return ret; diff --git a/src/services/execution/manager/restore.c b/src/services/execution/manager/restore.c index 9e7e1f7..046f76f 100644 --- a/src/services/execution/manager/restore.c +++ b/src/services/execution/manager/restore.c @@ -371,6 +371,13 @@ static int remove_invalid_container(const container_t *cont, const char *runtime goto out; } + ret = cleanup_mounts_by_id(id, root); + if (ret != 0) { + ERROR("Failed to clean container's mounts"); + ret = -1; + goto out; + } + ret = snprintf(container_root, sizeof(container_root), "%s/%s", root, id); if (ret < 0 || (size_t)ret >= sizeof(container_root)) { ERROR("Failed to sprintf invalid root directory %s/%s", root, id); diff --git a/src/services/execution/spec/specs.c b/src/services/execution/spec/specs.c index 7b759e8..67b6544 100644 --- a/src/services/execution/spec/specs.c +++ b/src/services/execution/spec/specs.c @@ -2191,7 +2191,7 @@ int save_oci_config(const char *id, const char *rootpath, const oci_runtime_spec goto out_free; } - if (util_write_file(file_path, json_container, strlen(json_container), DEFAULT_SECURE_FILE_MODE) != 0) { + if (util_atomic_write_file(file_path, json_container, strlen(json_container), DEFAULT_SECURE_FILE_MODE) != 0) { ERROR("write json container failed: %s", strerror(errno)); ret = -1; goto out_free; diff --git a/src/services/execution/spec/specs_mount.c b/src/services/execution/spec/specs_mount.c index ff89083..751b314 100644 --- a/src/services/execution/spec/specs_mount.c +++ b/src/services/execution/spec/specs_mount.c @@ -2125,8 +2125,8 @@ static char *get_prepare_share_shm_path(const char *truntime, const char *cid) goto err_out; } - nret = sprintf(spath, "%s/%s/mounts/shm/", c_root_path, cid); - if (nret < 0) { + nret = snprintf(spath, slen, "%s/%s/mounts/shm/", c_root_path, cid); + if (nret < 0 || nret >= slen) { ERROR("Sprintf failed"); goto err_out; } @@ -2187,8 +2187,8 @@ static int prepare_share_shm(oci_runtime_spec *oci_spec, host_config *host_spec, ERROR("Build shm dir failed"); goto out; } - nret = sprintf(shmproperty, "mode=1777,size=%"PRId64, host_spec->shm_size); - if (nret < 0) { + nret = snprintf(shmproperty, MAX_PROPERTY_LEN, "mode=1777,size=%"PRId64, host_spec->shm_size); + if (nret < 0 || nret >= MAX_PROPERTY_LEN) { ERROR("Sprintf failed"); goto out; } diff --git a/src/services/execution/spec/verify.c b/src/services/execution/spec/verify.c index a066492..3367fda 100644 --- a/src/services/execution/spec/verify.c +++ b/src/services/execution/spec/verify.c @@ -1482,6 +1482,29 @@ out: return ret; } +/* verify oci linux */ +static int verify_process_env(const defs_process *process) +{ + int ret = 0; + size_t i = 0; + char *new_env = NULL; + + for (i = 0; i < process->env_len; i++) { + if (util_validate_env(process->env[i], &new_env) != 0) { + ERROR("Invalid environment %s", process->env[i]); + isulad_set_error_message("Invalid environment %s", process->env[i]); + ret = -1; + goto out; + } + free(new_env); + new_env = NULL; + } + +out: + free(new_env); + return ret; +} + static int verify_container_linux(const oci_runtime_spec *container, const sysinfo_t *sysinfo) { int ret = 0; @@ -1498,6 +1521,14 @@ static int verify_container_linux(const oci_runtime_spec *container, const sysin } } + /* verify oci spec process settings */ + if (container->process != NULL) { + ret = verify_process_env(container->process); + if (ret != 0) { + goto out; + } + } + out: return ret; } diff --git a/src/websocket/service/exec_serve.cc b/src/websocket/service/exec_serve.cc index 0437b93..66a0835 100644 --- a/src/websocket/service/exec_serve.cc +++ b/src/websocket/service/exec_serve.cc @@ -52,7 +52,9 @@ int ExecServe::Execute(struct lws *wsi, const std::string &token, StderrstringWriter.context = (void *)wsi; StderrstringWriter.write_func = WsWriteStderrToClient; int ret = cb->container.exec(container_req, &container_res, - container_req->attach_stdin ? read_pipe_fd : -1, &StdoutstringWriter, &StderrstringWriter); + container_req->attach_stdin ? read_pipe_fd : -1, + container_req->attach_stdout ? &StdoutstringWriter : nullptr, + container_req->attach_stderr ? &StderrstringWriter : nullptr); if (ret != 0) { std::string message; if (container_res != nullptr && container_res->errmsg != nullptr) { diff --git a/test/services/execution/spec/selinux_label_mock_llt.cc b/test/services/execution/spec/selinux_label_mock_llt.cc index 94f8417..eecdc4c 100644 --- a/test/services/execution/spec/selinux_label_mock_llt.cc +++ b/test/services/execution/spec/selinux_label_mock_llt.cc @@ -56,7 +56,7 @@ TEST_F(SELinuxGetEnableUnitTest, test_selinux_get_enable_normal) const uint32_t selinuxfsMagic = 0xf97cff8c; struct statfs sfbuf { .f_type = selinuxfsMagic, - .f_flags = 0 + .f_flags = 0 }; EXPECT_CALL(m_syscall, Statfs(_, _)) diff --git a/test/test.sh b/test/test.sh index 876d888..194bf12 100755 --- a/test/test.sh +++ b/test/test.sh @@ -16,7 +16,7 @@ #set -xe -usage() +function usage() { echo "Usage: sh llt.sh [OPTIONS]" echo "Use llt.sh to control llt operation"