From e62735943ed069ee0c3685c8bb8da3308ccd45ca Mon Sep 17 00:00:00 2001 From: haozi007 Date: Tue, 17 Oct 2023 15:52:11 +0800 Subject: [PATCH 149/181] [refactor] update possible changed resources for oci spec Signed-off-by: haozi007 --- src/cmd/isulad/main.c | 13 +- src/daemon/config/isulad_config.c | 6 +- src/daemon/modules/api/specs_api.h | 11 +- .../container/container_events_handler.c | 2 +- .../modules/service/service_container.c | 68 +++++++-- src/daemon/modules/spec/specs.c | 115 +++++++++------ src/daemon/modules/spec/specs_mount.c | 133 +++++++++++++++--- src/daemon/modules/spec/specs_mount.h | 6 +- .../image/oci/oci_config_merge/CMakeLists.txt | 4 + 9 files changed, 276 insertions(+), 82 deletions(-) diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c index e624cfdd..8d4d886b 100644 --- a/src/cmd/isulad/main.c +++ b/src/cmd/isulad/main.c @@ -1319,7 +1319,7 @@ static int ensure_isulad_tmpdir_security() if (do_ensure_isulad_tmpdir_security("/tmp") != 0) { WARN("Failed to ensure the /tmp directory is a safe directory"); } - + return 0; } @@ -1371,12 +1371,17 @@ static int isulad_server_init_common() goto out; } - if (containers_store_init()) { + if (spec_module_init() != 0) { + ERROR("Failed to init spec module"); + goto out; + } + + if (containers_store_init() != 0) { ERROR("Failed to init containers store"); goto out; } - if (container_name_index_init()) { + if (container_name_index_init() != 0) { ERROR("Failed to init name index"); goto out; } @@ -1782,7 +1787,7 @@ int main(int argc, char **argv) msg = "Failed to init plugin_manager"; goto failure; } -#endif +#endif clock_gettime(CLOCK_MONOTONIC, &t_end); use_time = (double)(t_end.tv_sec - t_start.tv_sec) * (double)1000000000 + (double)(t_end.tv_nsec - t_start.tv_nsec); diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c index 0e389dd1..ce2ae1c1 100644 --- a/src/daemon/config/isulad_config.c +++ b/src/daemon/config/isulad_config.c @@ -998,7 +998,7 @@ static defs_hook *hooks_elem_dup(const defs_hook *src) dest = (defs_hook *)util_common_calloc_s(sizeof(defs_hook)); if (dest == NULL) { ERROR("Out of memory"); - return NULL; + return NULL; } dest->path = util_strdup_s(src->path); @@ -1046,7 +1046,7 @@ static int hooks_array_dup(const defs_hook **src, const size_t src_len, defs_hoo return -1; } - for(i = 0; i < src_len; i++) { + for (i = 0; i < src_len; i++) { tmp_dst[i] = hooks_elem_dup(src[i]); if (tmp_dst[i] == NULL) { ERROR("Failed to duplicate hooks element"); @@ -1060,7 +1060,7 @@ static int hooks_array_dup(const defs_hook **src, const size_t src_len, defs_hoo return 0; err_out: - for(i = 0; i < tmp_len; i++) { + for (i = 0; i < tmp_len; i++) { free_defs_hook(tmp_dst[i]); } free(tmp_dst); diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h index 0a594d81..1a6af9ae 100644 --- a/src/daemon/modules/api/specs_api.h +++ b/src/daemon/modules/api/specs_api.h @@ -29,8 +29,7 @@ int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_c oci_runtime_spec *oci_spec); char *merge_container_cgroups_path(const char *id, const host_config *host_spec); int merge_global_config(oci_runtime_spec *oci_spec); -oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); -oci_runtime_spec *default_spec(bool system_container); + int merge_conf_cgroup(oci_runtime_spec *oci_spec, const host_config *host_spec); int save_oci_config(const char *id, const char *rootpath, const oci_runtime_spec *oci_spec); @@ -40,6 +39,14 @@ int parse_security_opt(const host_config *host_spec, bool *no_new_privileges, ch int merge_share_namespace(oci_runtime_spec *oci_spec, const host_config *host_spec, const container_config_v2_common_config_network_settings *network_settings); +oci_runtime_spec *load_oci_config(const char *rootpath, const char *name); + +oci_runtime_spec *default_spec(bool system_container); + +const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container); + +int spec_module_init(void); + #ifdef __cplusplus } #endif diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c index d56c2ee0..6a223bd2 100644 --- a/src/daemon/modules/container/container_events_handler.c +++ b/src/daemon/modules/container/container_events_handler.c @@ -157,7 +157,7 @@ static int container_state_changed(container_t *cont, const struct isulad_events container_wait_stop_cond_broadcast(cont); #ifdef ENABLE_PLUGIN plugin_event_container_post_stop(cont); -#endif +#endif } auto_remove = !should_restart && cont->hostconfig != NULL && cont->hostconfig->auto_remove; diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c index 58b27f90..a5c12862 100644 --- a/src/daemon/modules/service/service_container.c +++ b/src/daemon/modules/service/service_container.c @@ -13,19 +13,11 @@ * Description: provide container supervisor functions ******************************************************************************/ #define _GNU_SOURCE -#include #include #include #include #include #include -#include -#include -#include -#include -#include -#include -#include #include #include #include @@ -35,15 +27,28 @@ #include #include #include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include #include "service_container_api.h" -#include "isula_libutils/log.h" #include "utils.h" #include "err_msg.h" #include "events_sender_api.h" #include "image_api.h" #include "specs_api.h" #include "specs_mount.h" +#include "specs_extend.h" #include "isulad_config.h" #include "verify.h" #include "plugin_api.h" @@ -678,6 +683,43 @@ out: epoll_loop_close(&descr); } +static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, host_config *hostconfig) +{ + char *cgroup_parent = NULL; + int ret; + + // If isulad daemon cgroup parent updated, we should update this config into oci spec + cgroup_parent = merge_container_cgroups_path(id, hostconfig); + if (cgroup_parent == NULL) { + return -1; + } + if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) { + free(oci_spec->linux->cgroups_path); + oci_spec->linux->cgroups_path = cgroup_parent; + cgroup_parent = NULL; + } + free(cgroup_parent); + + // For Linux.Resources, isula update will save changes into oci spec; + // so we just skip it; + + // Remove old devices and update all devices + ret = update_devcies_for_oci_spec(oci_spec, hostconfig); + if (ret != 0) { + ERROR("Failed to do update devices for oci spec"); + return -1; + } + + // If isulad daemon ulimit updated, we should update this config into oci spec. + if (merge_global_ulimit(oci_spec) != 0) { + return -1; + } + + // renew_oci_config() will update process->user and share namespace after. + + return 0; +} + static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, pid_ppid_info_t *pid_info) { int ret = 0; @@ -752,6 +794,14 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo goto close_exit_fd; } + // Update possible changes + nret = do_oci_spec_update(id, oci_spec, cont->hostconfig); + if (nret != 0) { + ERROR("Failed to update possible changes for oci spec"); + ret = -1; + goto close_exit_fd; + } + nret = setup_ipc_dirs(cont->hostconfig, cont->common_config); if (nret != 0) { ERROR("Failed to setup ipc dirs"); diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c index 0c7d58b3..a8912c96 100644 --- a/src/daemon/modules/spec/specs.c +++ b/src/daemon/modules/spec/specs.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -76,6 +78,13 @@ #define CLONE_NEWCGROUP 0x02000000 #endif +struct readonly_default_oci_spec { + oci_runtime_spec *cont; + oci_runtime_spec *system_cont; +}; + +static struct readonly_default_oci_spec g_rdspec; + static int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec) { if (oci_spec->annotations == NULL) { @@ -377,29 +386,6 @@ out: return ret; } -/* default_spec returns default oci spec used by isulad. */ -oci_runtime_spec *default_spec(bool system_container) -{ - const char *oci_file = OCICONFIG_PATH; - if (system_container) { - oci_file = OCI_SYSTEM_CONTAINER_CONFIG_PATH; - } - oci_runtime_spec *oci_spec = NULL; - parser_error err = NULL; - - /* parse the input oci file */ - oci_spec = oci_runtime_spec_parse_file(oci_file, NULL, &err); - if (oci_spec == NULL) { - ERROR("Failed to parse OCI specification file \"%s\", error message: %s", oci_file, err); - isulad_set_error_message("Can not read the default /etc/default/isulad/config.json file: %s", err); - goto out; - } - -out: - free(err); - return oci_spec; -} - static int make_sure_oci_spec_root(oci_runtime_spec *oci_spec) { if (oci_spec->root == NULL) { @@ -1711,20 +1697,20 @@ static int merge_resources_conf(oci_runtime_spec *oci_spec, host_config *host_sp ret = merge_conf_cgroup(oci_spec, host_spec); if (ret != 0) { - goto out; + return -1; } - ret = merge_conf_device(oci_spec, host_spec); + ret = merge_conf_blkio_device(oci_spec, host_spec); if (ret != 0) { - goto out; + return -1; } - ret = merge_conf_mounts(oci_spec, host_spec, v2_spec); - if (ret) { - goto out; + ret = merge_conf_devices(oci_spec, host_spec); + if (ret != 0) { + return -1; } -out: - return ret; + + return merge_conf_mounts(oci_spec, host_spec, v2_spec); } static int merge_terminal(oci_runtime_spec *oci_spec, bool terminal) @@ -2279,7 +2265,7 @@ oci_runtime_spec *load_oci_config(const char *rootpath, const char *name) nret = snprintf(filename, sizeof(filename), "%s/%s/%s", rootpath, name, OCI_CONFIG_JSON); if (nret < 0 || (size_t)nret >= sizeof(filename)) { ERROR("Failed to print string"); - goto out; + return NULL; } ociconfig = oci_runtime_spec_parse_file(filename, NULL, &err); @@ -2288,6 +2274,7 @@ oci_runtime_spec *load_oci_config(const char *rootpath, const char *name) isulad_set_error_message("Parse oci config file failed:%s", err); goto out; } + out: free(err); return ociconfig; @@ -2295,36 +2282,80 @@ out: int save_oci_config(const char *id, const char *rootpath, const oci_runtime_spec *oci_spec) { - int ret = 0; int nret = 0; - char *json_container = NULL; char file_path[PATH_MAX] = { 0x0 }; struct parser_context ctx = { OPT_PARSE_STRICT, stderr }; + char *json_container = NULL; parser_error err = NULL; + int ret = 0; nret = snprintf(file_path, PATH_MAX, "%s/%s/%s", rootpath, id, OCI_CONFIG_JSON); if (nret < 0 || (size_t)nret >= PATH_MAX) { ERROR("Failed to print string"); - ret = -1; - goto out_free; + return -1; } json_container = oci_runtime_spec_generate_json(oci_spec, &ctx, &err); if (json_container == NULL) { ERROR("Failed to generate json: %s", err); ret = -1; - goto out_free; + goto out; } - if (util_atomic_write_file(file_path, json_container, strlen(json_container), DEFAULT_SECURE_FILE_MODE, false) != - 0) { + nret = util_atomic_write_file(file_path, json_container, strlen(json_container), DEFAULT_SECURE_FILE_MODE, false); + if (nret != 0) { SYSERROR("write json container failed"); ret = -1; - goto out_free; + goto out; } -out_free: - free(err); +out: free(json_container); + free(err); return ret; } + +/* default_spec returns default oci spec used by isulad. */ +oci_runtime_spec *default_spec(bool system_container) +{ + const char *oci_file = OCICONFIG_PATH; + if (system_container) { + oci_file = OCI_SYSTEM_CONTAINER_CONFIG_PATH; + } + oci_runtime_spec *oci_spec = NULL; + parser_error err = NULL; + + /* parse the input oci file */ + oci_spec = oci_runtime_spec_parse_file(oci_file, NULL, &err); + if (oci_spec == NULL) { + ERROR("Failed to parse OCI specification file \"%s\", error message: %s", oci_file, err); + isulad_set_error_message("Can not read the default %s file: %s", oci_file, err); + goto out; + } + +out: + free(err); + return oci_spec; +} + +const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container) +{ + if (system_container) { + return g_rdspec.system_cont; + } + + return g_rdspec.cont; +} + +int spec_module_init(void) +{ + g_rdspec.cont = default_spec(false); + if (g_rdspec.cont == NULL) { + return -1; + } + g_rdspec.system_cont = default_spec(true); + if (g_rdspec.system_cont == NULL) { + return -1; + } + return 0; +} \ No newline at end of file diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c index cd3a5c9d..bb2b200a 100644 --- a/src/daemon/modules/spec/specs_mount.c +++ b/src/daemon/modules/spec/specs_mount.c @@ -53,6 +53,7 @@ #include "image_api.h" #include "volume_api.h" #include "parse_volume.h" +#include "specs_api.h" enum update_rw { update_rw_untouch, @@ -2212,7 +2213,24 @@ out: return ret; } -int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec) +int merge_conf_devices(oci_runtime_spec *oci_spec, host_config *host_spec) +{ + /* devices which will be populated into container */ + if (merge_conf_populate_device(oci_spec, host_spec)) { + ERROR("Merge user define devices failed"); + return -1; + } + + /* device cgroup rules which will be added into container */ + if (merge_conf_device_cgroup_rule(oci_spec, host_spec)) { + ERROR("Merge user define device cgroup rules failed"); + return -1; + } + + return 0; +} + +int merge_conf_blkio_device(oci_runtime_spec *oci_spec, host_config *host_spec) { int ret = 0; @@ -2226,7 +2244,7 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec) ret = merge_blkio_weight_device(oci_spec, host_spec->blkio_weight_device, host_spec->blkio_weight_device_len); if (ret != 0) { ERROR("Failed to merge blkio weight devices"); - goto out; + return -1; } } @@ -2236,7 +2254,7 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec) host_spec->blkio_device_read_bps_len); if (ret != 0) { ERROR("Failed to merge blkio read bps devices"); - goto out; + return -1; } } @@ -2246,7 +2264,7 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec) host_spec->blkio_device_write_bps_len); if (ret != 0) { ERROR("Failed to merge blkio write bps devices"); - goto out; + return -1; } } @@ -2256,7 +2274,7 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec) host_spec->blkio_device_read_iops_len); if (ret != 0) { ERROR("Failed to merge blkio read iops devices"); - goto out; + return -1; } } @@ -2266,24 +2284,11 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec) host_spec->blkio_device_write_iops_len); if (ret != 0) { ERROR("Failed to merge blkio write iops devices"); - goto out; + return -1; } } - /* devices which will be populated into container */ - if (merge_conf_populate_device(oci_spec, host_spec)) { - ret = -1; - goto out; - } - - /* device cgroup rules which will be added into container */ - if (merge_conf_device_cgroup_rule(oci_spec, host_spec)) { - ret = -1; - goto out; - } - -out: - return ret; + return 0; } static bool mounts_expand(defs_mount ***all_mounts, size_t *all_mounts_len, size_t add_len) @@ -3488,3 +3493,91 @@ out: free(mntparent); return ret; } + +int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig) +{ + const oci_runtime_spec *readonly_spec = NULL; + size_t i; + int ret; + + // Step1: get default oci spec config + readonly_spec = get_readonly_default_oci_spec(hostconfig->system_container); + + // Step2: clear oci_spec devices items + for (i = 0; i < oci_spec->linux->devices_len; i++) { + free_defs_device(oci_spec->linux->devices[i]); + oci_spec->linux->devices[i] = NULL; + } + // Step3: if default devices length more than old spec, just realloc memory + if (readonly_spec->linux->devices_len > oci_spec->linux->devices_len) { + free(oci_spec->linux->devices); + oci_spec->linux->devices = util_smart_calloc_s(sizeof(defs_device *), readonly_spec->linux->devices_len); + if (oci_spec->linux->devices == NULL) { + oci_spec->linux->devices_len = 0; + ERROR("Out of memory"); + return -1; + } + } + oci_spec->linux->devices_len = 0; + // Step4: copy default devices to oci spec + for (i = 0; i < readonly_spec->linux->devices_len; i++) { + defs_device *tmp_dev = util_common_calloc_s(sizeof(defs_device)); + if (tmp_dev == NULL) { + ERROR("Out of memory"); + return -1; + } + tmp_dev->type = util_strdup_s(readonly_spec->linux->devices[i]->type); + tmp_dev->path = util_strdup_s(readonly_spec->linux->devices[i]->path); + tmp_dev->file_mode = readonly_spec->linux->devices[i]->file_mode; + tmp_dev->major = readonly_spec->linux->devices[i]->major; + tmp_dev->minor = readonly_spec->linux->devices[i]->minor; + tmp_dev->uid = readonly_spec->linux->devices[i]->uid; + tmp_dev->gid = readonly_spec->linux->devices[i]->gid; + oci_spec->linux->devices[i] = tmp_dev; + oci_spec->linux->devices_len += 1; + } + + // Step5: clear oci_spec device cgroup rules + for (i = 0; i < oci_spec->linux->resources->devices_len; i++) { + free_defs_device_cgroup(oci_spec->linux->resources->devices[i]); + oci_spec->linux->resources->devices[i] = NULL; + } + // Step6: if default devices lenght more than old spec, just realloc memory + if (readonly_spec->linux->resources->devices_len > oci_spec->linux->resources->devices_len) { + free(oci_spec->linux->resources->devices); + oci_spec->linux->resources->devices = util_smart_calloc_s(sizeof(defs_device_cgroup *), + readonly_spec->linux->resources->devices_len); + if (oci_spec->linux->resources->devices == NULL) { + oci_spec->linux->resources->devices_len = 0; + ERROR("Out of memory"); + return -1; + } + } + oci_spec->linux->resources->devices_len = 0; + // Step7: copy default device cgroup rules to oci spec + for (i = 0; i < readonly_spec->linux->resources->devices_len; i++) { + defs_device_cgroup *tmp_dev_cg = util_common_calloc_s(sizeof(defs_device_cgroup)); + if (tmp_dev_cg == NULL) { + ERROR("Out of memory"); + return -1; + } + tmp_dev_cg->allow = readonly_spec->linux->resources->devices[i]->allow; + tmp_dev_cg->major = readonly_spec->linux->resources->devices[i]->major; + tmp_dev_cg->minor = readonly_spec->linux->resources->devices[i]->minor; + tmp_dev_cg->type = util_strdup_s(readonly_spec->linux->resources->devices[i]->type); + tmp_dev_cg->access = util_strdup_s(readonly_spec->linux->resources->devices[i]->access); + oci_spec->linux->resources->devices[i] = tmp_dev_cg; + oci_spec->linux->resources->devices_len += 1; + } + + // Step8: do update devices and cgroup device rules at here + if (hostconfig->privileged) { + // Step8.1: for priviledged container, we should merge all devices under /dev + ret = merge_all_devices_and_all_permission(oci_spec); + } else { + // Step8.2: for common container, we should merge devices defined by user in hostconfig + ret = merge_conf_devices(oci_spec, hostconfig); + } + + return ret; +} \ No newline at end of file diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h index 8a28f0e2..b742ca35 100644 --- a/src/daemon/modules/spec/specs_mount.h +++ b/src/daemon/modules/spec/specs_mount.h @@ -41,10 +41,14 @@ int set_mounts_readwrite_option(const oci_runtime_spec *oci_spec); int merge_all_devices_and_all_permission(oci_runtime_spec *oci_spec); -int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec); +int merge_conf_devices(oci_runtime_spec *oci_spec, host_config *host_spec); + +int merge_conf_blkio_device(oci_runtime_spec *oci_spec, host_config *host_spec); int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2_spec); +int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig); + #ifdef __cplusplus } #endif diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt index 42cd2e78..d76de35d 100644 --- a/test/image/oci/oci_config_merge/CMakeLists.txt +++ b/test/image/oci/oci_config_merge/CMakeLists.txt @@ -31,7 +31,11 @@ add_executable(${EXE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/namespace_mock.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/container_unix_mock.cc ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs_mount.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs_extend.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs_security.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume/volume.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume/local.c ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/selinux_label_mock.cc -- 2.42.0