From e44b2fc5ee8c8b9ca577265e88ae3518828cf8a0 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 8 Aug 2023 06:50:07 +0000 Subject: [PATCH 07/11] !2095 fix loading of nsswitch based config inside chroot under glibc * fix loading of nsswitch based config inside chroot under glibc --- cmake/checker.cmake | 6 + src/CMakeLists.txt | 2 +- src/cmd/isulad/main.c | 3 + src/daemon/modules/api/leftover_cleanup_api.h | 2 + .../container/leftover_cleanup/cleanup.c | 80 +++++++ .../container/leftover_cleanup/cleanup.h | 1 + .../leftover_cleanup/leftover_cleanup_api.c | 4 + src/utils/tar/util_archive.c | 217 ++++++++++++++++-- 8 files changed, 299 insertions(+), 16 deletions(-) diff --git a/cmake/checker.cmake b/cmake/checker.cmake index fea4f925..000c5e0c 100644 --- a/cmake/checker.cmake +++ b/cmake/checker.cmake @@ -30,6 +30,12 @@ else() message("-- found linux capability.h --- no") endif() +# check libcapability +pkg_check_modules(PC_LIBCAP REQUIRED "libcap") +find_library(CAP_LIBRARY cap + HINTS ${PC_LIBCAP_LIBDIR} ${PC_CAP_LIBRARY_DIRS}) +_CHECK(CAP_LIBRARY "CAP_LIBRARY-NOTFOUND" "libcap.so") + # check zlib pkg_check_modules(PC_ZLIB "zlib>=1.2.8") find_path(ZLIB_INCLUDE_DIR zlib.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 02d7b13f..7201a030 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,7 +48,7 @@ target_include_directories(libisulad_tools PUBLIC ${ISULA_LIBUTILS_INCLUDE_DIR} ) set_target_properties(libisulad_tools PROPERTIES PREFIX "") -target_link_libraries(libisulad_tools ${ZLIB_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} ${CRYPTO_LIBRARY}) +target_link_libraries(libisulad_tools ${ZLIB_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} ${CRYPTO_LIBRARY} ${CAP_LIBRARY}) if (ENABLE_OCI_IMAGE) target_link_libraries(libisulad_tools ${LIBARCHIVE_LIBRARY}) diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c index 0cb7b50e..1e51a8e7 100644 --- a/src/cmd/isulad/main.c +++ b/src/cmd/isulad/main.c @@ -1254,6 +1254,9 @@ static int isulad_server_init_common() goto out; } #endif + // clean tmpdir before image module init + // because tmpdir will remove failed if chroot mount point exist under tmpdir + isulad_tmpdir_cleaner(); if (volume_init(args->json_confs->graph) != 0) { ERROR("Failed to init volume"); diff --git a/src/daemon/modules/api/leftover_cleanup_api.h b/src/daemon/modules/api/leftover_cleanup_api.h index 26c4509b..0a8f9e0e 100644 --- a/src/daemon/modules/api/leftover_cleanup_api.h +++ b/src/daemon/modules/api/leftover_cleanup_api.h @@ -31,6 +31,8 @@ void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data); void clean_module_do_clean(); +void isulad_tmpdir_cleaner(void); + #if defined(__cplusplus) || defined(c_plusplus) } #endif diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c index 664988b5..eb9b5afb 100644 --- a/src/daemon/modules/container/leftover_cleanup/cleanup.c +++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c @@ -12,7 +12,11 @@ * Create: 2022-10-31 * Description: provide cleanup functions *********************************************************************************/ +#include + #include "utils.h" +#include "utils_fs.h" +#include "path.h" #include "cleanup.h" #include "oci_rootfs_clean.h" @@ -132,3 +136,79 @@ void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx) } } } + +// always return true; +// if umount/remove failed, just ignore it +static bool walk_isulad_tmpdir_cb(const char *path_name, const struct dirent *sub_dir, void *context) +{ + int nret = 0; + char tmpdir[PATH_MAX] = { 0 }; + const char *chroot_prefix = "tar-chroot-"; + + if (sub_dir == NULL || !util_has_prefix(sub_dir->d_name, chroot_prefix)) { + // only umount/remove chroot directory + return true; + } + + nret = snprintf(tmpdir, PATH_MAX, "%s/%s", path_name, sub_dir->d_name); + if (nret < 0 || nret >= PATH_MAX) { + WARN("Failed to snprintf for %s", sub_dir->d_name); + return true; + } + + if (util_detect_mounted(tmpdir)) { + if (umount(tmpdir) != 0) { + ERROR("Failed to umount target %s, error: %s", tmpdir, strerror(errno)); + } + } + + if (util_path_remove(tmpdir) != 0) { + WARN("Failed to remove path %s", tmpdir); + } + + return true; +} + +static void cleanup_path(char *dir) +{ + int nret; + char tmp_dir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; + + nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", dir); + if (nret < 0 || nret >= PATH_MAX) { + ERROR("Failed to snprintf"); + return; + } + + if (util_clean_path(tmp_dir, cleanpath, sizeof(cleanpath)) == NULL) { + ERROR("clean path for %s failed", tmp_dir); + return; + } + + if (!util_dir_exists(cleanpath)) { + return; + } + + nret = util_scan_subdirs(cleanpath, walk_isulad_tmpdir_cb, NULL); + if (nret != 0) { + ERROR("failed to scan isulad tmp subdirs"); + } +} + +// try to umount/remove isulad_tmpdir/tar-chroot-XXX directory +// ignore return value +void do_isulad_tmpdir_cleaner(void) +{ + char *isula_tmp_dir = NULL; + + isula_tmp_dir = getenv("ISULAD_TMPDIR"); + if (util_valid_str(isula_tmp_dir)) { + cleanup_path(isula_tmp_dir); + } + // No matter whether ISULAD_TMPDIR is set or not, + // clean up the "/tmp" directory to prevent the mount point from remaining + cleanup_path("/tmp"); + + return; +} diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.h b/src/daemon/modules/container/leftover_cleanup/cleanup.h index 8dd5e9bd..7ad124f4 100644 --- a/src/daemon/modules/container/leftover_cleanup/cleanup.h +++ b/src/daemon/modules/container/leftover_cleanup/cleanup.h @@ -45,6 +45,7 @@ void destroy_cleaners(struct cleaners *clns); void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx); +void do_isulad_tmpdir_cleaner(void); #if defined(__cplusplus) || defined(c_plusplus) } diff --git a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c index a20dbc3a..fc5b55e1 100644 --- a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c +++ b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c @@ -76,3 +76,7 @@ void clean_module_do_clean() g_clean_ctx = NULL; } +void isulad_tmpdir_cleaner(void) +{ + do_isulad_tmpdir_cleaner(); +} diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c index 630ad8f8..c72e63b8 100644 --- a/src/utils/tar/util_archive.c +++ b/src/utils/tar/util_archive.c @@ -27,6 +27,10 @@ #include #include #include +#include +#include +#include +#include #include "stdbool.h" #include "utils.h" @@ -54,6 +58,7 @@ struct archive_context { int stdout_fd; int stderr_fd; pid_t pid; + char *safe_dir; }; struct archive_content_data { @@ -72,6 +77,123 @@ ssize_t read_content(struct archive *a, void *client_data, const void **buff) return mydata->content->read(mydata->content->context, mydata->buff, sizeof(mydata->buff)); } +static void do_disable_unneccessary_caps() +{ + cap_t caps; + caps = cap_get_proc(); + if (caps == NULL) { + SYSERROR("Failed to do get cap"); + return; + } + cap_value_t cap_list[] = { CAP_SETUID }; + // clear all capabilities + cap_clear(caps); + + if (cap_set_flag(caps, CAP_EFFECTIVE, sizeof(cap_list) / sizeof(cap_value_t), cap_list, CAP_SET) != 0) { + SYSERROR("Failed to clear caps"); + return; + } + + cap_set_proc(caps); + cap_free(caps); +} + +static int make_safedir_is_noexec(const char *dstdir, char **safe_dir) +{ + struct stat buf; + char *isulad_tmpdir_env = NULL; + char isula_tmpdir[PATH_MAX] = { 0 }; + char cleanpath[PATH_MAX] = { 0 }; + char tmp_dir[PATH_MAX] = { 0 }; + int nret; + + isulad_tmpdir_env = getenv("ISULAD_TMPDIR"); + if (!util_valid_str(isulad_tmpdir_env)) { + // if not setted isulad tmpdir, just use /tmp + isulad_tmpdir_env = "/tmp"; + } + + nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmpdir_env); + if (nret < 0 || nret >= PATH_MAX) { + ERROR("Failed to snprintf"); + return -1; + } + + if (util_clean_path(isula_tmpdir, cleanpath, sizeof(cleanpath)) == NULL) { + ERROR("clean path for %s failed", isula_tmpdir); + return -1; + } + + nret = snprintf(tmp_dir, PATH_MAX, "%s/tar-chroot-XXXXXX", cleanpath); + if (nret < 0 || nret >= PATH_MAX) { + ERROR("Failed to snprintf string"); + return -1; + } + + if (stat(dstdir, &buf) < 0) { + SYSERROR("Check chroot dir failed"); + return -1; + } + + // ensure parent dir is exist + if (util_mkdir_p(cleanpath, buf.st_mode) != 0) { + return -1; + } + + if (mkdtemp(tmp_dir) == NULL) { + SYSERROR("Create temp dir failed"); + return -1; + } + + // ensure mode of new safe dir, same to dstdir + if (util_mkdir_p(tmp_dir, buf.st_mode) != 0) { + return -1; + } + + if (mount(dstdir, tmp_dir, "none", MS_BIND, NULL) != 0) { + SYSERROR("Mount safe dir failed"); + if (util_path_remove(tmp_dir) != 0) { + ERROR("Failed to remove path %s", tmp_dir); + } + return -1; + } + + if (mount(tmp_dir, tmp_dir, "none", MS_BIND | MS_REMOUNT | MS_NOEXEC, NULL) != 0) { + SYSERROR("Mount safe dir failed"); + if (umount(tmp_dir) != 0) { + ERROR("Failed to umount target %s", tmp_dir); + } + if (util_path_remove(tmp_dir) != 0) { + ERROR("Failed to remove path %s", tmp_dir); + } + return -1; + } + + *safe_dir = util_strdup_s(tmp_dir); + return 0; +} + +// fix loading of nsswitch based config inside chroot under glibc +static int do_safe_chroot(const char *dstdir) +{ + // don't call getpwnam + // because it will change file with nobody uid/gid which copied from host to container + // if nobody uid/gid is different between host and container + + // set No New Privileges + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + + if (chroot(dstdir) != 0) { + SYSERROR("Failed to chroot to %s", dstdir); + fprintf(stderr, "Failed to chroot to %s: %s", dstdir, strerror(errno)); + return -1; + } + + do_disable_unneccessary_caps(); + + return 0; +} + static bool overlay_whiteout_convert_read(struct archive_entry *entry, const char *dst_path, map_t *unpacked_path_map) { bool do_write = true; @@ -597,6 +719,12 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co int keepfds[] = { -1, -1, -1 }; int pipe_stderr[2] = { -1, -1 }; char errbuf[BUFSIZ + 1] = { 0 }; + char *safe_dir = NULL; + + if (make_safedir_is_noexec(dstdir, &safe_dir) != 0) { + ERROR("Prepare safe dir failed"); + return -1; + } if (pipe2(pipe_stderr, O_CLOEXEC) != 0) { ERROR("Failed to create pipe"); @@ -629,9 +757,9 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co goto child_out; } - if (chroot(dstdir) != 0) { - SYSERROR("Failed to chroot to %s", dstdir); - fprintf(stderr, "Failed to chroot to %s: %s", dstdir, strerror(errno)); + if (do_safe_chroot(safe_dir) != 0) { + SYSERROR("Failed to chroot to %s", safe_dir); + fprintf(stderr, "Failed to chroot to %s: %s", safe_dir, strerror(errno)); ret = -1; goto child_out; } @@ -669,6 +797,13 @@ cleanup: if (errmsg != NULL && strlen(errbuf) != 0) { *errmsg = util_strdup_s(errbuf); } + if (umount(safe_dir) != 0) { + ERROR("Failed to umount target %s", safe_dir); + } + if (util_path_remove(safe_dir) != 0) { + ERROR("Failed to remove path %s", safe_dir); + } + free(safe_dir); return ret; } @@ -982,6 +1117,12 @@ int archive_chroot_tar(char *path, char *file, char **errmsg) int keepfds[] = { -1, -1 }; char errbuf[BUFSIZ + 1] = { 0 }; int fd = 0; + char *safe_dir = NULL; + + if (make_safedir_is_noexec(path, &safe_dir) != 0) { + ERROR("Prepare safe dir failed"); + return -1; + } if (pipe2(pipe_for_read, O_CLOEXEC) != 0) { ERROR("Failed to create pipe"); @@ -1021,9 +1162,9 @@ int archive_chroot_tar(char *path, char *file, char **errmsg) goto child_out; } - if (chroot(path) != 0) { - ERROR("Failed to chroot to %s", path); - fprintf(stderr, "Failed to chroot to %s\n", path); + if (do_safe_chroot(safe_dir) != 0) { + ERROR("Failed to chroot to %s", safe_dir); + fprintf(stderr, "Failed to chroot to %s\n", safe_dir); ret = -1; goto child_out; } @@ -1064,7 +1205,13 @@ cleanup: if (errmsg != NULL && strlen(errbuf) != 0) { *errmsg = util_strdup_s(errbuf); } - + if (umount(safe_dir) != 0) { + ERROR("Failed to umount target %s", safe_dir); + } + if (util_path_remove(safe_dir) != 0) { + ERROR("Failed to remove path %s", safe_dir); + } + free(safe_dir); return ret; } @@ -1169,6 +1316,16 @@ static int archive_context_close(void *context, char **err) ret = -1; } + if (ctx->safe_dir != NULL) { + if (umount(ctx->safe_dir) != 0) { + ERROR("Failed to umount target %s", ctx->safe_dir); + } + if (util_path_remove(ctx->safe_dir) != 0) { + ERROR("Failed to remove path %s", ctx->safe_dir); + } + free(ctx->safe_dir); + ctx->safe_dir = NULL; + } free(marshaled); free(ctx); return ret; @@ -1192,10 +1349,10 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha .src_base = src_base, .dst_base = dst_base }; + char *safe_dir = NULL; - buf = util_common_calloc_s(buf_len); - if (buf == NULL) { - ERROR("Out of memory"); + if (make_safedir_is_noexec(chroot_dir, &safe_dir) != 0) { + ERROR("Prepare safe dir failed"); return -1; } @@ -1232,8 +1389,8 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha goto child_out; } - if (chroot(chroot_dir) != 0) { - SYSERROR("Failed to chroot to %s", chroot_dir); + if (do_safe_chroot(safe_dir) != 0) { + SYSERROR("Failed to chroot to %s", safe_dir); ret = -1; goto child_out; } @@ -1262,6 +1419,12 @@ child_out: close(pipe_stream[0]); pipe_stream[0] = -1; + buf = util_common_calloc_s(buf_len); + if (buf == NULL) { + ERROR("Out of memory"); + goto cleanup; + } + ctx = util_common_calloc_s(sizeof(struct archive_context)); if (ctx == NULL) { goto cleanup; @@ -1292,6 +1455,13 @@ cleanup: ret = (cret != 0) ? cret : ret; close_archive_pipes_fd(pipe_stderr, 2); close_archive_pipes_fd(pipe_stream, 2); + if (umount(safe_dir) != 0) { + ERROR("Failed to umount target %s", safe_dir); + } + if (util_path_remove(safe_dir) != 0) { + ERROR("Failed to remove path %s", safe_dir); + } + free(safe_dir); return ret; } @@ -1306,6 +1476,12 @@ int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, cons int ret = -1; pid_t pid; struct archive_context *ctx = NULL; + char *safe_dir = NULL; + + if (make_safedir_is_noexec(chroot_dir, &safe_dir) != 0) { + ERROR("Prepare safe dir failed"); + return -1; + } if (pipe(pipe_stderr) != 0) { ERROR("Failed to create pipe: %s", strerror(errno)); @@ -1343,9 +1519,9 @@ int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, cons goto child_out; } - if (chroot(chroot_dir) != 0) { - ERROR("Failed to chroot to %s", chroot_dir); - fprintf(stderr, "Failed to chroot to %s\n", chroot_dir); + if (do_safe_chroot(safe_dir) != 0) { + ERROR("Failed to chroot to %s", safe_dir); + fprintf(stderr, "Failed to chroot to %s\n", safe_dir); ret = -1; goto child_out; } @@ -1395,6 +1571,8 @@ child_out: ctx->stderr_fd = pipe_stderr[0]; pipe_stderr[0] = -1; ctx->pid = pid; + ctx->safe_dir = safe_dir; + safe_dir = NULL; reader->close = archive_context_close; reader->context = ctx; @@ -1406,6 +1584,15 @@ free_out: close_archive_pipes_fd(pipe_stderr, 2); close_archive_pipes_fd(pipe_stream, 2); free(ctx); + if (safe_dir != NULL) { + if (umount(safe_dir) != 0) { + ERROR("Failed to umount target %s", safe_dir); + } + if (util_path_remove(safe_dir) != 0) { + ERROR("Failed to remove path %s", safe_dir); + } + free(safe_dir); + } return ret; } -- 2.25.1