From e44b2fc5ee8c8b9ca577265e88ae3518828cf8a0 Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Tue, 8 Aug 2023 06:50:07 +0000
Subject: [PATCH 07/11] !2095 fix loading of nsswitch based config inside
 chroot under glibc * fix loading of nsswitch based config inside chroot under
 glibc

---
 cmake/checker.cmake                           |   6 +
 src/CMakeLists.txt                            |   2 +-
 src/cmd/isulad/main.c                         |   3 +
 src/daemon/modules/api/leftover_cleanup_api.h |   2 +
 .../container/leftover_cleanup/cleanup.c      |  80 +++++++
 .../container/leftover_cleanup/cleanup.h      |   1 +
 .../leftover_cleanup/leftover_cleanup_api.c   |   4 +
 src/utils/tar/util_archive.c                  | 217 ++++++++++++++++--
 8 files changed, 299 insertions(+), 16 deletions(-)

diff --git a/cmake/checker.cmake b/cmake/checker.cmake
index fea4f925..000c5e0c 100644
--- a/cmake/checker.cmake
+++ b/cmake/checker.cmake
@@ -30,6 +30,12 @@ else()
     message("--  found linux capability.h --- no")
 endif()
 
+# check libcapability
+pkg_check_modules(PC_LIBCAP REQUIRED "libcap")
+find_library(CAP_LIBRARY cap
+    HINTS ${PC_LIBCAP_LIBDIR} ${PC_CAP_LIBRARY_DIRS})
+_CHECK(CAP_LIBRARY "CAP_LIBRARY-NOTFOUND" "libcap.so")
+
 # check zlib
 pkg_check_modules(PC_ZLIB "zlib>=1.2.8")
 find_path(ZLIB_INCLUDE_DIR zlib.h
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 02d7b13f..7201a030 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -48,7 +48,7 @@ target_include_directories(libisulad_tools
     PUBLIC ${ISULA_LIBUTILS_INCLUDE_DIR}
 )
 set_target_properties(libisulad_tools PROPERTIES PREFIX "")
-target_link_libraries(libisulad_tools ${ZLIB_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} ${CRYPTO_LIBRARY})
+target_link_libraries(libisulad_tools ${ZLIB_LIBRARY} ${ISULA_LIBUTILS_LIBRARY} ${CRYPTO_LIBRARY} ${CAP_LIBRARY})
 
 if (ENABLE_OCI_IMAGE)
     target_link_libraries(libisulad_tools ${LIBARCHIVE_LIBRARY})
diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
index 0cb7b50e..1e51a8e7 100644
--- a/src/cmd/isulad/main.c
+++ b/src/cmd/isulad/main.c
@@ -1254,6 +1254,9 @@ static int isulad_server_init_common()
         goto out;
     }
 #endif
+    // clean tmpdir before image module init
+    // because tmpdir will remove failed if chroot mount point exist under tmpdir
+    isulad_tmpdir_cleaner();
 
     if (volume_init(args->json_confs->graph) != 0) {
         ERROR("Failed to init volume");
diff --git a/src/daemon/modules/api/leftover_cleanup_api.h b/src/daemon/modules/api/leftover_cleanup_api.h
index 26c4509b..0a8f9e0e 100644
--- a/src/daemon/modules/api/leftover_cleanup_api.h
+++ b/src/daemon/modules/api/leftover_cleanup_api.h
@@ -31,6 +31,8 @@ void clean_module_fill_ctx(cleanup_ctx_data_t data_type, void *data);
 
 void clean_module_do_clean();
 
+void isulad_tmpdir_cleaner(void);
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }
 #endif
diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.c b/src/daemon/modules/container/leftover_cleanup/cleanup.c
index 664988b5..eb9b5afb 100644
--- a/src/daemon/modules/container/leftover_cleanup/cleanup.c
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.c
@@ -12,7 +12,11 @@
  * Create: 2022-10-31
  * Description: provide cleanup functions
  *********************************************************************************/
+#include <sys/mount.h>
+
 #include "utils.h"
+#include "utils_fs.h"
+#include "path.h"
 #include "cleanup.h"
 #include "oci_rootfs_clean.h"
 
@@ -132,3 +136,79 @@ void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx)
         }
     }
 }
+
+// always return true;
+// if umount/remove failed, just ignore it
+static bool walk_isulad_tmpdir_cb(const char *path_name, const struct dirent *sub_dir, void *context)
+{
+    int nret = 0;
+    char tmpdir[PATH_MAX] = { 0 };
+    const char *chroot_prefix = "tar-chroot-";
+
+    if (sub_dir == NULL || !util_has_prefix(sub_dir->d_name, chroot_prefix)) {
+        // only umount/remove chroot directory
+        return true;
+    }
+
+    nret = snprintf(tmpdir, PATH_MAX, "%s/%s", path_name, sub_dir->d_name);
+    if (nret < 0 || nret >= PATH_MAX) {
+        WARN("Failed to snprintf for %s", sub_dir->d_name);
+        return true;
+    }
+
+    if (util_detect_mounted(tmpdir)) {
+        if (umount(tmpdir) != 0) {
+            ERROR("Failed to umount target %s, error: %s", tmpdir, strerror(errno));
+        }
+    }
+
+    if (util_path_remove(tmpdir) != 0) {
+        WARN("Failed to remove path %s", tmpdir);
+    }
+
+    return true;
+}
+
+static void cleanup_path(char *dir)
+{
+    int nret;
+    char tmp_dir[PATH_MAX] = { 0 };
+    char cleanpath[PATH_MAX] = { 0 };
+
+    nret = snprintf(tmp_dir, PATH_MAX, "%s/isulad_tmpdir", dir);
+    if (nret < 0 || nret >= PATH_MAX) {
+        ERROR("Failed to snprintf");
+        return;
+    }
+
+    if (util_clean_path(tmp_dir, cleanpath, sizeof(cleanpath)) == NULL) {
+        ERROR("clean path for %s failed", tmp_dir);
+        return;
+    }
+
+    if (!util_dir_exists(cleanpath)) {
+        return;
+    }
+
+    nret = util_scan_subdirs(cleanpath, walk_isulad_tmpdir_cb, NULL);
+    if (nret != 0) {
+        ERROR("failed to scan isulad tmp subdirs");
+    }
+}
+
+// try to umount/remove isulad_tmpdir/tar-chroot-XXX directory
+// ignore return value
+void do_isulad_tmpdir_cleaner(void)
+{
+    char *isula_tmp_dir = NULL;
+
+    isula_tmp_dir = getenv("ISULAD_TMPDIR");
+    if (util_valid_str(isula_tmp_dir)) {
+        cleanup_path(isula_tmp_dir);
+    }
+    // No matter whether ISULAD_TMPDIR is set or not,
+    // clean up the "/tmp" directory to prevent the mount point from remaining
+    cleanup_path("/tmp");
+
+    return;
+}
diff --git a/src/daemon/modules/container/leftover_cleanup/cleanup.h b/src/daemon/modules/container/leftover_cleanup/cleanup.h
index 8dd5e9bd..7ad124f4 100644
--- a/src/daemon/modules/container/leftover_cleanup/cleanup.h
+++ b/src/daemon/modules/container/leftover_cleanup/cleanup.h
@@ -45,6 +45,7 @@ void destroy_cleaners(struct cleaners *clns);
 
 void cleaners_do_clean(struct cleaners *clns, struct clean_ctx *ctx);
 
+void do_isulad_tmpdir_cleaner(void);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }
diff --git a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
index a20dbc3a..fc5b55e1 100644
--- a/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
+++ b/src/daemon/modules/container/leftover_cleanup/leftover_cleanup_api.c
@@ -76,3 +76,7 @@ void clean_module_do_clean()
     g_clean_ctx = NULL;
 }
 
+void isulad_tmpdir_cleaner(void)
+{
+    do_isulad_tmpdir_cleaner();
+}
diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c
index 630ad8f8..c72e63b8 100644
--- a/src/utils/tar/util_archive.c
+++ b/src/utils/tar/util_archive.c
@@ -27,6 +27,10 @@
 #include <stdarg.h>
 #include <stdint.h>
 #include <libgen.h>
+#include <pwd.h>
+#include <netdb.h>
+#include <sys/mount.h>
+#include <sys/capability.h>
 
 #include "stdbool.h"
 #include "utils.h"
@@ -54,6 +58,7 @@ struct archive_context {
     int stdout_fd;
     int stderr_fd;
     pid_t pid;
+    char *safe_dir;
 };
 
 struct archive_content_data {
@@ -72,6 +77,123 @@ ssize_t read_content(struct archive *a, void *client_data, const void **buff)
     return mydata->content->read(mydata->content->context, mydata->buff, sizeof(mydata->buff));
 }
 
+static void do_disable_unneccessary_caps()
+{
+    cap_t caps;
+    caps = cap_get_proc();
+    if (caps == NULL) {
+        SYSERROR("Failed to do get cap");
+        return;
+    }
+    cap_value_t cap_list[] = { CAP_SETUID };
+    // clear all capabilities
+    cap_clear(caps);
+
+    if (cap_set_flag(caps, CAP_EFFECTIVE, sizeof(cap_list) / sizeof(cap_value_t), cap_list, CAP_SET) != 0) {
+        SYSERROR("Failed to clear caps");
+        return;
+    }
+
+    cap_set_proc(caps);
+    cap_free(caps);
+}
+
+static int make_safedir_is_noexec(const char *dstdir, char **safe_dir)
+{
+    struct stat buf;
+    char *isulad_tmpdir_env = NULL;
+    char isula_tmpdir[PATH_MAX] = { 0 };
+    char cleanpath[PATH_MAX] = { 0 };
+    char tmp_dir[PATH_MAX] = { 0 };
+    int nret;
+
+    isulad_tmpdir_env = getenv("ISULAD_TMPDIR");
+    if (!util_valid_str(isulad_tmpdir_env)) {
+        // if not setted isulad tmpdir, just use /tmp
+        isulad_tmpdir_env = "/tmp";
+    }
+
+    nret = snprintf(isula_tmpdir, PATH_MAX, "%s/isulad_tmpdir", isulad_tmpdir_env);
+    if (nret < 0 || nret >= PATH_MAX) {
+        ERROR("Failed to snprintf");
+        return -1;
+    }
+
+    if (util_clean_path(isula_tmpdir, cleanpath, sizeof(cleanpath)) == NULL) {
+        ERROR("clean path for %s failed", isula_tmpdir);
+        return -1;
+    }
+
+    nret = snprintf(tmp_dir, PATH_MAX, "%s/tar-chroot-XXXXXX", cleanpath);
+    if (nret < 0 || nret >= PATH_MAX) {
+        ERROR("Failed to snprintf string");
+        return -1;
+    }
+
+    if (stat(dstdir, &buf) < 0) {
+        SYSERROR("Check chroot dir failed");
+        return -1;
+    }
+
+    // ensure parent dir is exist
+    if (util_mkdir_p(cleanpath, buf.st_mode) != 0) {
+        return -1;
+    }
+
+    if (mkdtemp(tmp_dir) == NULL) {
+        SYSERROR("Create temp dir failed");
+        return -1;
+    }
+
+    // ensure mode of new safe dir, same to dstdir
+    if (util_mkdir_p(tmp_dir, buf.st_mode) != 0) {
+        return -1;
+    }
+
+    if (mount(dstdir, tmp_dir, "none", MS_BIND, NULL) != 0) {
+        SYSERROR("Mount safe dir failed");
+        if (util_path_remove(tmp_dir) != 0) {
+            ERROR("Failed to remove path %s", tmp_dir);
+        }
+        return -1;
+    }
+
+    if (mount(tmp_dir, tmp_dir, "none", MS_BIND | MS_REMOUNT | MS_NOEXEC, NULL) != 0) {
+        SYSERROR("Mount safe dir failed");
+        if (umount(tmp_dir) != 0) {
+            ERROR("Failed to umount target %s", tmp_dir);
+        }
+        if (util_path_remove(tmp_dir) != 0) {
+            ERROR("Failed to remove path %s", tmp_dir);
+        }
+        return -1;
+    }
+
+    *safe_dir = util_strdup_s(tmp_dir);
+    return 0;
+}
+
+// fix loading of nsswitch based config inside chroot under glibc
+static int do_safe_chroot(const char *dstdir)
+{
+    // don't call getpwnam
+    // because it will change file with nobody uid/gid which copied from host to container
+    // if nobody uid/gid is different between host and container
+
+    // set No New Privileges
+    prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0);
+
+    if (chroot(dstdir) != 0) {
+        SYSERROR("Failed to chroot to %s", dstdir);
+        fprintf(stderr, "Failed to chroot to %s: %s", dstdir, strerror(errno));
+        return -1;
+    }
+
+    do_disable_unneccessary_caps();
+
+    return 0;
+}
+
 static bool overlay_whiteout_convert_read(struct archive_entry *entry, const char *dst_path, map_t *unpacked_path_map)
 {
     bool do_write = true;
@@ -597,6 +719,12 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co
     int keepfds[] = { -1, -1, -1 };
     int pipe_stderr[2] = { -1, -1 };
     char errbuf[BUFSIZ + 1] = { 0 };
+    char *safe_dir = NULL;
+
+    if (make_safedir_is_noexec(dstdir, &safe_dir) != 0) {
+        ERROR("Prepare safe dir failed");
+        return -1;
+    }
 
     if (pipe2(pipe_stderr, O_CLOEXEC) != 0) {
         ERROR("Failed to create pipe");
@@ -629,9 +757,9 @@ int archive_unpack(const struct io_read_wrapper *content, const char *dstdir, co
             goto child_out;
         }
 
-        if (chroot(dstdir) != 0) {
-            SYSERROR("Failed to chroot to %s", dstdir);
-            fprintf(stderr, "Failed to chroot to %s: %s", dstdir, strerror(errno));
+        if (do_safe_chroot(safe_dir) != 0) {
+            SYSERROR("Failed to chroot to %s", safe_dir);
+            fprintf(stderr, "Failed to chroot to %s: %s", safe_dir, strerror(errno));
             ret = -1;
             goto child_out;
         }
@@ -669,6 +797,13 @@ cleanup:
     if (errmsg != NULL && strlen(errbuf) != 0) {
         *errmsg = util_strdup_s(errbuf);
     }
+    if (umount(safe_dir) != 0) {
+        ERROR("Failed to umount target %s", safe_dir);
+    }
+    if (util_path_remove(safe_dir) != 0) {
+        ERROR("Failed to remove path %s", safe_dir);
+    }
+    free(safe_dir);
     return ret;
 }
 
@@ -982,6 +1117,12 @@ int archive_chroot_tar(char *path, char *file, char **errmsg)
     int keepfds[] = { -1, -1 };
     char errbuf[BUFSIZ + 1] = { 0 };
     int fd = 0;
+    char *safe_dir = NULL;
+
+    if (make_safedir_is_noexec(path, &safe_dir) != 0) {
+        ERROR("Prepare safe dir failed");
+        return -1;
+    }
 
     if (pipe2(pipe_for_read, O_CLOEXEC) != 0) {
         ERROR("Failed to create pipe");
@@ -1021,9 +1162,9 @@ int archive_chroot_tar(char *path, char *file, char **errmsg)
             goto child_out;
         }
 
-        if (chroot(path) != 0) {
-            ERROR("Failed to chroot to %s", path);
-            fprintf(stderr, "Failed to chroot to %s\n", path);
+        if (do_safe_chroot(safe_dir) != 0) {
+            ERROR("Failed to chroot to %s", safe_dir);
+            fprintf(stderr, "Failed to chroot to %s\n", safe_dir);
             ret = -1;
             goto child_out;
         }
@@ -1064,7 +1205,13 @@ cleanup:
     if (errmsg != NULL && strlen(errbuf) != 0) {
         *errmsg = util_strdup_s(errbuf);
     }
-
+    if (umount(safe_dir) != 0) {
+        ERROR("Failed to umount target %s", safe_dir);
+    }
+    if (util_path_remove(safe_dir) != 0) {
+        ERROR("Failed to remove path %s", safe_dir);
+    }
+    free(safe_dir);
     return ret;
 }
 
@@ -1169,6 +1316,16 @@ static int archive_context_close(void *context, char **err)
         ret = -1;
     }
 
+    if (ctx->safe_dir != NULL) {
+        if (umount(ctx->safe_dir) != 0) {
+            ERROR("Failed to umount target %s", ctx->safe_dir);
+        }
+        if (util_path_remove(ctx->safe_dir) != 0) {
+            ERROR("Failed to remove path %s", ctx->safe_dir);
+        }
+        free(ctx->safe_dir);
+        ctx->safe_dir = NULL;
+    }
     free(marshaled);
     free(ctx);
     return ret;
@@ -1192,10 +1349,10 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha
                .src_base = src_base,
                .dst_base = dst_base
     };
+    char *safe_dir = NULL;
 
-    buf = util_common_calloc_s(buf_len);
-    if (buf == NULL) {
-        ERROR("Out of memory");
+    if (make_safedir_is_noexec(chroot_dir, &safe_dir) != 0) {
+        ERROR("Prepare safe dir failed");
         return -1;
     }
 
@@ -1232,8 +1389,8 @@ int archive_chroot_untar_stream(const struct io_read_wrapper *context, const cha
             goto child_out;
         }
 
-        if (chroot(chroot_dir) != 0) {
-            SYSERROR("Failed to chroot to %s", chroot_dir);
+        if (do_safe_chroot(safe_dir) != 0) {
+            SYSERROR("Failed to chroot to %s", safe_dir);
             ret = -1;
             goto child_out;
         }
@@ -1262,6 +1419,12 @@ child_out:
     close(pipe_stream[0]);
     pipe_stream[0] = -1;
 
+    buf = util_common_calloc_s(buf_len);
+    if (buf == NULL) {
+        ERROR("Out of memory");
+        goto cleanup;
+    }
+
     ctx = util_common_calloc_s(sizeof(struct archive_context));
     if (ctx == NULL) {
         goto cleanup;
@@ -1292,6 +1455,13 @@ cleanup:
     ret = (cret != 0) ? cret : ret;
     close_archive_pipes_fd(pipe_stderr, 2);
     close_archive_pipes_fd(pipe_stream, 2);
+    if (umount(safe_dir) != 0) {
+        ERROR("Failed to umount target %s", safe_dir);
+    }
+    if (util_path_remove(safe_dir) != 0) {
+        ERROR("Failed to remove path %s", safe_dir);
+    }
+    free(safe_dir);
 
     return ret;
 }
@@ -1306,6 +1476,12 @@ int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, cons
     int ret = -1;
     pid_t pid;
     struct archive_context *ctx = NULL;
+    char *safe_dir = NULL;
+
+    if (make_safedir_is_noexec(chroot_dir, &safe_dir) != 0) {
+        ERROR("Prepare safe dir failed");
+        return -1;
+    }
 
     if (pipe(pipe_stderr) != 0) {
         ERROR("Failed to create pipe: %s", strerror(errno));
@@ -1343,9 +1519,9 @@ int archive_chroot_tar_stream(const char *chroot_dir, const char *tar_path, cons
             goto child_out;
         }
 
-        if (chroot(chroot_dir) != 0) {
-            ERROR("Failed to chroot to %s", chroot_dir);
-            fprintf(stderr, "Failed to chroot to %s\n", chroot_dir);
+        if (do_safe_chroot(safe_dir) != 0) {
+            ERROR("Failed to chroot to %s", safe_dir);
+            fprintf(stderr, "Failed to chroot to %s\n", safe_dir);
             ret = -1;
             goto child_out;
         }
@@ -1395,6 +1571,8 @@ child_out:
     ctx->stderr_fd = pipe_stderr[0];
     pipe_stderr[0] = -1;
     ctx->pid = pid;
+    ctx->safe_dir = safe_dir;
+    safe_dir = NULL;
 
     reader->close = archive_context_close;
     reader->context = ctx;
@@ -1406,6 +1584,15 @@ free_out:
     close_archive_pipes_fd(pipe_stderr, 2);
     close_archive_pipes_fd(pipe_stream, 2);
     free(ctx);
+    if (safe_dir != NULL) {
+        if (umount(safe_dir) != 0) {
+            ERROR("Failed to umount target %s", safe_dir);
+        }
+        if (util_path_remove(safe_dir) != 0) {
+            ERROR("Failed to remove path %s", safe_dir);
+        }
+        free(safe_dir);
+    }
 
     return ret;
 }
-- 
2.25.1

