From 48bcdbde2be8f8d1112046c6621200150ffc4a5d Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 19 Mar 2024 19:33:09 +0800 Subject: [PATCH] upgrade from upstream Signed-off-by: zhongtao (cherry picked from commit fc7c8485b85f940b9d1346514f476f1cd53560e5) --- ...ulad-to-pull-load-image-with-symlink.patch | 45 ++++++++++ ...in-ServiceWorkThread-to-prevent-the-.patch | 27 ++++++ ...oncurrency-competition-between-the-r.patch | 83 +++++++++++++++++++ 0202-add-concurrent-load-test.patch | 73 ++++++++++++++++ ...-of-the-host-path-for-archive-when-c.patch | 63 ++++++++++++++ 0204-bugfix-for-wrong-goto-branch.patch | 35 ++++++++ iSulad.spec | 14 +++- 7 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 0199-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch create mode 100644 0200-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch create mode 100644 0201-bugfix-for-the-concurrency-competition-between-the-r.patch create mode 100644 0202-add-concurrent-load-test.patch create mode 100644 0203-get-the-realpath-of-the-host-path-for-archive-when-c.patch create mode 100644 0204-bugfix-for-wrong-goto-branch.patch diff --git a/0199-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch b/0199-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch new file mode 100644 index 0000000..02d3c57 --- /dev/null +++ b/0199-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch @@ -0,0 +1,45 @@ +From 7cea2f9bd1eb4e887be69a0377299a573a980c2a Mon Sep 17 00:00:00 2001 +From: xuxuepeng +Date: Mon, 19 Feb 2024 01:05:18 +0000 +Subject: [PATCH 199/204] !2371 Allow iSulad to pull/load image with symlink * + Allow iSulad to pull/load image with symlink + +--- + src/utils/tar/util_archive.c | 13 ++++++++++--- + 1 file changed, 10 insertions(+), 3 deletions(-) + +diff --git a/src/utils/tar/util_archive.c b/src/utils/tar/util_archive.c +index 4db68f7c..31dea193 100644 +--- a/src/utils/tar/util_archive.c ++++ b/src/utils/tar/util_archive.c +@@ -634,6 +634,11 @@ static void try_to_replace_exited_dst(const char *dst_path, struct archive_entry + } + } + ++/** ++ * This function has to be used with chroot to prevent a potential attack from manipulating ++ * the path of the file to be extracted, such as using a symbolic link to extract the file to ++ * a location outside the path. ++ */ + int archive_unpack_handler(const struct io_read_wrapper *content, const struct archive_options *options) + { + int ret = 0; +@@ -668,10 +673,12 @@ int archive_unpack_handler(const struct io_read_wrapper *content, const struct a + flags |= ARCHIVE_EXTRACT_PERM; + flags |= ARCHIVE_EXTRACT_ACL; + flags |= ARCHIVE_EXTRACT_FFLAGS; +- flags |= ARCHIVE_EXTRACT_SECURE_SYMLINKS; +- flags |= ARCHIVE_EXTRACT_SECURE_NODOTDOT; + flags |= ARCHIVE_EXTRACT_XATTR; +- flags |= ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS; ++ /** ++ * ARCHIVE_EXTRACT_SECURE_SYMLINKS, ARCHIVE_EXTRACT_SECURE_NODOTDOT, ++ * ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS flags are not set here, ++ * since this function is called after chroot, the security of the path is guaranteed. ++ */ + + a = archive_read_new(); + if (a == NULL) { +-- +2.25.1 + diff --git a/0200-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch b/0200-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch new file mode 100644 index 0000000..dafaaef --- /dev/null +++ b/0200-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch @@ -0,0 +1,27 @@ +From cd08dff9e0cd0e01717d7cce2460c01f376b8234 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 6 Feb 2024 20:05:05 +0800 +Subject: [PATCH 200/204] sleep some time in ServiceWorkThread to prevent the + CPU from being occupied all the time + +Signed-off-by: zhongtao +--- + src/daemon/entry/cri/websocket/service/ws_server.cc | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/src/daemon/entry/cri/websocket/service/ws_server.cc b/src/daemon/entry/cri/websocket/service/ws_server.cc +index d439dd96..a85e4a02 100644 +--- a/src/daemon/entry/cri/websocket/service/ws_server.cc ++++ b/src/daemon/entry/cri/websocket/service/ws_server.cc +@@ -673,6 +673,8 @@ void WebsocketServer::ServiceWorkThread(int threadid) + + while (n >= 0 && !m_forceExit) { + n = lws_service(m_context, 0); ++ // sleep some time to prevent the CPU from being occupied all the time ++ std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + +-- +2.25.1 + diff --git a/0201-bugfix-for-the-concurrency-competition-between-the-r.patch b/0201-bugfix-for-the-concurrency-competition-between-the-r.patch new file mode 100644 index 0000000..60179d7 --- /dev/null +++ b/0201-bugfix-for-the-concurrency-competition-between-the-r.patch @@ -0,0 +1,83 @@ +From 1639a5ae0ec656f1c67f730e163cb1ccca5fd205 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 1 Mar 2024 15:04:09 +0800 +Subject: [PATCH 201/204] bugfix for the concurrency competition between the + reuse layer and the creation layer + +Signed-off-by: zhongtao +--- + src/daemon/modules/image/oci/oci_load.c | 9 ++++++++- + src/daemon/modules/image/oci/registry/registry.c | 9 ++++++++- + 2 files changed, 16 insertions(+), 2 deletions(-) + +diff --git a/src/daemon/modules/image/oci/oci_load.c b/src/daemon/modules/image/oci/oci_load.c +index efffa6f5..af4d134c 100644 +--- a/src/daemon/modules/image/oci/oci_load.c ++++ b/src/daemon/modules/image/oci/oci_load.c +@@ -676,6 +676,12 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + char *parent_chain_id_sha256 = ""; + char *id = NULL; + char *parent_chain_id = NULL; ++ // exist_flag is used to mark whether a non-existent layer has been encountered during this layer reuse process. ++ // 1.exist_flag is true if the layers are currently reusable; ++ // 2.exist_flag is false if encounter an uncreated layer that cannot be reused ++ // Prevent concurrent competition between the creation layer function ++ // and the reuse layer function on the im -> layer_of_hold_refs variable ++ bool exist_flag = true; + + if (im == NULL || manifest == NULL || dstdir == NULL) { + ERROR("Invalid input params image or manifest is null"); +@@ -757,7 +763,7 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + goto out; + } + +- if (storage_inc_hold_refs(id) == 0) { ++ if (exist_flag && storage_inc_hold_refs(id) == 0) { + free(im->layer_of_hold_refs); + im->layer_of_hold_refs = util_strdup_s(id); + if (parent_chain_id != NULL && storage_dec_hold_refs(parent_chain_id) != 0) { +@@ -777,6 +783,7 @@ static int oci_load_set_layers_info(load_image_t *im, const image_manifest_items + continue; + } + ++ exist_flag = false; + if (check_and_set_digest_from_tarball(im->layers[i], conf->rootfs->diff_ids[i]) != 0) { + ERROR("Check layer digest failed"); + ret = -1; +diff --git a/src/daemon/modules/image/oci/registry/registry.c b/src/daemon/modules/image/oci/registry/registry.c +index 5265b5ab..b612a618 100644 +--- a/src/daemon/modules/image/oci/registry/registry.c ++++ b/src/daemon/modules/image/oci/registry/registry.c +@@ -1510,6 +1510,12 @@ static int fetch_all(pull_descriptor *desc) + struct layer_list *list = NULL; + pthread_t tid = 0; + struct timespec ts = { 0 }; ++ // exist_flag is used to mark whether a non-existent layer has been encountered during this layer reuse process. ++ // 1.exist_flag is true if the layers are currently reusable; ++ // 2.exist_flag is false if encounter an uncreated layer that cannot be reused ++ // Prevent concurrent competition between the creation layer function ++ // and the reuse layer function on the im -> layer_of_hold_refs variable ++ bool exist_flag = true; + + if (desc == NULL) { + ERROR("Invalid NULL param"); +@@ -1541,7 +1547,7 @@ static int fetch_all(pull_descriptor *desc) + + // Skip layer that already exist in local store + list = storage_layers_get_by_compress_digest(desc->layers[i].digest); +- if (list != NULL) { ++ if (exist_flag && list != NULL) { + for (j = 0; j < list->layers_len; j++) { + if ((list->layers[j]->parent == NULL && i == 0) || + (parent_chain_id != NULL && list->layers[j]->parent != NULL && +@@ -1573,6 +1579,7 @@ static int fetch_all(pull_descriptor *desc) + continue; + } + } ++ exist_flag = false; + + // parent_chain_id = NULL means no parent chain match from now on, so no longer need + // to get layers by compressed digest to reuse layer. +-- +2.25.1 + diff --git a/0202-add-concurrent-load-test.patch b/0202-add-concurrent-load-test.patch new file mode 100644 index 0000000..500a9f8 --- /dev/null +++ b/0202-add-concurrent-load-test.patch @@ -0,0 +1,73 @@ +From cd7d96e777f33c802d9183afdc610cb620752d4c Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Fri, 1 Mar 2024 15:04:35 +0800 +Subject: [PATCH 202/204] add concurrent load test + +Signed-off-by: zhongtao +--- + CI/test_cases/image_cases/image_load.sh | 47 +++++++++++++++++++++++++ + 1 file changed, 47 insertions(+) + +diff --git a/CI/test_cases/image_cases/image_load.sh b/CI/test_cases/image_cases/image_load.sh +index 52b713d4..a2cada5f 100755 +--- a/CI/test_cases/image_cases/image_load.sh ++++ b/CI/test_cases/image_cases/image_load.sh +@@ -79,8 +79,55 @@ function test_image_load() + return ${ret} + } + ++function test_concurrent_load() ++{ ++ local ret=0 ++ local test="isula load image test => (${FUNCNAME[@]})" ++ ++ msg_info "${test} starting..." ++ ++ # clean exist image ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ busybox_id=`isula inspect -f '{{.image.id}}' busybox` ++ isula rmi $ubuntu_id $busybox_id ++ ++ concurrent_time=10 ++ for i in `seq 1 $concurrent_time` ++ do ++ isula load -i $mult_image & ++ pids[$i]=$! ++ done ++ ++ for i in `seq 1 $concurrent_time`;do ++ wait ${pids[$i]} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to do isulad load $i" && ((ret++)) ++ done ++ ++ ubuntu_id=`isula inspect -f '{{.image.id}}' ubuntu` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: ubuntu" && ((ret++)) ++ ++ top_layer_id=$(isula inspect -f '{{.image.top_layer}}' ${ubuntu_id}) ++ ++ busybox_id=`isula inspect -f '{{.image.id}}' busybox` ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - fail to inspect image: busybox" && ((ret++)) ++ ++ # delete image after concurrent load ++ isula rmi $ubuntu_id $busybox_id ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove image ${ubuntu_id} and ${busybox_id}" && ((ret++)) ++ ++ ls -l /var/lib/isulad/storage/overlay-layers ++ local top_layer_dir=/var/lib/isulad/storage/overlay-layers/${top_layer_id} ++ test -e ${top_layer_dir} ++ [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - top layer dir ${top_layer_id} exist after delete image" && ((ret++)) ++ ++ msg_info "${test} finished with return ${ret}..." ++ return ${ret} ++} ++ + declare -i ans=0 + ++test_concurrent_load || ((ans++)) ++ + test_image_load || ((ans++)) + + show_result ${ans} "${curr_path}/${0}" +-- +2.25.1 + diff --git a/0203-get-the-realpath-of-the-host-path-for-archive-when-c.patch b/0203-get-the-realpath-of-the-host-path-for-archive-when-c.patch new file mode 100644 index 0000000..f9db04c --- /dev/null +++ b/0203-get-the-realpath-of-the-host-path-for-archive-when-c.patch @@ -0,0 +1,63 @@ +From 028198a924f513347ac36aee10f39de37a886812 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Mon, 11 Mar 2024 15:50:45 +0800 +Subject: [PATCH 203/204] get the realpath of the host path for archive when cp + +Signed-off-by: zhongtao +--- + src/utils/tar/isulad_tar.c | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index 27b4ce73..073d1861 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -390,6 +390,7 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + { + int ret = -1; + struct archive_copy_info *dstinfo = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; + char *dstdir = NULL; + char *src_base = NULL; + char *dst_base = NULL; +@@ -410,7 +411,12 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + goto cleanup; + } + +- ret = archive_chroot_untar_stream(content, dstdir, ".", src_base, dst_base, root_dir, err); ++ if (realpath(dstdir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", dstdir); ++ return -1; ++ } ++ ++ ret = archive_chroot_untar_stream(content, cleanpath, ".", src_base, dst_base, root_dir, err); + + cleanup: + free_archive_copy_info(dstinfo); +@@ -427,6 +433,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + struct stat st; + char *srcdir = NULL; + char *srcbase = NULL; ++ char cleanpath[PATH_MAX] = { 0 }; + + if (lstat(path, &st) < 0) { + SYSERROR("lstat %s failed", path); +@@ -437,9 +444,14 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + ERROR("Can not split path: %s", path); + goto cleanup; + } ++ ++ if (realpath(srcdir, cleanpath) == NULL) { ++ ERROR("Failed to get real path for %s", srcdir); ++ return -1; ++ } + + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); +- nret = archive_chroot_tar_stream(srcdir, srcbase, srcbase, rebase, root_dir, archive_reader); ++ nret = archive_chroot_tar_stream(cleanpath, srcbase, srcbase, rebase, root_dir, archive_reader); + if (nret < 0) { + ERROR("Can not archive path: %s", path); + goto cleanup; +-- +2.25.1 + diff --git a/0204-bugfix-for-wrong-goto-branch.patch b/0204-bugfix-for-wrong-goto-branch.patch new file mode 100644 index 0000000..eb81bf3 --- /dev/null +++ b/0204-bugfix-for-wrong-goto-branch.patch @@ -0,0 +1,35 @@ +From 5da69b97ff3ceab0d1bdc523a9cb7058fb4b9487 Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 12 Mar 2024 11:15:26 +0800 +Subject: [PATCH 204/204] bugfix for wrong goto branch + +Signed-off-by: zhongtao +--- + src/utils/tar/isulad_tar.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/utils/tar/isulad_tar.c b/src/utils/tar/isulad_tar.c +index 073d1861..059bebc6 100644 +--- a/src/utils/tar/isulad_tar.c ++++ b/src/utils/tar/isulad_tar.c +@@ -413,7 +413,7 @@ int archive_copy_to(const struct io_read_wrapper *content, const struct archive_ + + if (realpath(dstdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", dstdir); +- return -1; ++ goto cleanup; + } + + ret = archive_chroot_untar_stream(content, cleanpath, ".", src_base, dst_base, root_dir, err); +@@ -447,7 +447,7 @@ static int tar_resource_rebase(const char *path, const char *rebase, const char + + if (realpath(srcdir, cleanpath) == NULL) { + ERROR("Failed to get real path for %s", srcdir); +- return -1; ++ goto cleanup; + } + + DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); +-- +2.25.1 + diff --git a/iSulad.spec b/iSulad.spec index bb3f2cf..5e7472b 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.0.18 -%global _release 16 +%global _release 17 %global is_systemd 1 %global enable_shimv2 1 %global is_embedded 1 @@ -211,6 +211,12 @@ Patch0195: 0195-improve-dt-for-oci-device-update.patch Patch0196: 0196-skip-devmapper-ut.patch Patch0197: 0197-bugfix-for-cont-restart-when-iSulad-drops-original-d.patch Patch0198: 0198-improve-dt-for-default-ulimit-change.patch +Patch0199: 0199-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch +Patch0200: 0200-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch +Patch0201: 0201-bugfix-for-the-concurrency-competition-between-the-r.patch +Patch0202: 0202-add-concurrent-load-test.patch +Patch0203: 0203-get-the-realpath-of-the-host-path-for-archive-when-c.patch +Patch0204: 0204-bugfix-for-wrong-goto-branch.patch %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) @@ -455,6 +461,12 @@ fi %endif %changelog +* Tue Mar 19 2024 zhongtao - 2.0.18-17 +- Type: bugfix +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + * Tue Jan 30 2024 zhongtao - 2.0.18-16 - Type: bugfix - ID: NA