From bafac55f6981d7d602e27277baf7b24ecad09306 Mon Sep 17 00:00:00 2001
From: zhangxiaoyu <zhangxiaoyu58@huawei.com>
Date: Wed, 19 Jul 2023 07:36:00 +0000
Subject: [PATCH] !2079 clean network reosurces if runpodsandbox failed * clean
 network reosurces if runpodsandbox failed

---
 .../cri_pod_sandbox_manager_service_impl.cc   | 122 +++++++++++-------
 .../cri_pod_sandbox_manager_service_impl.h    |   9 +-
 .../executor/container_cb/execution_create.c  |  12 +-
 .../modules/api/network_namespace_api.h       |   1 +
 .../oci/storage/image_store/image_store.c     |   2 +-
 .../modules/runtime/isula/isula_rt_ops.c      |   2 +-
 .../modules/service/network_namespace_api.c   |  22 ++++
 test/cutils/utils_utils/utils_utils_ut.cc     |   2 +-
 8 files changed, 120 insertions(+), 52 deletions(-)

diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
index 1cb3254d..91ca2735 100644
--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.cc
@@ -485,10 +485,9 @@ cleanup:
 }
 
 void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::PodSandboxConfig &config,
-                                                       const std::string &response_id,
-                                                       const std::string &jsonCheckpoint, const container_inspect *inspect_data, Errors &error)
+                                                       const std::string &response_id, const std::string &jsonCheckpoint,
+                                                       const container_inspect *inspect_data, std::map<std::string, std::string> &stdAnnos, Errors &error)
 {
-    std::map<std::string, std::string> stdAnnos;
     std::map<std::string, std::string> networkOptions;
 
     // Setup sandbox files
@@ -525,9 +524,81 @@ void PodSandboxManagerServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::
                               Network::DEFAULT_NETWORK_INTERFACE_NAME, response_id, stdAnnos, networkOptions, error);
     if (error.NotEmpty()) {
         ERROR("SetupPod failed: %s", error.GetCMessage());
+        return;
+    }
+
+    SetNetworkReady(response_id, true, error);
+    DEBUG("set %s network ready", response_id.c_str());
+}
+
+void PodSandboxManagerServiceImpl::SetupNetowrkAndStartPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config,
+                                                                  const container_inspect *inspect_data,
+                                                                  std::string &response_id, std::string &jsonCheckpoint,
+                                                                  Errors &error)
+{
+    std::map<std::string, std::string> stdAnnos;
+    char *netnsPath = nullptr;
+
+    netnsPath = get_sandbox_key(inspect_data);
+    if (netnsPath == nullptr || !util_file_exists(netnsPath)) {
+        error.Errorf("Network namespace not exist");
+        ERROR("Network namespace not exist for %s", response_id.c_str());
+        goto cleanup;
+    }
+
+    if (util_mount_namespace(netnsPath) != 0) {
+        error.Errorf("Failed to mount network namespace");
+        ERROR("Failed to mount network namespace for %s", response_id.c_str());
+        goto cleanup;
+    }
+
+    // Step 5: Setup networking for the sandbox.
+    SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, stdAnnos, error);
+    if (error.NotEmpty()) {
+        util_umount_namespace(netnsPath);
+        goto cleanup;
+    }
+
+    // Step 6: Start the sandbox container.
+    StartSandboxContainer(response_id, error);
+    if (error.NotEmpty()) {
+        std::vector<std::string> errlist;
+        if (ClearCniNetwork(response_id, config.linux().security_context().namespace_options().network() ==
+                            runtime::v1alpha2::NamespaceMode::NODE, config.metadata().namespace_(),
+                            config.metadata().name(), errlist, stdAnnos, error) != 0) {
+            Errors tmpErr;
+            tmpErr.SetAggregate(errlist);
+            ERROR("Failed to clear cni network: %s", tmpErr.GetCMessage());
+        }
+        // network namespace is umount in ClearCniNetwork
+        goto cleanup;
+    }
+
+cleanup:
+    free(netnsPath);
+}
+
+void PodSandboxManagerServiceImpl::StartPodSandboxAndSetupNetowrk(const runtime::v1alpha2::PodSandboxConfig &config,
+                                                                  const container_inspect *inspect_data,
+                                                                  std::string &response_id, std::string &jsonCheckpoint,
+                                                                  Errors &error)
+{
+    std::map<std::string, std::string> stdAnnos;
+
+    // Step 5: Start the sandbox container.
+    StartSandboxContainer(response_id, error);
+    if (error.NotEmpty()) {
+        return;
     }
 
-    return;
+    SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, stdAnnos, error);
+    if (error.NotEmpty()) {
+        Errors tmpErr;
+        StopContainerHelper(response_id, tmpErr);
+        if (tmpErr.NotEmpty()) {
+            ERROR("Failed to stop container: %s", tmpErr.GetCMessage());
+        }
+    }
 }
 
 auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config,
@@ -536,7 +607,6 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan
     std::string response_id;
     std::string jsonCheckpoint;
     container_inspect *inspect_data = nullptr;
-    char *netnsPath = nullptr;
 
     if (m_cb == nullptr || m_cb->container.create == nullptr || m_cb->container.start == nullptr) {
         error.SetError("Unimplemented callback");
@@ -576,49 +646,13 @@ auto PodSandboxManagerServiceImpl::RunPodSandbox(const runtime::v1alpha2::PodSan
     }
 
     if (namespace_is_file(inspect_data->host_config->network_mode)) {
-        netnsPath = get_sandbox_key(inspect_data);
-        if (!util_file_exists(netnsPath) || util_mount_namespace(netnsPath) != 0) {
-            error.Errorf("Failed to mount network namespace");
-            ERROR("Failed to mount network namespace");
-            goto cleanup;
-        }
-    }
-
-    // Step 5: Setup networking for the sandbox.
-    if (namespace_is_file(inspect_data->host_config->network_mode)) {
-        SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error);
-        if (error.NotEmpty()) {
-            goto cleanup;
-        }
-    }
-
-    // Step 6: Start the sandbox container.
-    StartSandboxContainer(response_id, error);
-    if (error.NotEmpty()) {
-        goto cleanup;
-    }
-
-    // If netns mode is not file, setup network after start sandbox container
-    if (!namespace_is_file(inspect_data->host_config->network_mode)) {
-        SetupSandboxNetwork(config, response_id, jsonCheckpoint, inspect_data, error);
-        if (error.NotEmpty()) {
-            StopContainerHelper(response_id, error);
-            goto cleanup;
-        }
+        SetupNetowrkAndStartPodSandbox(config, inspect_data, response_id, jsonCheckpoint, error);
+    } else {
+        StartPodSandboxAndSetupNetowrk(config, inspect_data, response_id, jsonCheckpoint, error);
     }
 
 cleanup:
-    if (error.Empty()) {
-        SetNetworkReady(response_id, true, error);
-        DEBUG("set %s ready", response_id.c_str());
-        error.Clear();
-    } else {
-        if (netnsPath != nullptr && remove_network_namespace(netnsPath) != 0) {
-            ERROR("Failed to remove network namespace");
-        }
-    }
     free_container_inspect(inspect_data);
-    free(netnsPath);
     return response_id;
 }
 
diff --git a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h
index f7c0aa00..6b98641e 100644
--- a/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h
+++ b/src/daemon/entry/cri/cri_pod_sandbox_manager_service_impl.h
@@ -83,7 +83,14 @@ private:
     void SetNetworkReady(const std::string &podSandboxID, bool ready, Errors &error);
     void StartSandboxContainer(const std::string &response_id, Errors &error);
     void SetupSandboxNetwork(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id,
-                             const std::string &jsonCheckpoint, const container_inspect *inspect_data, Errors &error);
+                             const std::string &jsonCheckpoint, const container_inspect *inspect_data,
+                             std::map<std::string, std::string> &stdAnnos, Errors &error);
+    void SetupNetowrkAndStartPodSandbox(const runtime::v1alpha2::PodSandboxConfig &config,
+                                        const container_inspect *inspect_data, std::string &response_id,
+                                        std::string &jsonCheckpoint, Errors &error);
+    void StartPodSandboxAndSetupNetowrk(const runtime::v1alpha2::PodSandboxConfig &config,
+                                        const container_inspect *inspect_data, std::string &response_id,
+                                        std::string &jsonCheckpoint, Errors &error);
     void SetupSandboxFiles(const std::string &resolvPath, const runtime::v1alpha2::PodSandboxConfig &config,
                            Errors &error);
     void StopContainerHelper(const std::string &containerID, Errors &error);
diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
index a039ba6b..29b5fc5e 100644
--- a/src/daemon/executor/container_cb/execution_create.c
+++ b/src/daemon/executor/container_cb/execution_create.c
@@ -1477,13 +1477,13 @@ int container_create_cb(const container_create_request *request, container_creat
     if (merge_config_for_syscontainer(request, host_spec, v2_spec->config, oci_spec) != 0) {
         ERROR("Failed to merge config for syscontainer");
         cc = ISULAD_ERR_EXEC;
-        goto umount_shm;
+        goto clean_netns;
     }
 
     if (merge_network(host_spec, request->rootfs, runtime_root, id, container_spec->hostname) != 0) {
         ERROR("Failed to merge network config");
         cc = ISULAD_ERR_EXEC;
-        goto umount_shm;
+        goto clean_netns;
     }
 
     /* modify oci_spec by plugin. */
@@ -1491,14 +1491,14 @@ int container_create_cb(const container_create_request *request, container_creat
         ERROR("Plugin event pre create failed");
         (void)plugin_event_container_post_remove2(id, oci_spec); /* ignore error */
         cc = ISULAD_ERR_EXEC;
-        goto umount_shm;
+        goto clean_netns;
     }
 
     host_channel = dup_host_channel(host_spec->host_channel);
     if (prepare_host_channel(host_channel, host_spec->user_remap)) {
         ERROR("Failed to prepare host channel");
         cc = ISULAD_ERR_EXEC;
-        goto umount_shm;
+        goto clean_netns;
     }
 
     if (verify_container_settings(oci_spec) != 0) {
@@ -1533,6 +1533,10 @@ int container_create_cb(const container_create_request *request, container_creat
 
 umount_channel:
     umount_host_channel(host_channel);
+clean_netns:
+    if (namespace_is_file(host_spec->network_mode) && v2_spec->network_settings != NULL) {
+        (void)remove_network_namespace_file(v2_spec->network_settings->sandbox_key);
+    }
 umount_shm:
     umount_shm_by_configs(host_spec, v2_spec);
 
diff --git a/src/daemon/modules/api/network_namespace_api.h b/src/daemon/modules/api/network_namespace_api.h
index 9a18b1c0..f6201771 100644
--- a/src/daemon/modules/api/network_namespace_api.h
+++ b/src/daemon/modules/api/network_namespace_api.h
@@ -26,6 +26,7 @@ extern "C" {
 
 int prepare_network_namespace(const char *netns_path);
 int remove_network_namespace(const char *netns);
+int remove_network_namespace_file(const char *netns_path);
 char *get_sandbox_key(const container_inspect *inspect_data);
 
 #ifdef __cplusplus
diff --git a/src/daemon/modules/image/oci/storage/image_store/image_store.c b/src/daemon/modules/image/oci/storage/image_store/image_store.c
index aad8329e..daf08c85 100644
--- a/src/daemon/modules/image/oci/storage/image_store/image_store.c
+++ b/src/daemon/modules/image/oci/storage/image_store/image_store.c
@@ -448,7 +448,7 @@ static image_t *by_digest(const char *name)
 {
     digest_image_t *digest_filter_images = NULL;
     char *digest = NULL;
-    
+
     // split digest for image name with digest
     digest = strrchr(name, '@');
     if (digest == NULL || util_reg_match(__DIGESTPattern, digest)) {
diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
index 9adaf613..0f18926a 100644
--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -795,7 +795,7 @@ realexec:
         /* real shim process. */
         close(shim_stderr_pipe[0]);
         close(shim_stdout_pipe[0]);
-        
+
         if (setsid() < 0) {
             (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", id, getpid());
             exit(EXIT_FAILURE);
diff --git a/src/daemon/modules/service/network_namespace_api.c b/src/daemon/modules/service/network_namespace_api.c
index e28e6f74..4cf44b6a 100644
--- a/src/daemon/modules/service/network_namespace_api.c
+++ b/src/daemon/modules/service/network_namespace_api.c
@@ -62,6 +62,28 @@ int remove_network_namespace(const char *netns_path)
     return 0;
 }
 
+int remove_network_namespace_file(const char *netns_path)
+{
+    int get_err = 0;
+
+    if (netns_path == NULL) {
+        ERROR("Invalid netns path");
+        return -1;
+    }
+
+    if (!util_file_exists(netns_path)) {
+        WARN("Namespace file does not exist");
+        return 0;
+    }
+
+    if (!util_force_remove_file(netns_path, &get_err)) {
+        ERROR("Failed to remove file %s, error: %s", netns_path, strerror(get_err));
+        return -1;
+    }
+
+    return 0;
+}
+
 char *get_sandbox_key(const container_inspect *inspect_data)
 {
     char *sandbox_key = NULL;
diff --git a/test/cutils/utils_utils/utils_utils_ut.cc b/test/cutils/utils_utils/utils_utils_ut.cc
index 79583174..0d77820a 100644
--- a/test/cutils/utils_utils/utils_utils_ut.cc
+++ b/test/cutils/utils_utils/utils_utils_ut.cc
@@ -21,7 +21,7 @@ static pid_t test_pid = -1;
 
 extern "C" {
     DECLARE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options));
-    DEFINE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options),(pid, stat_loc, options));
+    DEFINE_WRAPPER_V(waitpid, pid_t, (__pid_t pid, int *stat_loc, int options), (pid, stat_loc, options));
 }
 
 static pid_t waitpid_none_zero(__pid_t pid, int *stat_loc, int options)
-- 
2.25.1

