From 8ba1adc1b19692eb65051adac069ea2a14a62f3d Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Tue, 17 Oct 2023 15:52:11 +0800
Subject: [PATCH 186/198] [refactor] update possible changed resources for oci
 spec

Signed-off-by: haozi007 <liuhao27@huawei.com>
---
 src/cmd/isulad/main.c                         |   5 +
 src/daemon/modules/api/specs_api.h            |   4 +
 .../modules/service/service_container.c       |  68 ++++++++--
 src/daemon/modules/spec/specs.c               |  38 +++++-
 src/daemon/modules/spec/specs_mount.c         | 120 ++++++++++++++++--
 src/daemon/modules/spec/specs_mount.h         |   6 +-
 .../image/oci/oci_config_merge/CMakeLists.txt |   4 +
 7 files changed, 221 insertions(+), 24 deletions(-)

diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
index 05e899f6..1bcef43f 100644
--- a/src/cmd/isulad/main.c
+++ b/src/cmd/isulad/main.c
@@ -1372,6 +1372,11 @@ static int isulad_server_init_common()
         goto out;
     }
 
+    if (spec_module_init() != 0) {
+        ERROR("Failed to init spec module");
+        goto out;
+    }
+
     if (containers_store_init()) {
         ERROR("Failed to init containers store");
         goto out;
diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h
index 0a594d81..f48f0bda 100644
--- a/src/daemon/modules/api/specs_api.h
+++ b/src/daemon/modules/api/specs_api.h
@@ -40,6 +40,10 @@ int parse_security_opt(const host_config *host_spec, bool *no_new_privileges, ch
 int merge_share_namespace(oci_runtime_spec *oci_spec, const host_config *host_spec,
                           const container_config_v2_common_config_network_settings *network_settings);
 
+const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container);
+
+int spec_module_init(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
index 58b27f90..a5c12862 100644
--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
@@ -13,19 +13,11 @@
  * Description: provide container supervisor functions
  ******************************************************************************/
 #define _GNU_SOURCE
-#include <sys/stat.h>
 #include <unistd.h>
 #include <sys/mount.h>
 #include <sys/eventfd.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <isula_libutils/container_config.h>
-#include <isula_libutils/container_config_v2.h>
-#include <isula_libutils/container_exec_request.h>
-#include <isula_libutils/container_exec_response.h>
-#include <isula_libutils/defs.h>
-#include <isula_libutils/host_config.h>
-#include <isula_libutils/oci_runtime_spec.h>
 #include <limits.h>
 #include <pthread.h>
 #include <signal.h>
@@ -35,15 +27,28 @@
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+#include <sys/eventfd.h>
+#include <sys/epoll.h>
+
+#include <isula_libutils/container_config.h>
+#include <isula_libutils/container_config_v2.h>
+#include <isula_libutils/container_exec_request.h>
+#include <isula_libutils/container_exec_response.h>
+#include <isula_libutils/defs.h>
+#include <isula_libutils/host_config.h>
+#include <isula_libutils/oci_runtime_spec.h>
+#include <isula_libutils/log.h>
 
 #include "service_container_api.h"
-#include "isula_libutils/log.h"
 #include "utils.h"
 #include "err_msg.h"
 #include "events_sender_api.h"
 #include "image_api.h"
 #include "specs_api.h"
 #include "specs_mount.h"
+#include "specs_extend.h"
 #include "isulad_config.h"
 #include "verify.h"
 #include "plugin_api.h"
@@ -678,6 +683,43 @@ out:
     epoll_loop_close(&descr);
 }
 
+static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, host_config *hostconfig)
+{
+    char *cgroup_parent = NULL;
+    int ret;
+
+    // If isulad daemon cgroup parent updated, we should update this config into oci spec
+    cgroup_parent = merge_container_cgroups_path(id, hostconfig);
+    if (cgroup_parent == NULL) {
+        return -1;
+    }
+    if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) {
+        free(oci_spec->linux->cgroups_path);
+        oci_spec->linux->cgroups_path = cgroup_parent;
+        cgroup_parent = NULL;
+    }
+    free(cgroup_parent);
+
+    // For Linux.Resources, isula update will save changes into oci spec;
+    // so we just skip it;
+
+    // Remove old devices and update all devices
+    ret = update_devcies_for_oci_spec(oci_spec, hostconfig);
+    if (ret != 0) {
+        ERROR("Failed to do update devices for oci spec");
+        return -1;
+    }
+
+    // If isulad daemon ulimit updated, we should update this config into oci spec.
+    if (merge_global_ulimit(oci_spec) != 0) {
+        return -1;
+    }
+
+    // renew_oci_config() will update process->user and share namespace after.
+
+    return 0;
+}
+
 static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, pid_ppid_info_t *pid_info)
 {
     int ret = 0;
@@ -752,6 +794,14 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
         goto close_exit_fd;
     }
 
+    // Update possible changes
+    nret = do_oci_spec_update(id, oci_spec, cont->hostconfig);
+    if (nret != 0) {
+        ERROR("Failed to update possible changes for oci spec");
+        ret = -1;
+        goto close_exit_fd;
+    }
+
     nret = setup_ipc_dirs(cont->hostconfig, cont->common_config);
     if (nret != 0) {
         ERROR("Failed to setup ipc dirs");
diff --git a/src/daemon/modules/spec/specs.c b/src/daemon/modules/spec/specs.c
index 0c7d58b3..316e9c92 100644
--- a/src/daemon/modules/spec/specs.c
+++ b/src/daemon/modules/spec/specs.c
@@ -17,6 +17,8 @@
 #include <stdio.h>
 #include <stdbool.h>
 #include <errno.h>
+#include <limits.h>
+#include <stdint.h>
 #include <isula_libutils/container_config.h>
 #include <isula_libutils/container_config_v2.h>
 #include <isula_libutils/defs.h>
@@ -76,6 +78,13 @@
 #define CLONE_NEWCGROUP 0x02000000
 #endif
 
+struct readonly_default_oci_spec {
+    oci_runtime_spec *cont;
+    oci_runtime_spec *system_cont;
+};
+
+static struct readonly_default_oci_spec g_rdspec;
+
 static int make_sure_oci_spec_annotations(oci_runtime_spec *oci_spec)
 {
     if (oci_spec->annotations == NULL) {
@@ -1714,7 +1723,12 @@ static int merge_resources_conf(oci_runtime_spec *oci_spec, host_config *host_sp
         goto out;
     }
 
-    ret = merge_conf_device(oci_spec, host_spec);
+    ret = merge_conf_blkio_device(oci_spec, host_spec);
+    if (ret != 0) {
+        goto out;
+    }
+
+    ret = merge_conf_devices(oci_spec, host_spec);
     if (ret != 0) {
         goto out;
     }
@@ -2328,3 +2342,25 @@ out_free:
     free(json_container);
     return ret;
 }
+
+const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container)
+{
+    if (system_container) {
+        return g_rdspec.system_cont;
+    }
+
+    return g_rdspec.cont;
+}
+
+int spec_module_init(void)
+{
+    g_rdspec.cont = default_spec(false);
+    if (g_rdspec.cont == NULL) {
+        return -1;
+    }
+    g_rdspec.system_cont = default_spec(true);
+    if (g_rdspec.system_cont == NULL) {
+        return -1;
+    }
+    return 0;
+}
\ No newline at end of file
diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c
index cd3a5c9d..8bff6cda 100644
--- a/src/daemon/modules/spec/specs_mount.c
+++ b/src/daemon/modules/spec/specs_mount.c
@@ -53,6 +53,7 @@
 #include "image_api.h"
 #include "volume_api.h"
 #include "parse_volume.h"
+#include "specs_api.h"
 
 enum update_rw {
     update_rw_untouch,
@@ -2212,7 +2213,24 @@ out:
     return ret;
 }
 
-int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec)
+int merge_conf_devices(oci_runtime_spec *oci_spec, host_config *host_spec)
+{
+    /* devices which will be populated into container */
+    if (merge_conf_populate_device(oci_spec, host_spec)) {
+        ERROR("Merge user define devices failed");
+        return -1;
+    }
+
+    /* device cgroup rules which will be added into container */
+    if (merge_conf_device_cgroup_rule(oci_spec, host_spec)) {
+        ERROR("Merge user define device cgroup rules failed");
+        return -1;
+    }
+
+    return 0;
+}
+
+int merge_conf_blkio_device(oci_runtime_spec *oci_spec, host_config *host_spec)
 {
     int ret = 0;
 
@@ -2270,18 +2288,6 @@ int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec)
         }
     }
 
-    /* devices which will be populated into container */
-    if (merge_conf_populate_device(oci_spec, host_spec)) {
-        ret = -1;
-        goto out;
-    }
-
-    /* device cgroup rules which will be added into container */
-    if (merge_conf_device_cgroup_rule(oci_spec, host_spec)) {
-        ret = -1;
-        goto out;
-    }
-
 out:
     return ret;
 }
@@ -3488,3 +3494,91 @@ out:
     free(mntparent);
     return ret;
 }
+
+int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig)
+{
+    const oci_runtime_spec *readonly_spec = NULL;
+    size_t i;
+    int ret;
+
+    // Step1: get default oci spec config
+    readonly_spec = get_readonly_default_oci_spec(hostconfig->system_container);
+
+    // Step2: clear oci_spec devices items
+    for (i = 0; i < oci_spec->linux->devices_len; i++) {
+        free_defs_device(oci_spec->linux->devices[i]);
+        oci_spec->linux->devices[i] = NULL;
+    }
+    // Step3: if default devices length more than old spec, just realloc memory
+    if (readonly_spec->linux->devices_len > oci_spec->linux->devices_len) {
+        free(oci_spec->linux->devices);
+        oci_spec->linux->devices = util_smart_calloc_s(sizeof(defs_device *), readonly_spec->linux->devices_len);
+        if (oci_spec->linux->devices == NULL) {
+            oci_spec->linux->devices_len = 0;
+            ERROR("Out of memory");
+            return -1;
+        }
+    }
+    oci_spec->linux->devices_len = 0;
+    // Step4: copy default devices to oci spec
+    for (i = 0; i < readonly_spec->linux->devices_len; i++) {
+        defs_device *tmp_dev = util_common_calloc_s(sizeof(defs_device));
+        if (tmp_dev == NULL) {
+            ERROR("Out of memory");
+            return -1;
+        }
+        tmp_dev->type = util_strdup_s(readonly_spec->linux->devices[i]->type);
+        tmp_dev->path = util_strdup_s(readonly_spec->linux->devices[i]->path);
+        tmp_dev->file_mode = readonly_spec->linux->devices[i]->file_mode;
+        tmp_dev->major = readonly_spec->linux->devices[i]->major;
+        tmp_dev->minor = readonly_spec->linux->devices[i]->minor;
+        tmp_dev->uid = readonly_spec->linux->devices[i]->uid;
+        tmp_dev->gid = readonly_spec->linux->devices[i]->gid;
+        oci_spec->linux->devices[i] = tmp_dev;
+        oci_spec->linux->devices_len += 1;
+    }
+
+    // Step5: clear oci_spec device cgroup rules
+    for (i = 0; i < oci_spec->linux->resources->devices_len; i++) {
+        free_defs_device_cgroup(oci_spec->linux->resources->devices[i]);
+        oci_spec->linux->resources->devices[i] = NULL;
+    }
+    // Step6: if default devices lenght more than old spec, just realloc memory
+    if (readonly_spec->linux->resources->devices_len > oci_spec->linux->resources->devices_len) {
+        free(oci_spec->linux->resources->devices);
+        oci_spec->linux->resources->devices = util_smart_calloc_s(sizeof(defs_device_cgroup *),
+                                                                  readonly_spec->linux->resources->devices_len);
+        if (oci_spec->linux->resources->devices == NULL) {
+            oci_spec->linux->resources->devices_len = 0;
+            ERROR("Out of memory");
+            return -1;
+        }
+    }
+    oci_spec->linux->resources->devices_len = 0;
+    // Step7: copy default device cgroup rules to oci spec
+    for (i = 0; i < readonly_spec->linux->resources->devices_len; i++) {
+        defs_device_cgroup *tmp_dev_cg = util_common_calloc_s(sizeof(defs_device_cgroup));
+        if (tmp_dev_cg == NULL) {
+            ERROR("Out of memory");
+            return -1;
+        }
+        tmp_dev_cg->allow = readonly_spec->linux->resources->devices[i]->allow;
+        tmp_dev_cg->major = readonly_spec->linux->resources->devices[i]->major;
+        tmp_dev_cg->minor = readonly_spec->linux->resources->devices[i]->minor;
+        tmp_dev_cg->type = util_strdup_s(readonly_spec->linux->resources->devices[i]->type);
+        tmp_dev_cg->access = util_strdup_s(readonly_spec->linux->resources->devices[i]->access);
+        oci_spec->linux->resources->devices[i] = tmp_dev_cg;
+        oci_spec->linux->resources->devices_len += 1;
+    }
+
+    // Step8: do update devices and cgroup device rules at here
+    if (hostconfig->privileged) {
+        // Step8.1: for priviledged container, we should merge all devices under /dev
+        ret = merge_all_devices_and_all_permission(oci_spec);
+    } else {
+        // Step8.2: for common container, we should merge devices defined by user in hostconfig
+        ret = merge_conf_devices(oci_spec, hostconfig);
+    }
+
+    return ret;
+}
\ No newline at end of file
diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h
index 8a28f0e2..b742ca35 100644
--- a/src/daemon/modules/spec/specs_mount.h
+++ b/src/daemon/modules/spec/specs_mount.h
@@ -41,10 +41,14 @@ int set_mounts_readwrite_option(const oci_runtime_spec *oci_spec);
 
 int merge_all_devices_and_all_permission(oci_runtime_spec *oci_spec);
 
-int merge_conf_device(oci_runtime_spec *oci_spec, host_config *host_spec);
+int merge_conf_devices(oci_runtime_spec *oci_spec, host_config *host_spec);
+
+int merge_conf_blkio_device(oci_runtime_spec *oci_spec, host_config *host_spec);
 
 int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2_spec);
 
+int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt
index 42cd2e78..d76de35d 100644
--- a/test/image/oci/oci_config_merge/CMakeLists.txt
+++ b/test/image/oci/oci_config_merge/CMakeLists.txt
@@ -31,7 +31,11 @@ add_executable(${EXE}
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/namespace_mock.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/container_unix_mock.cc
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs_mount.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs_extend.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs_security.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume/volume.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/volume/local.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/selinux_label_mock.cc
-- 
2.25.1

