From c70e4aaaf5b1748432fd325856e07770a267336e Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Tue, 6 Jun 2023 11:50:26 +0000
Subject: [PATCH 09/15] !2046 reinforce cri_stream.sh and health_check.sh *
 reinforce cri_stream.sh and health_check.sh

---
 CI/test_cases/container_cases/cri_stream.sh   |  35 +++++-
 CI/test_cases/container_cases/health_check.sh | 103 +++++++++++++-----
 2 files changed, 103 insertions(+), 35 deletions(-)

diff --git a/CI/test_cases/container_cases/cri_stream.sh b/CI/test_cases/container_cases/cri_stream.sh
index 8b5440d3..2360e240 100755
--- a/CI/test_cases/container_cases/cri_stream.sh
+++ b/CI/test_cases/container_cases/cri_stream.sh
@@ -9,6 +9,27 @@ data_path=$(realpath $curr_path/criconfigs)
 pause_img_path=$(realpath $curr_path/test_data)
 source ../helpers.sh
 
+# $1 : retry limit
+# $2 : retry_interval
+# $3 : retry function
+function do_retry()
+{
+    for i in $(seq 1 "$1"); do
+        $3
+        if [ $? -ne 0 ]; then
+            return 0
+        fi
+        sleep $2
+    done
+    return 1
+}
+
+function get_ioCopy()
+{
+    ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
+    return $?
+}
+
 function do_pre()
 {
     local ret=0
@@ -58,6 +79,8 @@ function set_up()
 function test_cri_exec_fun()
 {
     local ret=0
+    local retry_limit=20
+    local retry_interval=1
     local test="test_cri_exec_fun => (${FUNCNAME[@]})"
     msg_info "${test} starting..."
     declare -a fun_pids
@@ -74,9 +97,8 @@ function test_cri_exec_fun()
     done
     wait ${abn_pids[*]// /|}
 
-    sleep 2
-    ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
-    [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
+    do_retry ${retry_limit} ${retry_interval} get_ioCopy
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
 
     msg_info "${test} finished with return ${ret}..."
     return ${ret}
@@ -85,6 +107,8 @@ function test_cri_exec_fun()
 function test_cri_exec_abn
 {
     local ret=0
+    local retry_limit=20
+    local retry_interval=1
     local test="test_cri_exec_abn => (${FUNCNAME[@]})"
     msg_info "${test} starting..."
 
@@ -92,10 +116,9 @@ function test_cri_exec_abn
     pid=$!
     sleep 3
     kill -9 $pid
-    sleep 2
 
-    ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
-    [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
+    do_retry ${retry_limit} ${retry_interval} get_ioCopy
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
 
     msg_info "${test} finished with return ${ret}..."
     return ${ret}
diff --git a/CI/test_cases/container_cases/health_check.sh b/CI/test_cases/container_cases/health_check.sh
index b8256087..efb357e0 100755
--- a/CI/test_cases/container_cases/health_check.sh
+++ b/CI/test_cases/container_cases/health_check.sh
@@ -26,9 +26,40 @@ image="busybox"
 isula pull ${image}
 [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" &&  exit ${FAILURE}
 
+# $1 : retry limit
+# $2 : retry_interval
+# $3 : retry function
+# $4 : retry function parms 1
+# $5 : retry function parms 2
+function do_retry()
+{
+    for i in $(seq 1 "$1"); do
+        $3 $4 $5
+        if [ $? -eq 0 ]; then
+            return 0
+        fi
+        sleep $2
+    done
+    return 1
+}
+
+function inspect_container_status()
+{
+    [[ $(isula inspect -f '{{.State.Health.Status}}' ${1}) == "${2}" ]]
+    return $?
+}
+
+function inspect_container_exitcode()
+{
+    [[ $(isula inspect -f '{{.State.ExitCode}}' ${1}) == "${2}" ]]
+    return $?
+}
+
 function test_health_check_paraments()
 {
     local ret=0
+    local retry_limit=10
+    local retry_interval=1
     local test="list && inspect image info test => (${FUNCNAME[@]})"
 
     msg_info "${test} starting..."
@@ -37,7 +68,7 @@ function test_health_check_paraments()
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
 
     container_name="health_check_para"
-    isula run -itd -n ${container_name} --health-cmd 'echo "iSulad" ; exit 1' \
+    isula run -itd --runtime $1 -n ${container_name} --health-cmd 'echo "iSulad" ; exit 1' \
         --health-interval 5s --health-retries 2 --health-start-period 8s --health-exit-on-unhealthy ${image} /bin/sh
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
@@ -45,15 +76,16 @@ function test_health_check_paraments()
     [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
-    sleep 13 # finish first health check
-
+    # finish first health check
+    sleep 10
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} starting
     # keep starting status with health check return non-zero at always until status change to unhealthy
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "starting" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
 
     sleep 6 # finish second health check
 
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "unhealthy" ]]
+    success=1
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} unhealthy
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
     # validate --health-retries option
@@ -77,6 +109,8 @@ function test_health_check_normally()
 {
     local ret=0
     local image="busybox"
+    local retry_limit=10
+    local retry_interval=1
     local test="list && inspect image info test => (${FUNCNAME[@]})"
 
     msg_info "${test} starting..."
@@ -85,31 +119,32 @@ function test_health_check_normally()
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
 
     container_name="health_check_normally"
-    isula run -itd -n ${container_name} --health-cmd 'date' --health-interval 5s ${image} /bin/sh
+    isula run -itd --runtime $1 -n ${container_name} --health-cmd 'date' --health-interval 5s ${image} /bin/sh
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
     # start period : 0s => interval: 2s => do health check => interval: 2s => do health check => ...
     [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
-    sleep 2 # Health check has been performed yet
-
+    # Health check has been performed yet
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} starting
     # Initial status when the container is still starting
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "starting" ]]
-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
+    [[ $? -ne 0 ]]  && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
 
     sleep 8 # finish first health check
+    
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} healthy
     # When the health check returns successfully, status immediately becomes healthy
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "healthy" ]]
-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not healthy" && ((ret++))
-
-    kill -9 $(isula inspect -f '{{.State.Pid}}' ${container_name}) && sleep 1 # Wait for the container to be killed
+    [[ $? -ne 0 ]]  && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not healthy" && ((ret++))
 
+    kill -9 $(isula inspect -f '{{.State.Pid}}' ${container_name})
+    
+    # Wait for the container to be killed
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} unhealthy
     # The container process exits abnormally and the health check status becomes unhealthy
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "unhealthy" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
-    [[ $(isula inspect -f '{{.State.ExitCode}}' ${container_name}) == "137" ]]
+    do_retry ${retry_limit} ${retry_interval} inspect_container_exitcode  ${container_name} 137
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container exit code: not 137" && ((ret++))
 
     isula rm -f ${container_name}
@@ -123,6 +158,9 @@ function test_health_check_timeout()
 {
     local ret=0
     local image="busybox"
+    local retry_limit=10
+    local retry_interval=1
+    local success=1
     local test="list && inspect image info test => (${FUNCNAME[@]})"
 
     msg_info "${test} starting..."
@@ -131,7 +169,7 @@ function test_health_check_timeout()
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
 
     container_name="health_check_timeout"
-    isula run -itd -n ${container_name} --health-cmd 'sleep 5' --health-interval 5s --health-timeout 1s \
+    isula run -itd --runtime $1 -n ${container_name} --health-cmd 'sleep 5' --health-interval 5s --health-timeout 1s \
         --health-retries 1 --health-exit-on-unhealthy ${image} /bin/sh
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
@@ -139,19 +177,19 @@ function test_health_check_timeout()
     [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
-    sleep 1 # Health check has been performed yet
-
+    # Health check has been performed yet
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} starting
     # Initial status when the container is still starting
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "starting" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
 
     sleep 7 # finish first health check
+    
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} unhealthy
     # The container process exits and the health check status becomes unhealthy
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "unhealthy" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
-    [[ $(isula inspect -f '{{.State.ExitCode}}' ${container_name}) == "137" ]]
-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container exit code: not 137" && ((ret++))
+    do_retry ${retry_limit} ${retry_interval} inspect_container_exitcode  ${container_name} 137
+    [[ $? -ne 0 ]]  && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container exit code: not 137" && ((ret++))
 
     isula rm -f ${container_name}
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove container: ${container_name}" && ((ret++))
@@ -174,7 +212,7 @@ function test_health_check_monitor()
     isula rm -f $(isula ps -qa)
 
     container_name="health_check_monitor"
-    isula run -itd -n ${container_name} --health-cmd="sleep 3" --health-interval 3s  busybox
+    isula run -itd --runtime $1 -n ${container_name} --health-cmd="sleep 3" --health-interval 3s  busybox
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
     isula stop -t 0 ${container_name} && isula start ${container_name} && \
@@ -193,13 +231,20 @@ function test_health_check_monitor()
 
 declare -i ans=0
 
-test_health_check_paraments || ((ans++))
+for element in ${RUNTIME_LIST[@]};
+do
+    test="health check test => (${element})"
+    msg_info "${test} starting..."
 
-test_health_check_normally || ((ans++))
+    test_health_check_paraments $element || ((ans++))
 
-test_health_check_timeout || ((ans++))
+    test_health_check_normally $element || ((ans++))
 
-test_health_check_monitor || ((ans++))
+    test_health_check_timeout $element || ((ans++))
 
-show_result ${ans} "${curr_path}/${0}"
+    test_health_check_monitor $element || ((ans++))
 
+    msg_info "${test} finished with return ${ans}..."
+done
+
+show_result ${ans} "${curr_path}/${0}"
-- 
2.25.1

