226 lines
11 KiB
Diff
226 lines
11 KiB
Diff
From 5ec7b92cf94a4b7eb4587763aab9f814286bfe27 Mon Sep 17 00:00:00 2001
|
|
From: zhongtao <zhongtao17@huawei.com>
|
|
Date: Thu, 27 Jul 2023 20:02:33 +1400
|
|
Subject: [PATCH 02/11] add health check abnormal test_cases
|
|
|
|
Signed-off-by: zhongtao <zhongtao17@huawei.com>
|
|
---
|
|
CI/install_depends.sh | 1 +
|
|
CI/pr-gateway.sh | 1 +
|
|
CI/test_cases/container_cases/health_check.sh | 90 +++++++++++++++----
|
|
3 files changed, 76 insertions(+), 16 deletions(-)
|
|
|
|
diff --git a/CI/install_depends.sh b/CI/install_depends.sh
|
|
index 88922686..df0474c7 100755
|
|
--- a/CI/install_depends.sh
|
|
+++ b/CI/install_depends.sh
|
|
@@ -91,6 +91,7 @@ check_make_status make_cni_plugins ${build_log_cni_plugins} &
|
|
cd ~
|
|
git clone https://gitee.com/src-openeuler/lxc.git
|
|
cd lxc
|
|
+git checkout origin/openEuler-22.03-LTS-SP2
|
|
tar xf lxc-4.0.3.tar.gz
|
|
cd lxc-4.0.3
|
|
mv ../*.patch .
|
|
diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh
|
|
index b3da52d0..8223ddf5 100755
|
|
--- a/CI/pr-gateway.sh
|
|
+++ b/CI/pr-gateway.sh
|
|
@@ -35,6 +35,7 @@ cd ~
|
|
rm -rf lxc
|
|
git clone https://gitee.com/src-openeuler/lxc.git
|
|
pushd lxc
|
|
+git checkout origin/openEuler-22.03-LTS-SP2
|
|
rm -rf lxc-4.0.3
|
|
./apply-patches || exit 1
|
|
pushd lxc-4.0.3
|
|
diff --git a/CI/test_cases/container_cases/health_check.sh b/CI/test_cases/container_cases/health_check.sh
|
|
index efb357e0..5e6b5641 100755
|
|
--- a/CI/test_cases/container_cases/health_check.sh
|
|
+++ b/CI/test_cases/container_cases/health_check.sh
|
|
@@ -43,18 +43,30 @@ function do_retry()
|
|
return 1
|
|
}
|
|
|
|
-function inspect_container_status()
|
|
+function inspect_container_health_status()
|
|
{
|
|
[[ $(isula inspect -f '{{.State.Health.Status}}' ${1}) == "${2}" ]]
|
|
return $?
|
|
}
|
|
|
|
+function inspect_container_status()
|
|
+{
|
|
+ [[ $(isula inspect -f '{{.State.Status}}' ${1}) == "${2}" ]]
|
|
+ return $?
|
|
+}
|
|
+
|
|
function inspect_container_exitcode()
|
|
{
|
|
[[ $(isula inspect -f '{{.State.ExitCode}}' ${1}) == "${2}" ]]
|
|
return $?
|
|
}
|
|
|
|
+function inspect_container_failing_streak()
|
|
+{
|
|
+ [[ $(isula inspect -f '{{.State.Health.FailingStreak}}' ${1}) == "${2}" ]]
|
|
+ return $?
|
|
+}
|
|
+
|
|
function test_health_check_paraments()
|
|
{
|
|
local ret=0
|
|
@@ -73,29 +85,29 @@ function test_health_check_paraments()
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
|
|
|
|
# start period : 2s => do health check => interval: 2s => do health check => exit on unhealthy
|
|
- [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} running
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not running" && ((ret++))
|
|
|
|
# finish first health check
|
|
sleep 10
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} starting
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} starting
|
|
# keep starting status with health check return non-zero at always until status change to unhealthy
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not starting" && ((ret++))
|
|
|
|
sleep 6 # finish second health check
|
|
|
|
success=1
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} unhealthy
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} unhealthy
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not unhealthy" && ((ret++))
|
|
|
|
# validate --health-retries option
|
|
- [[ $(isula inspect -f '{{.State.Health.FailingStreak}}' ${container_name}) == "2" ]]
|
|
- [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not unhealthy" && ((ret++))
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_failing_streak ${container_name} 2
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check failing streak: not 2" && ((ret++))
|
|
|
|
- [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "exited" ]]
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} exited
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not exited" && ((ret++))
|
|
|
|
- [[ $(isula inspect -f '{{.State.ExitCode}}' ${container_name}) == "137" ]]
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_exitcode ${container_name} 137
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container exit code: not 137" && ((ret++))
|
|
|
|
isula rm -f ${container_name}
|
|
@@ -123,24 +135,24 @@ function test_health_check_normally()
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
|
|
|
|
# start period : 0s => interval: 2s => do health check => interval: 2s => do health check => ...
|
|
- [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} running
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not running" && ((ret++))
|
|
|
|
# Health check has been performed yet
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} starting
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} starting
|
|
# Initial status when the container is still starting
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not starting" && ((ret++))
|
|
|
|
sleep 8 # finish first health check
|
|
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} healthy
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} healthy
|
|
# When the health check returns successfully, status immediately becomes healthy
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not healthy" && ((ret++))
|
|
|
|
kill -9 $(isula inspect -f '{{.State.Pid}}' ${container_name})
|
|
|
|
# Wait for the container to be killed
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} unhealthy
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} unhealthy
|
|
# The container process exits abnormally and the health check status becomes unhealthy
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not unhealthy" && ((ret++))
|
|
|
|
@@ -174,17 +186,17 @@ function test_health_check_timeout()
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
|
|
|
|
# start period : 0s => interval: 5s => do health check(1s timeout) => unhealthy(exited)
|
|
- [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} running
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not running" && ((ret++))
|
|
|
|
# Health check has been performed yet
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} starting
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} starting
|
|
# Initial status when the container is still starting
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not starting" && ((ret++))
|
|
|
|
sleep 7 # finish first health check
|
|
|
|
- do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} unhealthy
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} unhealthy
|
|
# The container process exits and the health check status becomes unhealthy
|
|
[[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not unhealthy" && ((ret++))
|
|
|
|
@@ -229,6 +241,50 @@ function test_health_check_monitor()
|
|
return ${ret}
|
|
}
|
|
|
|
+function test_health_check_abnormal()
|
|
+{
|
|
+ local ret=0
|
|
+ local image="busybox"
|
|
+ local retry_limit=10
|
|
+ local retry_interval=1
|
|
+ local test="health check abnormal => (${FUNCNAME[@]})"
|
|
+
|
|
+ msg_info "${test} starting..."
|
|
+
|
|
+ isula images | grep ${image}
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
|
|
+
|
|
+ container_name="health_check_abnormal"
|
|
+ isula run -itd --runtime $1 -n ${container_name} --health-cmd="sleep 999" --health-timeout=1000000s --health-retries=1 ${image} /bin/sh
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
|
|
+
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} running
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not running" && ((ret++))
|
|
+
|
|
+ # Health check has been performed yet
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} starting
|
|
+ # Initial status when the container is still starting
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not starting" && ((ret++))
|
|
+
|
|
+ sleep 30 # wait health check exec
|
|
+
|
|
+ isula stop -t 0 ${container_name} &
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop container: ${container_name}" && ((ret++))
|
|
+
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_status ${container_name} exited
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container status: not exited" && ((ret++))
|
|
+
|
|
+ do_retry ${retry_limit} ${retry_interval} inspect_container_health_status ${container_name} unhealthy
|
|
+ # The container process exits and the health check status becomes unhealthy
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - incorrent container health check status: not unhealthy" && ((ret++))
|
|
+
|
|
+ isula rm -f ${container_name}
|
|
+ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove container: ${container_name}" && ((ret++))
|
|
+
|
|
+ msg_info "${test} finished with return ${ret}..."
|
|
+ return ${ret}
|
|
+}
|
|
+
|
|
declare -i ans=0
|
|
|
|
for element in ${RUNTIME_LIST[@]};
|
|
@@ -244,7 +300,9 @@ do
|
|
|
|
test_health_check_monitor $element || ((ans++))
|
|
|
|
+ test_health_check_abnormal $element || ((ans++))
|
|
+
|
|
msg_info "${test} finished with return ${ans}..."
|
|
done
|
|
|
|
-show_result ${ans} "${curr_path}/${0}"
|
|
+show_result ${ans} "${curr_path}/${0}"
|
|
\ No newline at end of file
|
|
--
|
|
2.25.1
|
|
|