packages/aops-zeus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init aops-zeus in 20.03 SP4

Browse Source
This commit is contained in:
liulei 2023-11-06 15:45:48 +08:00
parent 469b1f357e
commit 0308b441eb
15 changed files with 1716 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
0001-add-file-sync-func.patch
View File

@ -1,141 +0,0 @@
From 0e2ad3cc141390e26d7a20bccf5ccc0660f7c172 Mon Sep 17 00:00:00 2001
From: smjiao <smjiao@isoftstone.com>
Date: Thu, 7 Sep 2023 16:24:42 +0800
Subject: [PATCH] add file sync func
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/conf/constant.py | 2 ++
zeus/config_manager/view.py | 45 ++++++++++++++++++++++++++++++++--
zeus/function/verify/config.py | 9 +++++++
zeus/url.py | 6 ++++-
4 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index 2fbf1be..3175c65 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -33,6 +33,7 @@ CERES_CVE_REPO_SET = "aops-ceres apollo --set-repo '%s'"
CERES_CVE_SCAN = "aops-ceres apollo --scan '%s'"
CERES_CVE_FIX = "aops-ceres apollo --fix '%s'"
CERES_CVE_ROLLBACK = "aops-ceres apollo --rollback '%s'"
+CERES_SYNC_CONF = "aops-ceres sync --conf '%s'"
# zeus route
ADD_HOST = "/manage/host/add"
@@ -54,6 +55,7 @@ DELETE_GROUP = "/manage/host/group/delete"
GET_GROUP = "/manage/host/group/get"
COLLECT_CONFIG = '/manage/config/collect'
+SYNC_CONFIG = '/manage/config/sync'
USER_LOGIN = "/manage/account/login"
LOGOUT = "/manage/account/logout"
diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py
index cd83e35..baeef7e 100644
--- a/zeus/config_manager/view.py
+++ b/zeus/config_manager/view.py
@@ -21,10 +21,11 @@ from typing import List, Dict
from vulcanus.multi_thread_handler import MultiThreadHandler
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
-from zeus.conf.constant import CERES_COLLECT_FILE
+from zeus.conf import configuration
+from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF
from zeus.database.proxy.host import HostProxy
from zeus.function.model import ClientConnectArgs
-from zeus.function.verify.config import CollectConfigSchema
+from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema
from zeus.host_manager.ssh import execute_command_and_parse_its_result
@@ -218,3 +219,43 @@ class CollectConfig(BaseResponse):
return self.response(
state.SUCCEED, None, self.generate_target_data_format(multi_thread.get_result(), host_id_with_config_file)
)
+
+
+class SyncConfig(BaseResponse):
+
+ @staticmethod
+ def sync_config_content(host_info: Dict, sync_config_info: Dict):
+ command = CERES_SYNC_CONF % json.dumps(sync_config_info)
+ status, content = execute_command_and_parse_its_result(
+ ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"),
+ host_info.get("ssh_user"), host_info.get("pkey")), command)
+ return status
+
+ @BaseResponse.handle(schema=SyncConfigSchema, token=False)
+ def put(self, **params):
+
+ sync_config_info = dict()
+ sync_config_info['file_path'] = params.get('file_path')
+ sync_config_info['content'] = params.get('content')
+
+ sync_result = {
+ "file_path": sync_config_info['file_path'],
+ "sync_result": False
+ }
+
+ # Query host address from database
+ proxy = HostProxy(configuration)
+ if not proxy.connect():
+ return self.response(code=state.DATABASE_CONNECT_ERROR, data={"resp": sync_result})
+
+ status, host_list = proxy.get_host_info(
+ {"username": "admin", "host_list": [params.get('host_id')]}, True)
+ if status != state.SUCCEED or len(host_list) == 1:
+ return self.response(code=status, data={"resp": sync_result})
+
+ host_info = host_list[0]
+ status = self.sync_config_content(host_info, sync_config_info)
+ if status == state.SUCCEED:
+ sync_result['sync_result'] = True
+ return self.response(code=state.SUCCEED, data={"resp": sync_result})
+ return self.response(code=state.UNKNOWN_ERROR, data={"resp": sync_result})
diff --git a/zeus/function/verify/config.py b/zeus/function/verify/config.py
index 230b65d..6e5bf64 100644
--- a/zeus/function/verify/config.py
+++ b/zeus/function/verify/config.py
@@ -36,3 +36,12 @@ class CollectConfigSchema(Schema):
"""
infos = fields.List(fields.Nested(ConfigSchema(), required=True), required=True, validate=lambda s: len(s) > 0)
+
+
+class SyncConfigSchema(Schema):
+ """
+ validators for SyncConfigSchema
+ """
+ host_id = fields.Integer(required=True, validate=lambda s: s > 0)
+ file_path = fields.String(required=True, validate=lambda s: len(s) > 0)
+ content = fields.String(required=True, validate=lambda s: len(s) > 0)
diff --git a/zeus/url.py b/zeus/url.py
index e0cf1de..597dcc7 100644
--- a/zeus/url.py
+++ b/zeus/url.py
@@ -49,6 +49,7 @@ from zeus.conf.constant import (
REFRESH_TOKEN,
UPDATE_HOST,
USER_LOGIN,
+ SYNC_CONFIG,
)
from zeus.config_manager import view as config_view
from zeus.host_manager import view as host_view
@@ -83,7 +84,10 @@ SPECIFIC_URLS = {
(host_view.DeleteHostGroup, DELETE_GROUP),
(host_view.GetHostGroup, GET_GROUP),
],
- "CONFIG_URLS": [(config_view.CollectConfig, COLLECT_CONFIG)],
+ "CONFIG_URLS": [
+ (config_view.CollectConfig, COLLECT_CONFIG),
+ (config_view.SyncConfig, SYNC_CONFIG)
+ ],
'AGENT_URLS': [
(agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO),
(agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET),
--
Gitee

46
0001-fix-metric-proxy-init-failed-error.patch Normal file
View File

@ -0,0 +1,46 @@
From 833a9d721bed5b3e64ea49710a477a52b74c5255 Mon Sep 17 00:00:00 2001
From: zhangdaolong <dlzhangak@isoftstone.com>
Date: Thu, 12 Oct 2023 14:28:06 +0800
Subject: [PATCH 1/1] fix metric proxy init failed error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/database/proxy/metric.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py
index e899e26..5fa75e9 100644
--- a/zeus/database/proxy/metric.py
+++ b/zeus/database/proxy/metric.py
@@ -21,6 +21,7 @@ from prometheus_api_client import PrometheusApiClientException
from vulcanus.database.proxy import PromDbProxy
from vulcanus.log.log import LOGGER
from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED
+from zeus.conf import configuration
class MetricProxy(PromDbProxy):
@@ -28,16 +29,15 @@ class MetricProxy(PromDbProxy):
Proxy of prometheus time series database
"""
- def __init__(self, configuration, host=None, port=None):
+ def __init__(self, host=None, port=None):
"""
Init MetricProxy
Args:
- configuration (Config)
host (str)
port (int)
"""
- PromDbProxy.__init__(self, configuration, host, port)
+ PromDbProxy.__init__(self, host, port)
self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100
self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s"
--
2.33.0

298
0002-add-key-authentication-for-add-host-api.patch Normal file
View File

@ -0,0 +1,298 @@
From 7a8164696bb913a75cf79cf6b57c9973530efefa Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Sun, 15 Oct 2023 16:37:55 +0800
Subject: [PATCH 1/1] add a way about key authentication for add host api
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
database/zeus.sql | 2 +-
zeus/conf/constant.py | 6 +--
zeus/database/table.py | 2 +-
zeus/function/verify/host.py | 4 +-
zeus/host_manager/view.py | 99 ++++++++++++++++++++++++++++--------
5 files changed, 85 insertions(+), 28 deletions(-)
diff --git a/database/zeus.sql b/database/zeus.sql
index 3dc9f3c..7db734e 100644
--- a/database/zeus.sql
+++ b/database/zeus.sql
@@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS `host` (
`os_version` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL,
`ssh_user` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL,
`ssh_port` int(11) NULL DEFAULT NULL,
- `pkey` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL,
+ `pkey` varchar(4096) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL,
`status` int(11) NULL DEFAULT NULL,
`user` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL,
`host_group_id` int(11) NULL DEFAULT NULL,
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index 3175c65..bf8792a 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -90,9 +90,9 @@ CHECK_IDENTIFY_SCENE = "/check/scene/identify"
CHECK_WORKFLOW_HOST_EXIST = '/check/workflow/host/exist'
# host template file content
-HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,host_name,host_group_name,management
-test_ip_1,22,root,password,test_host,test_host_group,False
-test_ip_2,22,root,password,test_host,test_host_group,False
+HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,ssh_pkey,host_name,host_group_name,management
+test_ip_1,22,root,password,ssh_pkey,test_host,test_host_group,False
+test_ip_2,22,root,password,ssh_pkey,test_host,test_host_group,False
"""
diff --git a/zeus/database/table.py b/zeus/database/table.py
index 9596492..265eb45 100644
--- a/zeus/database/table.py
+++ b/zeus/database/table.py
@@ -59,7 +59,7 @@ class Host(Base, MyBase): # pylint: disable=R0903
os_version = Column(String(40))
ssh_user = Column(String(40), default="root")
ssh_port = Column(Integer(), default=22)
- pkey = Column(String(2048))
+ pkey = Column(String(4096))
status = Column(Integer(), default=2)
user = Column(String(40), ForeignKey('user.username'))
diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py
index b054d62..d09eedd 100644
--- a/zeus/function/verify/host.py
+++ b/zeus/function/verify/host.py
@@ -103,11 +103,12 @@ class AddHostSchema(Schema):
"""
ssh_user = fields.String(required=True, validate=lambda s: len(s) > 0)
- password = fields.String(required=True, validate=lambda s: len(s) > 0)
+ password = fields.String(required=True, allow_none=True, validate=lambda s: len(s) >= 0)
host_name = fields.String(
required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check]
)
host_ip = fields.IP(required=True)
+ ssh_pkey = fields.String(required=True, allow_none=True, validate=lambda s: 4096 >= len(s) >= 0)
ssh_port = fields.Integer(required=True, validate=lambda s: 65535 >= s > 0)
host_group_name = fields.String(required=True, validate=lambda s: len(s) > 0)
management = fields.Boolean(required=True)
@@ -133,3 +134,4 @@ class UpdateHostSchema(Schema):
host_name = fields.String(required=False, validate=lambda s: len(s) > 0)
host_group_name = fields.String(required=False, validate=lambda s: len(s) > 0)
management = fields.Boolean(required=False)
+ ssh_pkey = fields.String(required=False, validate=lambda s: 4096 >= len(s) >= 0)
diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py
index 768d2cd..95e1434 100644
--- a/zeus/host_manager/view.py
+++ b/zeus/host_manager/view.py
@@ -16,12 +16,13 @@ Author:
Description: Restful APIs for host
"""
import json
-from io import BytesIO
+from io import BytesIO, StringIO
from typing import Iterable, List, Tuple, Union
import socket
import gevent
import paramiko
+from paramiko.ssh_exception import SSHException
from flask import request, send_file
from marshmallow import Schema
from marshmallow.fields import Boolean
@@ -333,7 +334,8 @@ class AddHost(BaseResponse):
"host_ip":"127.0.0.1",
"ssh_port":"22",
"management":false,
- "username": "admin"
+ "username": "admin",
+ "ssh_pkey": "RSA key"
}
Returns:
@@ -363,6 +365,7 @@ class AddHost(BaseResponse):
"ssh_port": host_info.get("ssh_port"),
"user": host_info.get("username"),
"management": host_info.get("management"),
+ "pkey": host_info.get("ssh_pkey"),
}
)
if host in hosts:
@@ -384,7 +387,8 @@ class AddHost(BaseResponse):
"host_ip":"127.0.0.1",
"ssh_port":"22",
"management":false,
- "username": "admin"
+ "username": "admin",
+ "ssh_pkey": "RSA key"
}
Returns:
@@ -396,15 +400,55 @@ class AddHost(BaseResponse):
if status != state.SUCCEED:
return self.response(code=status)
- status, private_key = save_ssh_public_key_to_client(
- params.get('host_ip'), params.get('ssh_port'), params.get('ssh_user'), params.get('password')
- )
- if status == state.SUCCEED:
- host.pkey = private_key
- host.status = HostStatus.ONLINE
+ if params.get("ssh_pkey"):
+ status = verify_ssh_login_info(
+ ClientConnectArgs(
+ params.get("host_ip"), params.get("ssh_port"), params.get("ssh_user"), params.get("ssh_pkey")
+ )
+ )
+ host.status = HostStatus.ONLINE if status == state.SUCCEED else HostStatus.UNESTABLISHED
+ else:
+ status, private_key = save_ssh_public_key_to_client(
+ params.get('host_ip'), params.get('ssh_port'), params.get('ssh_user'), params.get('password')
+ )
+ if status == state.SUCCEED:
+ host.pkey = private_key
+ host.status = HostStatus.ONLINE
return self.response(code=self.proxy.add_host(host))
+def verify_ssh_login_info(ssh_login_info: ClientConnectArgs) -> str:
+ """
+ Verify that the ssh login information is correct
+
+ Args:
+ ssh_login_info(ClientConnectArgs): e.g
+ ClientConnectArgs(host_ip='127.0.0.1', ssh_port=22, ssh_user='root', pkey=RSAKey string)
+
+ Returns:
+ status code
+ """
+ try:
+ client = SSH(
+ ip=ssh_login_info.host_ip,
+ username=ssh_login_info.ssh_user,
+ port=ssh_login_info.ssh_port,
+ pkey=paramiko.RSAKey.from_private_key(StringIO(ssh_login_info.pkey)),
+ )
+ client.close()
+ except socket.error as error:
+ LOGGER.error(error)
+ return state.SSH_CONNECTION_ERROR
+ except SSHException as error:
+ LOGGER.error(error)
+ return state.SSH_AUTHENTICATION_ERROR
+ except Exception as error:
+ LOGGER.error(error)
+ return state.SSH_CONNECTION_ERROR
+
+ return state.SUCCEED
+
+
def save_ssh_public_key_to_client(ip: str, port: int, username: str, password: str) -> tuple:
"""
generate RSA key pair,save public key to the target host machine
@@ -465,7 +509,7 @@ class GetHostTemplateFile(BaseResponse):
file = BytesIO()
file.write(HOST_TEMPLATE_FILE_CONTENT.encode('utf-8'))
file.seek(0)
- response = send_file(file,mimetype="application/octet-stream")
+ response = send_file(file, mimetype="application/octet-stream")
response.headers['Content-Disposition'] = 'attachment; filename=template.csv'
return response
@@ -574,6 +618,7 @@ class AddHostBatch(BaseResponse):
continue
password = host_info.pop("password")
+ pkey = host_info.pop("ssh_pkey", None)
host_info.update(
{"host_group_id": group_id_info.get(host_info['host_group_name']), "user": data["username"]}
)
@@ -585,7 +630,7 @@ class AddHostBatch(BaseResponse):
)
continue
- valid_host.append((host, password))
+ valid_host.append((host, password, pkey))
return valid_host
def save_key_to_client(self, host_connect_infos: List[tuple]) -> list:
@@ -598,8 +643,8 @@ class AddHostBatch(BaseResponse):
Returns:
host object list
"""
- # 30 connections are created at a time.
- tasks = [host_connect_infos[index : index + 30] for index in range(0, len(host_connect_infos), 30)]
+ # 100 connections are created at a time.
+ tasks = [host_connect_infos[index : index + 100] for index in range(0, len(host_connect_infos), 100)]
result = []
for task in tasks:
@@ -612,18 +657,23 @@ class AddHostBatch(BaseResponse):
return result
@staticmethod
- def update_rsa_key_to_host(host: Host, password: str) -> Host:
+ def update_rsa_key_to_host(host: Host, password: str = None, pkey: str = None) -> Host:
"""
save ssh public key to client and update its private key in host
Args:
host(Host): host object
password(str): password for ssh login
+ pkey(str): rsa key for ssh login
Returns:
host object
"""
- status, pkey = save_ssh_public_key_to_client(host.host_ip, host.ssh_port, host.ssh_user, password)
+ if pkey:
+ status = verify_ssh_login_info(ClientConnectArgs(host.host_ip, host.ssh_port, host.ssh_user, pkey))
+ else:
+ status, pkey = save_ssh_public_key_to_client(host.host_ip, host.ssh_port, host.ssh_user, password)
+
if status == state.SUCCEED:
host.status = HostStatus.ONLINE
host.pkey = pkey
@@ -654,7 +704,7 @@ class AddHostBatch(BaseResponse):
new_host.update(update_info)
self.add_result.append(new_host)
else:
- for host, _ in hosts:
+ for host, _, _ in hosts:
new_host = {
"host_ip": host.host_ip,
"ssh_port": host.ssh_port,
@@ -789,9 +839,14 @@ class UpdateHost(BaseResponse):
"""
ssh_user = params.get("ssh_user") or self.host.ssh_user
ssh_port = params.get("ssh_port") or self.host.ssh_port
- status, private_key = save_ssh_public_key_to_client(
- self.host.host_ip, ssh_port, ssh_user, params.pop("password", None)
- )
+ private_key = params.pop("ssh_pkey", None)
+ if private_key:
+ status = verify_ssh_login_info(ClientConnectArgs(self.host.host_ip, ssh_port, ssh_user, private_key))
+ else:
+ status, private_key = save_ssh_public_key_to_client(
+ self.host.host_ip, ssh_port, ssh_user, params.pop("password", None)
+ )
+
params.update(
{
"ssh_user": ssh_user,
@@ -876,10 +931,10 @@ class UpdateHost(BaseResponse):
return self.response(code=state.PARAM_ERROR, message="there is a duplicate host ssh address in database!")
if params.get("ssh_user") or params.get("ssh_port"):
- if not params.get("password"):
- return self.response(code=state.PARAM_ERROR, message="please update password")
+ if not params.get("password") or not params.get("ssh_pkey"):
+ return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.")
self._save_ssh_key(params)
- elif params.get("password"):
+ elif params.get("password") or params.get("ssh_pkey"):
self._save_ssh_key(params)
return self.response(callback.update_host_info(params.pop("host_id"), params))
--
2.33.0

68
0002-update-callback-request-headers.patch
View File

@ -1,68 +0,0 @@
From 51be015c84619925873ff377a72827b7c9770632 Mon Sep 17 00:00:00 2001
From: rabbitali <shusheng.wen@outlook.com>
Date: Mon, 18 Sep 2023 12:18:52 +0800
Subject: [PATCH 1/1] update callback request headers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/vulnerability_manage/view.py | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py
index 1bfed63..b5360e2 100644
--- a/zeus/vulnerability_manage/view.py
+++ b/zeus/vulnerability_manage/view.py
@@ -119,7 +119,7 @@ def generate_tasks(tasks: list, host_infos: dict, **kwargs: dict) -> list:
class BaseExcuteTask:
def __init__(self) -> None:
- self._header = {'content-type': 'application/json', 'access_token': request.headers.get('access_token')}
+ self._header = {"exempt_authentication": configuration.individuation.get("EXEMPT_AUTHENTICATION")}
class ExecuteRepoSetTask(BaseResponse, BaseExcuteTask):
@@ -234,7 +234,7 @@ class ExecuteRepoSetTask(BaseResponse, BaseExcuteTask):
return self.response(code=status_code)
self._callback_url = params.get('callback')
- self._header = {'content-type': 'application/json', 'access_token': request.headers.get('access_token')}
+ self._header["local_account"] = params.get("username")
self._task_id = params.get("task_id")
self._task_name = params.get("task_name")
self._task_type = params.get("task_type")
@@ -337,13 +337,7 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask):
return self.response(code=status_code)
# parse args
self._callback_url = params.get('callback')
- if params.get("timed"):
- self._header.update(
- {
- "exempt_authentication": configuration.individuation.get("EXEMPT_AUTHENTICATION"),
- "local_account": params.get("username"),
- }
- )
+ self._header["local_account"] = params.get("username")
self._task_id = params.get("task_id")
self._check_items = params.get('check_items')
# Execute task
@@ -465,6 +459,7 @@ class ExecuteCveFixTask(BaseResponse, BaseExcuteTask):
self._task_id = params.get("task_id")
self._task_name = params.get("task_name")
self._task_type = params.get("task_type")
+ self._header["local_account"] = params.get("username")
self._accepted = params.get('accepted')
self._takeover = params.get('takeover')
self._check_items = params.get('check_items')
@@ -582,6 +577,7 @@ class ExecuteCveRollbackTask(BaseResponse, BaseExcuteTask):
# Generate tasks
self._callback_url = params.get('callback')
self._task_id = params.get("task_id")
+ self._header["local_account"] = params.get("username")
self._task_name = params.get("task_name")
self._task_type = params.get("task_type")
--
2.33.1.windows.1

220
0003-fix-bash-file-sync-error.patch
View File

@ -1,220 +0,0 @@
From 80272352b3067ebb2cb3011cfbeeef5e9d464fa6 Mon Sep 17 00:00:00 2001
From: smjiao <smjiao@isoftstone.com>
Date: Mon, 18 Sep 2023 20:37:50 +0800
Subject: [PATCH 1/1] fix bash file sync error
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/config_manager/view.py | 48 ++++++++++++++++++++-----
zeus/host_manager/ssh.py | 72 +++++++++++++++++++++++++++++++------
2 files changed, 102 insertions(+), 18 deletions(-)
diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py
index baeef7e..6779153 100644
--- a/zeus/config_manager/view.py
+++ b/zeus/config_manager/view.py
@@ -16,6 +16,7 @@ Author:
Description: Restful APIs for host
"""
import json
+import os
from typing import List, Dict
from vulcanus.multi_thread_handler import MultiThreadHandler
@@ -26,7 +27,7 @@ from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF
from zeus.database.proxy.host import HostProxy
from zeus.function.model import ClientConnectArgs
from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema
-from zeus.host_manager.ssh import execute_command_and_parse_its_result
+from zeus.host_manager.ssh import execute_command_and_parse_its_result, execute_command_sftp_result
class CollectConfig(BaseResponse):
@@ -222,15 +223,46 @@ class CollectConfig(BaseResponse):
class SyncConfig(BaseResponse):
-
@staticmethod
- def sync_config_content(host_info: Dict, sync_config_info: Dict):
- command = CERES_SYNC_CONF % json.dumps(sync_config_info)
- status, content = execute_command_and_parse_its_result(
+ def sync_config_by_execute_command_sftp(host_info: Dict, sync_config_info: Dict, local_path: str,
+ remote_path: str):
+ content = sync_config_info.get("content")
+ with open(local_path, "w", encoding="UTF-8") as f:
+ f.write(content)
+ status = execute_command_sftp_result(
ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"),
- host_info.get("ssh_user"), host_info.get("pkey")), command)
+ host_info.get("ssh_user"), host_info.get("pkey")), local_path, remote_path)
return status
+ @staticmethod
+ def sync_config_content(host_info: Dict, sync_config_info: Dict):
+ join_path = "/tmp"
+ if sync_config_info.get("file_path") == "/etc/profile":
+ local_path = os.path.join(join_path, "profile")
+ remote_path = "/etc/profile"
+ status = SyncConfig.sync_config_by_execute_command_sftp(host_info, sync_config_info, local_path,
+ remote_path)
+ return status
+ elif sync_config_info.get("file_path") == "/etc/rc.local":
+ local_path = os.path.join(join_path, "rc.local")
+ remote_path = "/etc/rc.local"
+ status = SyncConfig.sync_config_by_execute_command_sftp(host_info, sync_config_info, local_path,
+ remote_path)
+ return status
+ elif sync_config_info.get("file_path") == "/etc/bashrc":
+ local_path = os.path.join(join_path, "bashrc")
+ remote_path = "/etc/bashrc"
+ status = SyncConfig.sync_config_by_execute_command_sftp(host_info, sync_config_info, local_path,
+ remote_path)
+ return status
+ else:
+ command = CERES_SYNC_CONF % json.dumps(sync_config_info)
+
+ status, content = execute_command_and_parse_its_result(
+ ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"),
+ host_info.get("ssh_user"), host_info.get("pkey")), command)
+ return status
+
@BaseResponse.handle(schema=SyncConfigSchema, token=False)
def put(self, **params):
@@ -244,13 +276,13 @@ class SyncConfig(BaseResponse):
}
# Query host address from database
- proxy = HostProxy(configuration)
+ proxy = HostProxy()
if not proxy.connect():
return self.response(code=state.DATABASE_CONNECT_ERROR, data={"resp": sync_result})
status, host_list = proxy.get_host_info(
{"username": "admin", "host_list": [params.get('host_id')]}, True)
- if status != state.SUCCEED or len(host_list) == 1:
+ if status != state.SUCCEED:
return self.response(code=status, data={"resp": sync_result})
host_info = host_list[0]
diff --git a/zeus/host_manager/ssh.py b/zeus/host_manager/ssh.py
index 4c3a259..11f6383 100644
--- a/zeus/host_manager/ssh.py
+++ b/zeus/host_manager/ssh.py
@@ -15,6 +15,7 @@ from io import StringIO
from typing import Tuple
import paramiko
+from paramiko import sftp
from vulcanus.log.log import LOGGER
from vulcanus.restful.resp import state
@@ -57,7 +58,13 @@ class SSH:
"""
def __init__(self, ip, username, port, password=None, pkey=None):
- self._client_args = {'hostname': ip, 'username': username, 'port': port, "password": password, "pkey": pkey}
+ self._client_args = {
+ 'hostname': ip,
+ 'username': username,
+ 'port': port,
+ "password": password,
+ "pkey": pkey
+ }
self._client = self.client()
def client(self):
@@ -71,15 +78,15 @@ class SSH:
def execute_command(self, command: str, timeout: float = None) -> tuple:
"""
- create a ssh client, execute command and parse result
+ create a ssh client, execute command and parse result
- Args:
- command(str): shell command
- timeout(float): the maximum time to wait for the result of command execution
+ Args:
+ command(str): shell command
+ timeout(float): the maximum time to wait for the result of command execution
- Returns:
- tuple:
- status, result, error message
+ Returns:
+ tuple:
+ status, result, error message
"""
open_channel = self._client.get_transport().open_session(timeout=timeout)
open_channel.set_combine_stderr(False)
@@ -110,13 +117,14 @@ def execute_command_and_parse_its_result(connect_args: ClientConnectArgs, comman
status, result
"""
if not connect_args.pkey:
- return state.SSH_AUTHENTICATION_ERROR, f"ssh authentication failed when connect host " f"{connect_args.host_ip}"
+ return state.SSH_AUTHENTICATION_ERROR, f"ssh authentication failed when connect host " \
+ f"{connect_args.host_ip}"
try:
client = SSH(
ip=connect_args.host_ip,
username=connect_args.ssh_user,
port=connect_args.ssh_port,
- pkey=paramiko.RSAKey.from_private_key(StringIO(connect_args.pkey)),
+ pkey=paramiko.RSAKey.from_private_key(StringIO(connect_args.pkey))
)
exit_status, stdout, stderr = client.execute_command(command, connect_args.timeout)
except socket.error as error:
@@ -131,3 +139,47 @@ def execute_command_and_parse_its_result(connect_args: ClientConnectArgs, comman
return state.SUCCEED, stdout
LOGGER.error(stderr)
return state.EXECUTE_COMMAND_ERROR, stderr
+
+
+def execute_command_sftp_result(connect_args: ClientConnectArgs, local_path=None, remote_path=None):
+ """
+ create a ssh client, execute command and parse result
+
+ Args:
+ connect_args(ClientConnectArgs): e.g
+ ClientArgs(host_ip='127.0.0.1', ssh_port=22, ssh_user='root', pkey=RSAKey string)
+ command(str): shell command
+
+ Returns:
+ tuple:
+ status, result
+ """
+ global sftp_client, client
+ if not connect_args.pkey:
+ return state.SSH_AUTHENTICATION_ERROR, f"ssh authentication failed when connect host " \
+ f"{connect_args.host_ip}"
+ try:
+ client = SSH(
+ ip=connect_args.host_ip,
+ username=connect_args.ssh_user,
+ port=connect_args.ssh_port,
+ pkey=paramiko.RSAKey.from_private_key(StringIO(connect_args.pkey))
+ )
+ sftp_client = client.client().open_sftp()
+
+ # Specifies the path to the local file and the remote file
+ # Upload files to a remote server
+ sftp_client.put(local_path, remote_path)
+ return state.SUCCEED
+ except socket.error as error:
+ LOGGER.error(error)
+ return state.SSH_CONNECTION_ERROR, "SSH.Connection.Error"
+ except paramiko.ssh_exception.SSHException as error:
+ LOGGER.error(error)
+ return state.SSH_AUTHENTICATION_ERROR, "SSH.Authentication.Error"
+ except Exception as error:
+ LOGGER.error(error)
+ return state.SSH_AUTHENTICATION_ERROR, "SSH.Authentication.Error"
+ finally:
+ sftp_client.close()
+ client.close()
--
2.33.1.windows.1

643
0003-fix-python-prometheus-api-client-import-error.patch Normal file
View File

@ -0,0 +1,643 @@
From cca4f59f2c68b20804fae06692179b8d3e181327 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Mon, 6 Nov 2023 09:36:24 +0800
Subject: [PATCH] fix python-prometheus-api-client import error
---
zeus/conf/constant.py | 4 -
zeus/database/proxy/metric.py | 458 --------------------------------
zeus/function/verify/metric.py | 28 --
zeus/metric_manager/__init__.py | 12 -
zeus/metric_manager/view.py | 51 ----
zeus/url.py | 9 -
6 files changed, 562 deletions(-)
delete mode 100644 zeus/database/proxy/metric.py
delete mode 100644 zeus/function/verify/metric.py
delete mode 100644 zeus/metric_manager/__init__.py
delete mode 100644 zeus/metric_manager/view.py
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index bf8792a..304ed7e 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -72,10 +72,6 @@ EXECUTE_CVE_FIX = '/manage/vulnerability/cve/fix'
EXECUTE_CVE_SCAN = '/manage/vulnerability/cve/scan'
EXECUTE_CVE_ROLLBACK = "/manage/vulnerability/cve/rollback"
-# metric config
-QUERY_METRIC_NAMES = '/manage/host/metric/names'
-QUERY_METRIC_DATA = '/manage/host/metric/data'
-QUERY_METRIC_LIST = '/manage/host/metric/list'
# auth login
GITEE_OAUTH = "https://gitee.com/oauth/authorize"
diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py
deleted file mode 100644
index 5fa75e9..0000000
--- a/zeus/database/proxy/metric.py
+++ /dev/null
@@ -1,458 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-"""
-Time: 2022-07-27
-Author: YangYunYi
-Description: Query raw data from Prometheus
-"""
-from typing import Dict, Tuple, List, Optional
-import datetime
-from prometheus_api_client import PrometheusApiClientException
-from vulcanus.database.proxy import PromDbProxy
-from vulcanus.log.log import LOGGER
-from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED
-from zeus.conf import configuration
-
-
-class MetricProxy(PromDbProxy):
- """
- Proxy of prometheus time series database
- """
-
- def __init__(self, host=None, port=None):
- """
- Init MetricProxy
-
- Args:
- host (str)
- port (int)
- """
- PromDbProxy.__init__(self, host, port)
- self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100
- self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s"
-
- @staticmethod
- def __metric_dict2str(metric: Dict) -> str:
- """
- Trans metric dict to string
- Args:
- metric (Dict):
- {
- "__name__": "metric name1",
- 'instance': '172.168.128.164:9100',
- 'job': 'prometheus',
- 'label1': 'label_value1'
- ...
- }
-
- Returns:
- metric_str(str): 'metric_name1{label1="value1", label2="values"}'
- """
-
- label_str = ""
- if "__name__" not in metric.keys():
- return label_str
- sorted_label_list = sorted(metric.items(), reverse=False)
- for label in sorted_label_list:
- # The job label is usually "prometheus" in this framework and
- # has no effect on subsequent data requests, so remove it to save space.
- # __name__ label move to the front.
- metric_key = label[0]
- metric_value = label[1]
- if metric_key in ["__name__", "job"]:
- continue
- label_str += "%s=\"%s\"," % (metric_key, metric_value)
-
- if label_str == "":
- # It's a metric with only a name and no personalized label
- ret = metric["__name__"]
- else:
- # Remove the comma of the last element
- ret = "%s{%s}" % (metric["__name__"], label_str[:-1])
- return ret
-
- def query_metric_names(self, query_ip: str) -> Tuple[int, dict]:
- """
- Query data
- Args:
- query_ip(str): query ip address
-
- Returns:
- int: status code
- dict: e.g
- {
- "results": ["metric1", "metric2"]
- }
- """
-
- query_ip = query_ip["query_ip"]
- query_host = {"host_id": "query_host_id", "host_ip": query_ip}
-
- query_metric_names = []
- res = {'results': query_metric_names}
-
- host_ip = query_host["host_ip"]
- host_port = query_host.get("instance_port", self.default_instance_port)
-
- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port)
-
- if ret != SUCCEED:
- return ret, res
-
- for metric in metric_list:
- metric = metric.split('{')[0]
- if metric not in query_metric_names:
- query_metric_names.append(metric)
-
- return ret, res
-
- def query_metric_list(self, data: Dict[str, str]) -> Tuple[int, dict]:
- """
- Query metric list
- Args:
- data(dict): param e.g
- {
- "query_ip": "str",
- "metric_names": ["metric1", "metric2"]
- }
-
- Returns:
- int: status code
- dict: e.g
- {
- 'results': {
- "metric1": [
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- ],
- "metric2": [
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- ]
- }
- }
- """
-
- query_ip = data.get('query_ip')
- query_metric = data.get('metric_names')
-
- query_host = {"host_id": "query_host_id", "host_ip": query_ip}
-
- query_metric_list = {}
- res = {'results': query_metric_list}
-
- host_ip = query_host["host_ip"]
- host_port = query_host.get("instance_port", self.default_instance_port)
-
- ret, metric_list = self.query_metric_list_of_host(
- host_ip,
- host_port,
- )
-
- if ret != SUCCEED:
- LOGGER.warning("Host metric list query error")
- return ret, res
-
- for query_metric_name in query_metric:
- query_metric_list[query_metric_name] = []
- for metric in metric_list:
- metric_name = metric.split('{')[0]
- if metric_name != query_metric_name:
- continue
- query_metric_list[query_metric_name].append(metric)
-
- return ret, res
-
- def query_metric_data(self, data: Dict[str, str]) -> Tuple[int, dict]:
- """
- Query metric data
- Args:
- data(dict): param e.g
- {
- "time_range": "list",
- "query_ip": "str",
- "query_info": {
- "metric1": [
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- ],
- "metric2": [
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- ]
- }
- }
-
- Returns:
- int: status code
- dict: e.g
- {
- 'results': {
- "metric1": {
- "metric1{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- "metric1{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- },
- "metric1": {
- "metric2{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- "metric2{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- },
- }
- }
- """
- time_range = data.get('time_range')
- query_ip = data.get('query_ip')
- query_info = data.get('query_info')
-
- query_host = {"host_id": "query_host_id", "host_ip": query_ip}
-
- host_ip = query_host["host_ip"]
- host_port = query_host.get("instance_port", self.default_instance_port)
-
- query_host_list = [query_host]
- query_data = {}
- res = {'results': query_data}
-
- # adjust query range step based on query time range
- # the default query range step is 15 seconds
- QUERY_RANGE_STEP = 15
- query_range_step = QUERY_RANGE_STEP
- query_range_seconds = time_range[1] - time_range[0]
- total_query_times = query_range_seconds / query_range_step
-
- while total_query_times > 11000:
- query_range_step += 15
- total_query_times = query_range_seconds / query_range_step
-
- def add_two_dim_dict(thedict, key_a, key_b, val):
- if key_a in thedict:
- thedict[key_a].update({key_b: val})
- else:
- thedict.update({key_a: {key_b: val}})
-
- if not query_info:
- return SUCCEED, res
-
- for metric_name, metric_list in query_info.items():
- if not metric_list:
- _, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric_name)
- if not metric_list:
- query_data[metric_name] = []
- for metric_info in metric_list:
- data_status, monitor_data = self.query_data(
- time_range=time_range,
- host_list=query_host_list,
- metric=metric_info,
- adjusted_range_step=query_range_step,
- )
- values = []
- if data_status == SUCCEED:
- values = monitor_data[query_host["host_id"]][metric_info]
- add_two_dim_dict(query_data, metric_name, metric_info, values)
-
- return SUCCEED, res
-
- def query_data(
- self,
- time_range: List[int],
- host_list: list,
- metric: Optional[str] = None,
- adjusted_range_step: Optional[int] = None,
- ) -> Tuple[int, Dict]:
- """
- Query data
- Args:
- time_range(list): time range
- host_list(list): host list, If the port is not specified, the default value is used
- [{"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 9100},
- {"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 8080},
- {"host_id": "id2", "host_ip": "172.168.128.165"}]
-
- Returns:
- ret(int): query ret
- host_data_list(dict): host data list ret
- {
- 'id1': {
- 'metric1'{instance="172.168.128.164:9100",label1="value2"}':
- [[time1, 'value1'],
- [time2, 'value2'],
- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty
- 'metric13{instance="172.168.128.164:8080"}': None => get no data
- },
- 'id2': None => get no metric list of this host
- }
- """
-
- host_data_list = {}
- if not host_list:
- return PARAM_ERROR, host_data_list
-
- status = SUCCEED
- for host in host_list:
- host_id = host["host_id"]
- host_ip = host["host_ip"]
- host_port = host.get("instance_port", self.default_instance_port)
- if host_id not in host_data_list.keys():
- host_data_list[host_id] = None
-
- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric)
- if ret != SUCCEED:
- status = PARTIAL_SUCCEED
- host_data_list[host_id] = None
- continue
- ret, data_list = self.__query_data_by_host(metric_list, time_range, adjusted_range_step)
- if ret != SUCCEED:
- status = PARTIAL_SUCCEED
- if not host_data_list[host_id]:
- host_data_list[host_id] = data_list
- else:
- host_data_list[host_id].update(data_list)
-
- return status, host_data_list
-
- def __parse_metric_data(self, metric_data: List) -> List[str]:
- """
- Parse metric data from prometheus to name<-> label_config dict
- Args:
- metric_data(List): metric list data from prometheus
- [{'metric':{
- "__name__": "metric_name1",
- 'instance': '172.168.128.164:9100',
- 'job': 'prometheus',
- 'label1': 'label_value1'
- ...
- },
- 'value': [1658926441.526, '0']
- }]
-
- Returns:
- metric_list(List[str]):
- [
- 'metric_name1{label1="value1", label2="value2"}'
- ]
- """
-
- metric_list = []
- for metric_item in metric_data:
- metric_dict = metric_item["metric"]
- metric_str = self.__metric_dict2str(metric_dict)
- if metric_str == "":
- continue
- metric_list.append(metric_str)
-
- return metric_list
-
- def query_metric_list_of_host(
- self, host_ip: str, host_port: Optional[int] = None, metric: Optional[str] = None
- ) -> Tuple[int, List[str]]:
- """
- Query metric list of a host
- Args:
- host_ip(str): host ip
- host_port(int): host port
-
- Returns:
- ret(int): query ret
- metric_list(list): metric list ret
- [
- 'metric_name1{label1="value1", label2="value2"}'
- ]
- """
- if not host_port:
- host_port = self.default_instance_port
- query_str = "{instance=\"%s:%s\"}" % (host_ip, str(host_port))
- if metric is not None:
- if metric.find('{') != -1:
- query_str = metric
- else:
- query_str = metric + query_str
- try:
- data = self._prom.custom_query(query=query_str)
- if not data:
- if query_str.startswith('{'):
- LOGGER.error(
- "Query metric list result is empty. "
- "Can not get metric list of host %s:%s " % (host_ip, host_port)
- )
- return NO_DATA, []
- metric_list = self.__parse_metric_data(data)
- return SUCCEED, metric_list
-
- except (ValueError, TypeError, PrometheusApiClientException) as error:
- LOGGER.error("host %s:%d Prometheus query metric list failed. %s" % (host_ip, host_port, error))
- return DATABASE_QUERY_ERROR, []
-
- def __query_data_by_host(
- self, metrics_list: List[str], time_range: List[int], adjusted_range_step: Optional[int] = None
- ) -> Tuple[int, Dict]:
- """
- Query data of a host
- Args:
- metrics_list(list): metric list of this host
- time_range(list): time range to query
-
- Returns:
- ret(int): query ret
- data_list(list): data list ret
- {
- 'metric1'{instance="172.168.128.164:9100",label1="value2"}':
- [[time1, 'value1'],
- [time2, 'value2'],
- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty
- 'metric13{instance="172.168.128.164:8080"}': None => get no data
- }
-
- """
- start_time = datetime.datetime.fromtimestamp(time_range[0])
- end_time = datetime.datetime.fromtimestamp(time_range[1])
-
- query_range_step = self.query_range_step
- if adjusted_range_step is not None:
- query_range_step = adjusted_range_step
-
- data_list = {}
- ret = SUCCEED
- for metric in metrics_list:
- try:
- data = self._prom.custom_query_range(
- query=metric, start_time=start_time, end_time=end_time, step=query_range_step
- )
- if not data or "values" not in data[0]:
- LOGGER.debug(
- "Query data result is empty. "
- "metric %s in %d-%d doesn't record in the prometheus " % (metric, time_range[0], time_range[1])
- )
- data_list[metric] = None
- ret = PARTIAL_SUCCEED
- continue
- data_list[metric] = data[0]["values"]
-
- except (ValueError, TypeError, PrometheusApiClientException) as error:
- LOGGER.error(
- "Prometheus metric %s in %d-%d query data failed. %s"
- % (metric, time_range[0], time_range[1], error)
- )
- data_list[metric] = None
- ret = PARTIAL_SUCCEED
- return ret, data_list
diff --git a/zeus/function/verify/metric.py b/zeus/function/verify/metric.py
deleted file mode 100644
index 5df28a2..0000000
--- a/zeus/function/verify/metric.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-from marshmallow import Schema, fields
-
-
-class QueryHostMetricNamesSchema(Schema):
- query_ip = fields.String(required=True)
-
-
-class QueryHostMetricDataSchema(Schema):
- time_range = fields.List(fields.Integer, required=True)
- query_ip = fields.String(required=True)
- query_info = fields.Dict()
-
-
-class QueryHostMetricListSchema(Schema):
- query_ip = fields.String(required=True)
- metric_names = fields.List(fields.String)
diff --git a/zeus/metric_manager/__init__.py b/zeus/metric_manager/__init__.py
deleted file mode 100644
index e90ecf9..0000000
--- a/zeus/metric_manager/__init__.py
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
diff --git a/zeus/metric_manager/view.py b/zeus/metric_manager/view.py
deleted file mode 100644
index 4d98cf2..0000000
--- a/zeus/metric_manager/view.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-from vulcanus.restful.response import BaseResponse
-from zeus.database.proxy.metric import MetricProxy
-from zeus.function.verify.metric import QueryHostMetricDataSchema, QueryHostMetricListSchema, QueryHostMetricNamesSchema
-
-
-class QueryHostMetricNames(BaseResponse):
- """
- Interface for query host metric names from web.
- Restful API: GET
- """
-
- @BaseResponse.handle(schema=QueryHostMetricNamesSchema, proxy=MetricProxy)
- def get(self, callback: MetricProxy, **params):
- status_code, result = callback.query_metric_names(params)
- return self.response(code=status_code, data=result)
-
-
-class QueryHostMetricData(BaseResponse):
- """
- Interface for query host metric data from web.
- Restful API: POST
- """
-
- @BaseResponse.handle(schema=QueryHostMetricDataSchema, proxy=MetricProxy)
- def post(self, callback: MetricProxy, **params):
- status_code, result = callback.query_metric_data(params)
- return self.response(code=status_code, data=result)
-
-
-class QueryHostMetricList(BaseResponse):
- """
- Interface for query host metric list from web.
- Restful API: POST
- """
-
- @BaseResponse.handle(schema=QueryHostMetricListSchema, proxy=MetricProxy)
- def post(self, callback: MetricProxy, **params):
- status_code, result = callback.query_metric_list(params)
- return self.response(code=status_code, data=result)
diff --git a/zeus/url.py b/zeus/url.py
index 597dcc7..3ec8d21 100644
--- a/zeus/url.py
+++ b/zeus/url.py
@@ -43,9 +43,6 @@ from zeus.conf.constant import (
LOGOUT,
QUERY_HOST,
QUERY_HOST_DETAIL,
- QUERY_METRIC_DATA,
- QUERY_METRIC_LIST,
- QUERY_METRIC_NAMES,
REFRESH_TOKEN,
UPDATE_HOST,
USER_LOGIN,
@@ -53,7 +50,6 @@ from zeus.conf.constant import (
)
from zeus.config_manager import view as config_view
from zeus.host_manager import view as host_view
-from zeus.metric_manager import view as metric_view
from zeus.vulnerability_manage import view as vulnerability_view
URLS = []
@@ -100,11 +96,6 @@ SPECIFIC_URLS = {
(vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX),
(vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK),
],
- 'METRIC': [
- (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES),
- (metric_view.QueryHostMetricData, QUERY_METRIC_DATA),
- (metric_view.QueryHostMetricList, QUERY_METRIC_LIST),
- ],
}
for _, value in SPECIFIC_URLS.items():
--
2.33.0

29
0004-bugfix-cve-fix-result-parsing-error.patch
View File

@ -1,29 +0,0 @@
From 21a8a9db54175547675bda5214d98ffa5a654d03 Mon Sep 17 00:00:00 2001
From: rabbitali <shusheng.wen@outlook.com>
Date: Tue, 19 Sep 2023 16:58:56 +0800
Subject: [PATCH 1/1] bugfix: cve fix result parsing error
---
zeus/vulnerability_manage/view.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py
index b5360e2..345978f 100644
--- a/zeus/vulnerability_manage/view.py
+++ b/zeus/vulnerability_manage/view.py
@@ -416,10 +416,11 @@ class ExecuteCveFixTask(BaseResponse, BaseExcuteTask):
cve_fix_result_json = json.loads(cve_fix_result)
if cve_fix_result_json.pop("code") != state.SUCCEED:
request_body["status"] = CveTaskStatus.FAIL
+ else:
+ request_body["status"] = CveTaskStatus.SUCCEED
request_body.update(
{
- "status": CveTaskStatus.SUCCEED,
"check_items": cve_fix_result_json.get("check_items"),
"cves": cve_fix_result_json.get("cves"),
}
--
2.33.0

33
0004-update-the-template-file-contents-for-adding-hosts.patch Normal file
View File

@ -0,0 +1,33 @@
From 36d5b7a26f9f470e0b9a593edb7f198cd9022c47 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Thu, 26 Oct 2023 18:43:37 +0800
Subject: [PATCH] update the template file contents for adding hosts
---
zeus/conf/constant.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index bf8792a..df2948d 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -91,8 +91,14 @@ CHECK_WORKFLOW_HOST_EXIST = '/check/workflow/host/exist'
# host template file content
HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,ssh_pkey,host_name,host_group_name,management
-test_ip_1,22,root,password,ssh_pkey,test_host,test_host_group,False
-test_ip_2,22,root,password,ssh_pkey,test_host,test_host_group,False
+127.0.0.1,22,root,password,private key,test_host,test_host_group,FALSE
+127.0.0.1,23,root,password,private key,test_host,test_host_group,FALSE
+,,,,,,,
+"提示:",,,,,,,
+"1. 除登录密码与SSH登录秘钥外,其余信息都应提供有效值",,,,,,,
+"2. 登录密码与SSH登录秘钥可选择一种填入,当两者都提供时,以SSH登录秘钥为准",,,,,,,
+"3. 添加的主机信息不应存在重复信息(主机名称重复或者主机IP+端口重复)",,,,,,,
+"4. 上传本文件前,请删除此部分提示内容",,,,,,,
"""
--
2.33.0

33
0005-bugfix-update-host-api-request-error.patch Normal file
View File

@ -0,0 +1,33 @@
From 36f98b43bd571ac9f2f4f9a9fe658684d591d52e Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Fri, 27 Oct 2023 15:21:54 +0800
Subject: [PATCH] bugfix: update host api request error when changing username
---
zeus/host_manager/view.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py
index 95e1434..10418d1 100644
--- a/zeus/host_manager/view.py
+++ b/zeus/host_manager/view.py
@@ -930,11 +930,11 @@ class UpdateHost(BaseResponse):
LOGGER.warning(f"there is a duplicate host address in database " f"when update host {self.host.host_id}!")
return self.response(code=state.PARAM_ERROR, message="there is a duplicate host ssh address in database!")
- if params.get("ssh_user") or params.get("ssh_port"):
- if not params.get("password") or not params.get("ssh_pkey"):
- return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.")
- self._save_ssh_key(params)
- elif params.get("password") or params.get("ssh_pkey"):
+ if params.get("password") or params.get("ssh_pkey"):
self._save_ssh_key(params)
+ return self.response(callback.update_host_info(params.pop("host_id"), params))
+
+ if params.get("ssh_user") or params.get("ssh_port"):
+ return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.")
return self.response(callback.update_host_info(params.pop("host_id"), params))
--
2.33.0

139
0006-optimize-import.patch Normal file
View File

@ -0,0 +1,139 @@
From 6ba93db6c012b7547e80fc71e7dc3dd96d6f0aa4 Mon Sep 17 00:00:00 2001
From: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
Date: Wed, 18 Oct 2023 02:49:04 +0000
Subject: [PATCH 1/5] optimize Import
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
---
zeus/config_manager/view.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py
index 6779153..0aa9682 100644
--- a/zeus/config_manager/view.py
+++ b/zeus/config_manager/view.py
@@ -22,7 +22,6 @@ from typing import List, Dict
from vulcanus.multi_thread_handler import MultiThreadHandler
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
-from zeus.conf import configuration
from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF
from zeus.database.proxy.host import HostProxy
from zeus.function.model import ClientConnectArgs
--
Gitee
From a43c526f7e8dcda9fe9b46ff6e1f80f370954259 Mon Sep 17 00:00:00 2001
From: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
Date: Wed, 18 Oct 2023 02:56:33 +0000
Subject: [PATCH 2/5] update zeus/tests/host_manager/test_add_host.py.
Signed-off-by: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
---
zeus/tests/host_manager/test_add_host.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/zeus/tests/host_manager/test_add_host.py b/zeus/tests/host_manager/test_add_host.py
index c51be95..6b8b1b0 100644
--- a/zeus/tests/host_manager/test_add_host.py
+++ b/zeus/tests/host_manager/test_add_host.py
@@ -17,7 +17,6 @@ from io import BytesIO
from unittest import mock
import paramiko
-import sqlalchemy
from paramiko import AuthenticationException
from sqlalchemy.orm.collections import InstrumentedList
--
Gitee
From d03456b27fdc16c868c0b871b3f43786cc383146 Mon Sep 17 00:00:00 2001
From: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
Date: Wed, 18 Oct 2023 02:58:24 +0000
Subject: [PATCH 3/5] update zeus/host_manager/ssh.py.
Signed-off-by: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
---
zeus/host_manager/ssh.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/zeus/host_manager/ssh.py b/zeus/host_manager/ssh.py
index 11f6383..a4e7628 100644
--- a/zeus/host_manager/ssh.py
+++ b/zeus/host_manager/ssh.py
@@ -15,7 +15,6 @@ from io import StringIO
from typing import Tuple
import paramiko
-from paramiko import sftp
from vulcanus.log.log import LOGGER
from vulcanus.restful.resp import state
--
Gitee
From 887b9525192140c5142b62c14b91c1656b3a657a Mon Sep 17 00:00:00 2001
From: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
Date: Wed, 18 Oct 2023 02:59:12 +0000
Subject: [PATCH 4/5] update zeus/tests/host_manager/test_delete_host.py.
Signed-off-by: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
---
zeus/tests/host_manager/test_delete_host.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/zeus/tests/host_manager/test_delete_host.py b/zeus/tests/host_manager/test_delete_host.py
index 59a4098..5a0aea6 100644
--- a/zeus/tests/host_manager/test_delete_host.py
+++ b/zeus/tests/host_manager/test_delete_host.py
@@ -12,8 +12,6 @@
# ******************************************************************************/
from unittest import mock
-import sqlalchemy
-
from vulcanus.exceptions import DatabaseConnectionFailed
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
--
Gitee
From 089c24550ae9f5720629376692344870316e97a6 Mon Sep 17 00:00:00 2001
From: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
Date: Wed, 18 Oct 2023 03:00:38 +0000
Subject: [PATCH 5/5] update zeus/vulnerability_manage/view.py.
Signed-off-by: xuyongliang_01 <xuyongliang_yewu@cmss.chinamobile.com>
---
zeus/vulnerability_manage/view.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py
index 345978f..c56f672 100644
--- a/zeus/vulnerability_manage/view.py
+++ b/zeus/vulnerability_manage/view.py
@@ -15,12 +15,11 @@ import time
import threading
from typing import Dict, Tuple
-from flask import Response, request
+from flask import Response
import sqlalchemy
import gevent
from vulcanus.log.log import LOGGER
-from vulcanus.multi_thread_handler import MultiThreadHandler
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
from zeus.conf import configuration
--
Gitee

392
0007-add-an-api-for-querying-file-list.patch Normal file
View File

@ -0,0 +1,392 @@
From 318ad494e275d0404b2f1af768506a25c27028e2 Mon Sep 17 00:00:00 2001
From: liulei <450962@qq.com>
Date: Tue, 31 Oct 2023 10:17:37 +0800
Subject: [PATCH 1/2] Add an interface for querying file list
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/conf/constant.py | 2 +
zeus/config_manager/view.py | 42 +++++++++++-
zeus/function/verify/config.py | 8 +++
zeus/vulnerability_manage/url.py | 113 +++++++++++++++++++++++++++++++
4 files changed, 163 insertions(+), 2 deletions(-)
create mode 100644 zeus/vulnerability_manage/url.py
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index df2948d..44065be 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -34,6 +34,7 @@ CERES_CVE_SCAN = "aops-ceres apollo --scan '%s'"
CERES_CVE_FIX = "aops-ceres apollo --fix '%s'"
CERES_CVE_ROLLBACK = "aops-ceres apollo --rollback '%s'"
CERES_SYNC_CONF = "aops-ceres sync --conf '%s'"
+CERES_OBJECT_FILE_CONF = "aops-ceres ragdoll --list '%s'"
# zeus route
ADD_HOST = "/manage/host/add"
@@ -56,6 +57,7 @@ GET_GROUP = "/manage/host/group/get"
COLLECT_CONFIG = '/manage/config/collect'
SYNC_CONFIG = '/manage/config/sync'
+OBJECT_FILE_CONFIG = '/manage/config/objectfile'
USER_LOGIN = "/manage/account/login"
LOGOUT = "/manage/account/logout"
diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py
index 0aa9682..bde6dbf 100644
--- a/zeus/config_manager/view.py
+++ b/zeus/config_manager/view.py
@@ -22,10 +22,10 @@ from typing import List, Dict
from vulcanus.multi_thread_handler import MultiThreadHandler
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
-from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF
+from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF, OBJECT_FILE_CONF, CERES_OBJECT_FILE_CONF
from zeus.database.proxy.host import HostProxy
from zeus.function.model import ClientConnectArgs
-from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema
+from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema, ObjectFileConfigSchema
from zeus.host_manager.ssh import execute_command_and_parse_its_result, execute_command_sftp_result
@@ -290,3 +290,41 @@ class SyncConfig(BaseResponse):
sync_result['sync_result'] = True
return self.response(code=state.SUCCEED, data={"resp": sync_result})
return self.response(code=state.UNKNOWN_ERROR, data={"resp": sync_result})
+
+
+class ObjectFileConfig(BaseResponse):
+
+ @staticmethod
+ def object_file_config_content(host_info: Dict, file_directory: str):
+ command = CERES_OBJECT_FILE_CONF % file_directory
+ status, content = execute_command_and_parse_its_result(
+ ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"),
+ host_info.get("ssh_user"), host_info.get("pkey")), command)
+ return status, content
+
+ @BaseResponse.handle(schema=ObjectFileConfigSchema, token=False)
+ def post(self, **params):
+ object_file_result = {
+ "object_file_paths": list(),
+ "object_file_result": False
+ }
+ # Query host address from database
+ proxy = HostProxy()
+ if not proxy.connect():
+ return self.response(code=state.DATABASE_CONNECT_ERROR, data={"resp": object_file_result})
+
+ status, host_list = proxy.get_host_info(
+ {"username": "admin", "host_list": [params.get('host_id')]}, True)
+ if status != state.SUCCEED:
+ return self.response(code=status, data={"resp": object_file_result})
+
+ host_info = host_list[0]
+ status, content = self.object_file_config_content(host_info, params.get('file_directory'))
+ if status == state.SUCCEED:
+ object_file_result['object_file_result'] = True
+ content_res = json.loads(content)
+ if content_res.get("resp"):
+ resp = content_res.get("resp")
+ object_file_result['object_file_paths'] = resp
+ return self.response(code=state.SUCCEED, data={"resp": object_file_result})
+ return self.response(code=state.UNKNOWN_ERROR, data={"resp": object_file_result})
diff --git a/zeus/function/verify/config.py b/zeus/function/verify/config.py
index 6e5bf64..1ef7b97 100644
--- a/zeus/function/verify/config.py
+++ b/zeus/function/verify/config.py
@@ -45,3 +45,11 @@ class SyncConfigSchema(Schema):
host_id = fields.Integer(required=True, validate=lambda s: s > 0)
file_path = fields.String(required=True, validate=lambda s: len(s) > 0)
content = fields.String(required=True, validate=lambda s: len(s) > 0)
+
+
+class ObjectFileConfigSchema(Schema):
+ """
+ validators for ObjectFileConfigSchema
+ """
+ host_id = fields.Integer(required=True, validate=lambda s: s > 0)
+ file_directory = fields.String(required=True, validate=lambda s: len(s) > 0)
diff --git a/zeus/vulnerability_manage/url.py b/zeus/vulnerability_manage/url.py
new file mode 100644
index 0000000..fe464ac
--- /dev/null
+++ b/zeus/vulnerability_manage/url.py
@@ -0,0 +1,113 @@
+#!/usr/bin/python3
+# ******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ******************************************************************************/
+"""
+Time:
+Author:
+Description: url set
+"""
+from zeus.account_manager import view as account_view
+from zeus.agent_manager import view as agent_view
+from zeus.conf.constant import (
+ ADD_GROUP,
+ ADD_HOST,
+ ADD_HOST_BATCH,
+ ADD_USER,
+ AGENT_METRIC_SET,
+ AGENT_PLUGIN_INFO,
+ AGENT_PLUGIN_SET,
+ AUTH_REDIRECT_URL,
+ BIND_AUTH_ACCOUNT,
+ CHANGE_PASSWORD,
+ COLLECT_CONFIG,
+ DELETE_GROUP,
+ DELETE_HOST,
+ EXECUTE_CVE_FIX,
+ EXECUTE_CVE_ROLLBACK,
+ EXECUTE_CVE_SCAN,
+ EXECUTE_REPO_SET,
+ GET_GROUP,
+ GET_HOST_COUNT,
+ GET_HOST_TEMPLATE_FILE,
+ GITEE_AUTH_LOGIN,
+ HOST_SCENE_GET,
+ LOGOUT,
+ QUERY_HOST,
+ QUERY_HOST_DETAIL,
+ QUERY_METRIC_DATA,
+ QUERY_METRIC_LIST,
+ QUERY_METRIC_NAMES,
+ REFRESH_TOKEN,
+ UPDATE_HOST,
+ USER_LOGIN,
+ SYNC_CONFIG,
+ OBJECT_FILE_CONFIG
+)
+from zeus.config_manager import view as config_view
+from zeus.host_manager import view as host_view
+from zeus.metric_manager import view as metric_view
+from zeus.vulnerability_manage import view as vulnerability_view
+
+URLS = []
+
+SPECIFIC_URLS = {
+ "ACCOUNT_URLS": [
+ (account_view.Login, USER_LOGIN),
+ (account_view.ChangePassword, CHANGE_PASSWORD),
+ (account_view.AddUser, ADD_USER),
+ (account_view.GiteeAuthLogin, GITEE_AUTH_LOGIN),
+ (account_view.AuthRedirectUrl, AUTH_REDIRECT_URL),
+ (account_view.BindAuthAccount, BIND_AUTH_ACCOUNT),
+ (account_view.RefreshToken, REFRESH_TOKEN),
+ (account_view.Logout, LOGOUT),
+ ],
+ "HOST_URLS": [
+ (host_view.AddHost, ADD_HOST),
+ (host_view.AddHostBatch, ADD_HOST_BATCH),
+ (host_view.DeleteHost, DELETE_HOST),
+ (host_view.UpdateHost, UPDATE_HOST),
+ (host_view.GetHost, QUERY_HOST),
+ (host_view.GetHostInfo, QUERY_HOST_DETAIL),
+ (host_view.GetHostCount, GET_HOST_COUNT),
+ (host_view.GetHostTemplateFile, GET_HOST_TEMPLATE_FILE),
+ ],
+ "HOST_GROUP_URLS": [
+ (host_view.AddHostGroup, ADD_GROUP),
+ (host_view.DeleteHostGroup, DELETE_GROUP),
+ (host_view.GetHostGroup, GET_GROUP),
+ ],
+ "CONFIG_URLS": [
+ (config_view.CollectConfig, COLLECT_CONFIG),
+ (config_view.SyncConfig, SYNC_CONFIG),
+ (config_view.ObjectFileConfig, OBJECT_FILE_CONFIG)
+ ],
+ 'AGENT_URLS': [
+ (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO),
+ (agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET),
+ (agent_view.SetAgentMetricStatus, AGENT_METRIC_SET),
+ (agent_view.GetHostScene, HOST_SCENE_GET),
+ ],
+ 'CVE_URLS': [
+ (vulnerability_view.ExecuteRepoSetTask, EXECUTE_REPO_SET),
+ (vulnerability_view.ExecuteCveScanTask, EXECUTE_CVE_SCAN),
+ (vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX),
+ (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK),
+ ],
+ 'METRIC': [
+ (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES),
+ (metric_view.QueryHostMetricData, QUERY_METRIC_DATA),
+ (metric_view.QueryHostMetricList, QUERY_METRIC_LIST),
+ ],
+}
+
+for _, value in SPECIFIC_URLS.items():
+ URLS.extend(value)
--
Gitee
From bf654cd0d5086b29ab3e5b27cdc42cc0f8cc24ed Mon Sep 17 00:00:00 2001
From: liulei <450962@qq.com>
Date: Tue, 31 Oct 2023 10:29:27 +0800
Subject: [PATCH 2/2] Add an interface for querying file list
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
zeus/url.py | 4 +-
zeus/vulnerability_manage/url.py | 113 -------------------------------
2 files changed, 3 insertions(+), 114 deletions(-)
delete mode 100644 zeus/vulnerability_manage/url.py
diff --git a/zeus/url.py b/zeus/url.py
index 597dcc7..fe464ac 100644
--- a/zeus/url.py
+++ b/zeus/url.py
@@ -50,6 +50,7 @@ from zeus.conf.constant import (
UPDATE_HOST,
USER_LOGIN,
SYNC_CONFIG,
+ OBJECT_FILE_CONFIG
)
from zeus.config_manager import view as config_view
from zeus.host_manager import view as host_view
@@ -86,7 +87,8 @@ SPECIFIC_URLS = {
],
"CONFIG_URLS": [
(config_view.CollectConfig, COLLECT_CONFIG),
- (config_view.SyncConfig, SYNC_CONFIG)
+ (config_view.SyncConfig, SYNC_CONFIG),
+ (config_view.ObjectFileConfig, OBJECT_FILE_CONFIG)
],
'AGENT_URLS': [
(agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO),
diff --git a/zeus/vulnerability_manage/url.py b/zeus/vulnerability_manage/url.py
deleted file mode 100644
index fe464ac..0000000
--- a/zeus/vulnerability_manage/url.py
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-"""
-Time:
-Author:
-Description: url set
-"""
-from zeus.account_manager import view as account_view
-from zeus.agent_manager import view as agent_view
-from zeus.conf.constant import (
- ADD_GROUP,
- ADD_HOST,
- ADD_HOST_BATCH,
- ADD_USER,
- AGENT_METRIC_SET,
- AGENT_PLUGIN_INFO,
- AGENT_PLUGIN_SET,
- AUTH_REDIRECT_URL,
- BIND_AUTH_ACCOUNT,
- CHANGE_PASSWORD,
- COLLECT_CONFIG,
- DELETE_GROUP,
- DELETE_HOST,
- EXECUTE_CVE_FIX,
- EXECUTE_CVE_ROLLBACK,
- EXECUTE_CVE_SCAN,
- EXECUTE_REPO_SET,
- GET_GROUP,
- GET_HOST_COUNT,
- GET_HOST_TEMPLATE_FILE,
- GITEE_AUTH_LOGIN,
- HOST_SCENE_GET,
- LOGOUT,
- QUERY_HOST,
- QUERY_HOST_DETAIL,
- QUERY_METRIC_DATA,
- QUERY_METRIC_LIST,
- QUERY_METRIC_NAMES,
- REFRESH_TOKEN,
- UPDATE_HOST,
- USER_LOGIN,
- SYNC_CONFIG,
- OBJECT_FILE_CONFIG
-)
-from zeus.config_manager import view as config_view
-from zeus.host_manager import view as host_view
-from zeus.metric_manager import view as metric_view
-from zeus.vulnerability_manage import view as vulnerability_view
-
-URLS = []
-
-SPECIFIC_URLS = {
- "ACCOUNT_URLS": [
- (account_view.Login, USER_LOGIN),
- (account_view.ChangePassword, CHANGE_PASSWORD),
- (account_view.AddUser, ADD_USER),
- (account_view.GiteeAuthLogin, GITEE_AUTH_LOGIN),
- (account_view.AuthRedirectUrl, AUTH_REDIRECT_URL),
- (account_view.BindAuthAccount, BIND_AUTH_ACCOUNT),
- (account_view.RefreshToken, REFRESH_TOKEN),
- (account_view.Logout, LOGOUT),
- ],
- "HOST_URLS": [
- (host_view.AddHost, ADD_HOST),
- (host_view.AddHostBatch, ADD_HOST_BATCH),
- (host_view.DeleteHost, DELETE_HOST),
- (host_view.UpdateHost, UPDATE_HOST),
- (host_view.GetHost, QUERY_HOST),
- (host_view.GetHostInfo, QUERY_HOST_DETAIL),
- (host_view.GetHostCount, GET_HOST_COUNT),
- (host_view.GetHostTemplateFile, GET_HOST_TEMPLATE_FILE),
- ],
- "HOST_GROUP_URLS": [
- (host_view.AddHostGroup, ADD_GROUP),
- (host_view.DeleteHostGroup, DELETE_GROUP),
- (host_view.GetHostGroup, GET_GROUP),
- ],
- "CONFIG_URLS": [
- (config_view.CollectConfig, COLLECT_CONFIG),
- (config_view.SyncConfig, SYNC_CONFIG),
- (config_view.ObjectFileConfig, OBJECT_FILE_CONFIG)
- ],
- 'AGENT_URLS': [
- (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO),
- (agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET),
- (agent_view.SetAgentMetricStatus, AGENT_METRIC_SET),
- (agent_view.GetHostScene, HOST_SCENE_GET),
- ],
- 'CVE_URLS': [
- (vulnerability_view.ExecuteRepoSetTask, EXECUTE_REPO_SET),
- (vulnerability_view.ExecuteCveScanTask, EXECUTE_CVE_SCAN),
- (vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX),
- (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK),
- ],
- 'METRIC': [
- (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES),
- (metric_view.QueryHostMetricData, QUERY_METRIC_DATA),
- (metric_view.QueryHostMetricList, QUERY_METRIC_LIST),
- ],
-}
-
-for _, value in SPECIFIC_URLS.items():
- URLS.extend(value)
--
Gitee

75
0008-bugfix-email-config-does-not-take-effect.patch Normal file
View File

@ -0,0 +1,75 @@
From 2ccba1565c7d2ad5e4bb821f05f09a3b63edbd8b Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Tue, 31 Oct 2023 18:03:40 +0800
Subject: [PATCH] bugfix: email config does not take effect
---
zeus/vulnerability_manage/view.py | 31 ++++++++++++++++++-------------
1 file changed, 18 insertions(+), 13 deletions(-)
diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py
index c56f672..34a33c9 100644
--- a/zeus/vulnerability_manage/view.py
+++ b/zeus/vulnerability_manage/view.py
@@ -15,7 +15,7 @@ import time
import threading
from typing import Dict, Tuple
-from flask import Response
+from flask import Response, request
import sqlalchemy
import gevent
@@ -301,14 +301,13 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask):
you can turn it off by modifying email config configuration in aops-private-config.ini, but it will still be
triggered during scheduled cve scan.
"""
- if configuration.email.get("ENABLED") or self._header.get("exempt_authentication"):
- self.get_response(
- "post",
- f'http://{configuration.apollo.get("IP")}:{ configuration.apollo.get("PORT")}{VUL_TASK_CVE_SCAN_NOTICE}',
- {},
- self._header,
- timeout=10,
- )
+ self.get_response(
+ "post",
+ f'http://{configuration.apollo.get("IP")}:{ configuration.apollo.get("PORT")}{VUL_TASK_CVE_SCAN_NOTICE}',
+ {},
+ self._header,
+ timeout=10,
+ )
@BaseResponse.handle(schema=CveScanSchema)
def post(self, **params) -> Response:
@@ -330,6 +329,13 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask):
Returns:
response body
"""
+
+ def execute_task(host_infos, need_to_send_email):
+ gevent.joinall([gevent.spawn(self._execute_task, host) for host in host_infos])
+ if need_to_send_email:
+ LOGGER.info("Plan to request the interface for sending emails")
+ self._execute_send_email()
+
# Query host basic info from database
status_code, host_infos = query_host_basic_info(params.get('total_hosts'), params.get('username'))
if status_code != state.SUCCEED:
@@ -340,10 +346,9 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask):
self._task_id = params.get("task_id")
self._check_items = params.get('check_items')
# Execute task
- threading.Thread(
- target=lambda: gevent.joinall([gevent.spawn(self._execute_task, host) for host in host_infos.values()])
- ).start()
- threading.Thread(target=self._execute_send_email).start()
+ need_to_send_email = request.headers.get("exempt_authentication") or configuration.email.get("ENABLED")
+ threading.Thread(target=execute_task, args=(host_infos.values(), need_to_send_email)).start()
+
return self.response(code=state.SUCCEED)
--
Gitee

25
0009-fix-import-error.patch Normal file
View File

@ -0,0 +1,25 @@
From 1e49007ac150ef9d51f8e1feff106ebb22dd9e96 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Sat, 4 Nov 2023 00:28:30 +0800
Subject: [PATCH] fix import error
---
zeus/config_manager/view.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py
index bde6dbf..b012c62 100644
--- a/zeus/config_manager/view.py
+++ b/zeus/config_manager/view.py
@@ -22,7 +22,7 @@ from typing import List, Dict
from vulcanus.multi_thread_handler import MultiThreadHandler
from vulcanus.restful.resp import state
from vulcanus.restful.response import BaseResponse
-from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF, OBJECT_FILE_CONF, CERES_OBJECT_FILE_CONF
+from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF, CERES_OBJECT_FILE_CONF
from zeus.database.proxy.host import HostProxy
from zeus.function.model import ClientConnectArgs
from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema, ObjectFileConfigSchema
--
2.33.0

BIN
aops-zeus-v1.3.0.tar.gz → aops-zeus-v1.3.1.tar.gz
View File

Binary file not shown.

40
aops-zeus.spec
View File

@ -1,21 +1,26 @@
Name: aops-zeus Name: aops-zeus
Version: v1.3.0 Version: v1.3.1
Release: 4 Release: 5
Summary: A host and user manager service which is the foundation of aops. Summary: A host and user manager service which is the foundation of aops.
License: MulanPSL2 License: MulanPSL2
URL: https://gitee.com/openeuler/%{name} URL: https://gitee.com/openeuler/%{name}
Source0: %{name}-%{version}.tar.gz Source0: %{name}-%{version}.tar.gz
Patch0001: 0001-add-file-sync-func.patch Patch0001: 0001-fix-metric-proxy-init-failed-error.patch
Patch0002: 0002-update-callback-request-headers.patch Patch0002: 0002-add-key-authentication-for-add-host-api.patch
Patch0003: 0003-fix-bash-file-sync-error.patch Patch0003: 0003-fix-python-prometheus-api-client-import-error.patch
Patch0004: 0004-bugfix-cve-fix-result-parsing-error.patch Patch0004: 0004-update-the-template-file-contents-for-adding-hosts.patch
Patch0005: 0005-bugfix-update-host-api-request-error.patch
Patch0006: 0006-optimize-import.patch
Patch0007: 0007-add-an-api-for-querying-file-list.patch
Patch0008: 0008-bugfix-email-config-does-not-take-effect.patch
Patch0009: 0009-fix-import-error.patch
BuildRequires: python3-setuptools BuildRequires: python3-setuptools
Requires: aops-vulcanus >= v1.2.0 Requires: aops-vulcanus >= v1.3.0
Requires: python3-marshmallow >= 3.13.0 python3-flask python3-flask-restful python3-gevent Requires: python3-marshmallow >= 3.13.0 python3-flask python3-flask-restful python3-gevent
Requires: python3-requests python3-uWSGI python3-sqlalchemy python3-werkzeug python3-PyMySQL Requires: python3-requests python3-uWSGI python3-sqlalchemy python3-werkzeug python3-PyMySQL
Requires: python3-paramiko >= 2.11.0 python3-redis python3-prometheus-api-client Requires: python3-paramiko >= 2.11.0 python3-redis
Provides: aops-zeus Provides: aops-zeus
Conflicts: aops-manager Conflicts: aops-manager
@ -49,6 +54,25 @@ cp -r database %{buildroot}/opt/aops/
%changelog %changelog
* Mon Nov 06 2023 wenxin<wenxin32@foxmail.com> - v1.3.1-5
- Bugfix: email config does not take effect
- Add an interface for querying file list
* Fri Oct 27 2023 liulei<450962@qq.com> - v1.3.1-4
- Bugfix: update host api request error when changing username
* Thu Oct 26 2023 wenxin<wenxin32@foxmail.com> - v1.3.1-3
- update the template file contents for adding hosts
* Wed Oct 18 2023 wenxin<wenxin32@foxmail.com> - v1.3.1-2
- fix bug: metric proxy init failed
- add a way about key authentication for add host api
- remove python3-prometheus-api-client
* Thu Sep 21 2023 wenxin<shusheng.wen@outlook.com> - v1.3.1-1
- update spec requires
- update version info in setup.py file
* Tue Sep 19 2023 wenxin<shusheng.wen@outlook.com> - v1.3.0-4 * Tue Sep 19 2023 wenxin<shusheng.wen@outlook.com> - v1.3.0-4
- bugfix: cve fix result parsing error - bugfix: cve fix result parsing error