packages/aops-zeus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
aops-zeus/0003-fix-python-prometheus-api-client-import-error.patch
liulei 0308b441eb init aops-zeus in 20.03 SP4
2023-11-06 15:45:48 +08:00

644 lines
24 KiB
Diff
Raw Blame History

From cca4f59f2c68b20804fae06692179b8d3e181327 Mon Sep 17 00:00:00 2001
From: rabbitali <wenxin32@foxmail.com>
Date: Mon, 6 Nov 2023 09:36:24 +0800
Subject: [PATCH] fix python-prometheus-api-client import error
---
zeus/conf/constant.py | 4 -
zeus/database/proxy/metric.py | 458 --------------------------------
zeus/function/verify/metric.py | 28 --
zeus/metric_manager/__init__.py | 12 -
zeus/metric_manager/view.py | 51 ----
zeus/url.py | 9 -
6 files changed, 562 deletions(-)
delete mode 100644 zeus/database/proxy/metric.py
delete mode 100644 zeus/function/verify/metric.py
delete mode 100644 zeus/metric_manager/__init__.py
delete mode 100644 zeus/metric_manager/view.py
diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py
index bf8792a..304ed7e 100644
--- a/zeus/conf/constant.py
+++ b/zeus/conf/constant.py
@@ -72,10 +72,6 @@ EXECUTE_CVE_FIX = '/manage/vulnerability/cve/fix'
EXECUTE_CVE_SCAN = '/manage/vulnerability/cve/scan'
EXECUTE_CVE_ROLLBACK = "/manage/vulnerability/cve/rollback"
-# metric config
-QUERY_METRIC_NAMES = '/manage/host/metric/names'
-QUERY_METRIC_DATA = '/manage/host/metric/data'
-QUERY_METRIC_LIST = '/manage/host/metric/list'
# auth login
GITEE_OAUTH = "https://gitee.com/oauth/authorize"
diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py
deleted file mode 100644
index 5fa75e9..0000000
--- a/zeus/database/proxy/metric.py
+++ /dev/null
@@ -1,458 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-"""
-Time: 2022-07-27
-Author: YangYunYi
-Description: Query raw data from Prometheus
-"""
-from typing import Dict, Tuple, List, Optional
-import datetime
-from prometheus_api_client import PrometheusApiClientException
-from vulcanus.database.proxy import PromDbProxy
-from vulcanus.log.log import LOGGER
-from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED
-from zeus.conf import configuration
-
-
-class MetricProxy(PromDbProxy):
- """
- Proxy of prometheus time series database
- """
-
- def __init__(self, host=None, port=None):
- """
- Init MetricProxy
-
- Args:
- host (str)
- port (int)
- """
- PromDbProxy.__init__(self, host, port)
- self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100
- self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s"
-
- @staticmethod
- def __metric_dict2str(metric: Dict) -> str:
- """
- Trans metric dict to string
- Args:
- metric (Dict):
- {
- "__name__": "metric name1",
- 'instance': '172.168.128.164:9100',
- 'job': 'prometheus',
- 'label1': 'label_value1'
- ...
- }
-
- Returns:
- metric_str(str): 'metric_name1{label1="value1", label2="values"}'
- """
-
- label_str = ""
- if "__name__" not in metric.keys():
- return label_str
- sorted_label_list = sorted(metric.items(), reverse=False)
- for label in sorted_label_list:
- # The job label is usually "prometheus" in this framework and
- # has no effect on subsequent data requests, so remove it to save space.
- # __name__ label move to the front.
- metric_key = label[0]
- metric_value = label[1]
- if metric_key in ["__name__", "job"]:
- continue
- label_str += "%s=\"%s\"," % (metric_key, metric_value)
-
- if label_str == "":
- # It's a metric with only a name and no personalized label
- ret = metric["__name__"]
- else:
- # Remove the comma of the last element
- ret = "%s{%s}" % (metric["__name__"], label_str[:-1])
- return ret
-
- def query_metric_names(self, query_ip: str) -> Tuple[int, dict]:
- """
- Query data
- Args:
- query_ip(str): query ip address
-
- Returns:
- int: status code
- dict: e.g
- {
- "results": ["metric1", "metric2"]
- }
- """
-
- query_ip = query_ip["query_ip"]
- query_host = {"host_id": "query_host_id", "host_ip": query_ip}
-
- query_metric_names = []
- res = {'results': query_metric_names}
-
- host_ip = query_host["host_ip"]
- host_port = query_host.get("instance_port", self.default_instance_port)
-
- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port)
-
- if ret != SUCCEED:
- return ret, res
-
- for metric in metric_list:
- metric = metric.split('{')[0]
- if metric not in query_metric_names:
- query_metric_names.append(metric)
-
- return ret, res
-
- def query_metric_list(self, data: Dict[str, str]) -> Tuple[int, dict]:
- """
- Query metric list
- Args:
- data(dict): param e.g
- {
- "query_ip": "str",
- "metric_names": ["metric1", "metric2"]
- }
-
- Returns:
- int: status code
- dict: e.g
- {
- 'results': {
- "metric1": [
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- ],
- "metric2": [
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- ]
- }
- }
- """
-
- query_ip = data.get('query_ip')
- query_metric = data.get('metric_names')
-
- query_host = {"host_id": "query_host_id", "host_ip": query_ip}
-
- query_metric_list = {}
- res = {'results': query_metric_list}
-
- host_ip = query_host["host_ip"]
- host_port = query_host.get("instance_port", self.default_instance_port)
-
- ret, metric_list = self.query_metric_list_of_host(
- host_ip,
- host_port,
- )
-
- if ret != SUCCEED:
- LOGGER.warning("Host metric list query error")
- return ret, res
-
- for query_metric_name in query_metric:
- query_metric_list[query_metric_name] = []
- for metric in metric_list:
- metric_name = metric.split('{')[0]
- if metric_name != query_metric_name:
- continue
- query_metric_list[query_metric_name].append(metric)
-
- return ret, res
-
- def query_metric_data(self, data: Dict[str, str]) -> Tuple[int, dict]:
- """
- Query metric data
- Args:
- data(dict): param e.g
- {
- "time_range": "list",
- "query_ip": "str",
- "query_info": {
- "metric1": [
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- "metric1{label1="label1_value", label2="label2_value", ..., }",
- ],
- "metric2": [
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- "metric2{label1="label1_value", label2="label2_value", ..., }",
- ]
- }
- }
-
- Returns:
- int: status code
- dict: e.g
- {
- 'results': {
- "metric1": {
- "metric1{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- "metric1{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- },
- "metric1": {
- "metric2{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- "metric2{label1="label1_value", label2="label2_value", ..., }": [
- [1658926441, '0'],
- [1658926441, '0']
- ],
- },
- }
- }
- """
- time_range = data.get('time_range')
- query_ip = data.get('query_ip')
- query_info = data.get('query_info')
-
- query_host = {"host_id": "query_host_id", "host_ip": query_ip}
-
- host_ip = query_host["host_ip"]
- host_port = query_host.get("instance_port", self.default_instance_port)
-
- query_host_list = [query_host]
- query_data = {}
- res = {'results': query_data}
-
- # adjust query range step based on query time range
- # the default query range step is 15 seconds
- QUERY_RANGE_STEP = 15
- query_range_step = QUERY_RANGE_STEP
- query_range_seconds = time_range[1] - time_range[0]
- total_query_times = query_range_seconds / query_range_step
-
- while total_query_times > 11000:
- query_range_step += 15
- total_query_times = query_range_seconds / query_range_step
-
- def add_two_dim_dict(thedict, key_a, key_b, val):
- if key_a in thedict:
- thedict[key_a].update({key_b: val})
- else:
- thedict.update({key_a: {key_b: val}})
-
- if not query_info:
- return SUCCEED, res
-
- for metric_name, metric_list in query_info.items():
- if not metric_list:
- _, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric_name)
- if not metric_list:
- query_data[metric_name] = []
- for metric_info in metric_list:
- data_status, monitor_data = self.query_data(
- time_range=time_range,
- host_list=query_host_list,
- metric=metric_info,
- adjusted_range_step=query_range_step,
- )
- values = []
- if data_status == SUCCEED:
- values = monitor_data[query_host["host_id"]][metric_info]
- add_two_dim_dict(query_data, metric_name, metric_info, values)
-
- return SUCCEED, res
-
- def query_data(
- self,
- time_range: List[int],
- host_list: list,
- metric: Optional[str] = None,
- adjusted_range_step: Optional[int] = None,
- ) -> Tuple[int, Dict]:
- """
- Query data
- Args:
- time_range(list): time range
- host_list(list): host list, If the port is not specified, the default value is used
- [{"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 9100},
- {"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 8080},
- {"host_id": "id2", "host_ip": "172.168.128.165"}]
-
- Returns:
- ret(int): query ret
- host_data_list(dict): host data list ret
- {
- 'id1': {
- 'metric1'{instance="172.168.128.164:9100",label1="value2"}':
- [[time1, 'value1'],
- [time2, 'value2'],
- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty
- 'metric13{instance="172.168.128.164:8080"}': None => get no data
- },
- 'id2': None => get no metric list of this host
- }
- """
-
- host_data_list = {}
- if not host_list:
- return PARAM_ERROR, host_data_list
-
- status = SUCCEED
- for host in host_list:
- host_id = host["host_id"]
- host_ip = host["host_ip"]
- host_port = host.get("instance_port", self.default_instance_port)
- if host_id not in host_data_list.keys():
- host_data_list[host_id] = None
-
- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric)
- if ret != SUCCEED:
- status = PARTIAL_SUCCEED
- host_data_list[host_id] = None
- continue
- ret, data_list = self.__query_data_by_host(metric_list, time_range, adjusted_range_step)
- if ret != SUCCEED:
- status = PARTIAL_SUCCEED
- if not host_data_list[host_id]:
- host_data_list[host_id] = data_list
- else:
- host_data_list[host_id].update(data_list)
-
- return status, host_data_list
-
- def __parse_metric_data(self, metric_data: List) -> List[str]:
- """
- Parse metric data from prometheus to name<-> label_config dict
- Args:
- metric_data(List): metric list data from prometheus
- [{'metric':{
- "__name__": "metric_name1",
- 'instance': '172.168.128.164:9100',
- 'job': 'prometheus',
- 'label1': 'label_value1'
- ...
- },
- 'value': [1658926441.526, '0']
- }]
-
- Returns:
- metric_list(List[str]):
- [
- 'metric_name1{label1="value1", label2="value2"}'
- ]
- """
-
- metric_list = []
- for metric_item in metric_data:
- metric_dict = metric_item["metric"]
- metric_str = self.__metric_dict2str(metric_dict)
- if metric_str == "":
- continue
- metric_list.append(metric_str)
-
- return metric_list
-
- def query_metric_list_of_host(
- self, host_ip: str, host_port: Optional[int] = None, metric: Optional[str] = None
- ) -> Tuple[int, List[str]]:
- """
- Query metric list of a host
- Args:
- host_ip(str): host ip
- host_port(int): host port
-
- Returns:
- ret(int): query ret
- metric_list(list): metric list ret
- [
- 'metric_name1{label1="value1", label2="value2"}'
- ]
- """
- if not host_port:
- host_port = self.default_instance_port
- query_str = "{instance=\"%s:%s\"}" % (host_ip, str(host_port))
- if metric is not None:
- if metric.find('{') != -1:
- query_str = metric
- else:
- query_str = metric + query_str
- try:
- data = self._prom.custom_query(query=query_str)
- if not data:
- if query_str.startswith('{'):
- LOGGER.error(
- "Query metric list result is empty. "
- "Can not get metric list of host %s:%s " % (host_ip, host_port)
- )
- return NO_DATA, []
- metric_list = self.__parse_metric_data(data)
- return SUCCEED, metric_list
-
- except (ValueError, TypeError, PrometheusApiClientException) as error:
- LOGGER.error("host %s:%d Prometheus query metric list failed. %s" % (host_ip, host_port, error))
- return DATABASE_QUERY_ERROR, []
-
- def __query_data_by_host(
- self, metrics_list: List[str], time_range: List[int], adjusted_range_step: Optional[int] = None
- ) -> Tuple[int, Dict]:
- """
- Query data of a host
- Args:
- metrics_list(list): metric list of this host
- time_range(list): time range to query
-
- Returns:
- ret(int): query ret
- data_list(list): data list ret
- {
- 'metric1'{instance="172.168.128.164:9100",label1="value2"}':
- [[time1, 'value1'],
- [time2, 'value2'],
- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty
- 'metric13{instance="172.168.128.164:8080"}': None => get no data
- }
-
- """
- start_time = datetime.datetime.fromtimestamp(time_range[0])
- end_time = datetime.datetime.fromtimestamp(time_range[1])
-
- query_range_step = self.query_range_step
- if adjusted_range_step is not None:
- query_range_step = adjusted_range_step
-
- data_list = {}
- ret = SUCCEED
- for metric in metrics_list:
- try:
- data = self._prom.custom_query_range(
- query=metric, start_time=start_time, end_time=end_time, step=query_range_step
- )
- if not data or "values" not in data[0]:
- LOGGER.debug(
- "Query data result is empty. "
- "metric %s in %d-%d doesn't record in the prometheus " % (metric, time_range[0], time_range[1])
- )
- data_list[metric] = None
- ret = PARTIAL_SUCCEED
- continue
- data_list[metric] = data[0]["values"]
-
- except (ValueError, TypeError, PrometheusApiClientException) as error:
- LOGGER.error(
- "Prometheus metric %s in %d-%d query data failed. %s"
- % (metric, time_range[0], time_range[1], error)
- )
- data_list[metric] = None
- ret = PARTIAL_SUCCEED
- return ret, data_list
diff --git a/zeus/function/verify/metric.py b/zeus/function/verify/metric.py
deleted file mode 100644
index 5df28a2..0000000
--- a/zeus/function/verify/metric.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-from marshmallow import Schema, fields
-
-
-class QueryHostMetricNamesSchema(Schema):
- query_ip = fields.String(required=True)
-
-
-class QueryHostMetricDataSchema(Schema):
- time_range = fields.List(fields.Integer, required=True)
- query_ip = fields.String(required=True)
- query_info = fields.Dict()
-
-
-class QueryHostMetricListSchema(Schema):
- query_ip = fields.String(required=True)
- metric_names = fields.List(fields.String)
diff --git a/zeus/metric_manager/__init__.py b/zeus/metric_manager/__init__.py
deleted file mode 100644
index e90ecf9..0000000
--- a/zeus/metric_manager/__init__.py
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
diff --git a/zeus/metric_manager/view.py b/zeus/metric_manager/view.py
deleted file mode 100644
index 4d98cf2..0000000
--- a/zeus/metric_manager/view.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-from vulcanus.restful.response import BaseResponse
-from zeus.database.proxy.metric import MetricProxy
-from zeus.function.verify.metric import QueryHostMetricDataSchema, QueryHostMetricListSchema, QueryHostMetricNamesSchema
-
-
-class QueryHostMetricNames(BaseResponse):
- """
- Interface for query host metric names from web.
- Restful API: GET
- """
-
- @BaseResponse.handle(schema=QueryHostMetricNamesSchema, proxy=MetricProxy)
- def get(self, callback: MetricProxy, **params):
- status_code, result = callback.query_metric_names(params)
- return self.response(code=status_code, data=result)
-
-
-class QueryHostMetricData(BaseResponse):
- """
- Interface for query host metric data from web.
- Restful API: POST
- """
-
- @BaseResponse.handle(schema=QueryHostMetricDataSchema, proxy=MetricProxy)
- def post(self, callback: MetricProxy, **params):
- status_code, result = callback.query_metric_data(params)
- return self.response(code=status_code, data=result)
-
-
-class QueryHostMetricList(BaseResponse):
- """
- Interface for query host metric list from web.
- Restful API: POST
- """
-
- @BaseResponse.handle(schema=QueryHostMetricListSchema, proxy=MetricProxy)
- def post(self, callback: MetricProxy, **params):
- status_code, result = callback.query_metric_list(params)
- return self.response(code=status_code, data=result)
diff --git a/zeus/url.py b/zeus/url.py
index 597dcc7..3ec8d21 100644
--- a/zeus/url.py
+++ b/zeus/url.py
@@ -43,9 +43,6 @@ from zeus.conf.constant import (
LOGOUT,
QUERY_HOST,
QUERY_HOST_DETAIL,
- QUERY_METRIC_DATA,
- QUERY_METRIC_LIST,
- QUERY_METRIC_NAMES,
REFRESH_TOKEN,
UPDATE_HOST,
USER_LOGIN,
@@ -53,7 +50,6 @@ from zeus.conf.constant import (
)
from zeus.config_manager import view as config_view
from zeus.host_manager import view as host_view
-from zeus.metric_manager import view as metric_view
from zeus.vulnerability_manage import view as vulnerability_view
URLS = []
@@ -100,11 +96,6 @@ SPECIFIC_URLS = {
(vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX),
(vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK),
],
- 'METRIC': [
- (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES),
- (metric_view.QueryHostMetricData, QUERY_METRIC_DATA),
- (metric_view.QueryHostMetricList, QUERY_METRIC_LIST),
- ],
}
for _, value in SPECIFIC_URLS.items():
--
2.33.0
Reference in New Issue View Git Blame Copy Permalink