From c7b84c97fde88b30f39f01e3345e2649714c2b7e Mon Sep 17 00:00:00 2001
From: young <954906362@qq.com>
Date: Thu, 27 Apr 2023 17:25:40 +0800
Subject: [PATCH] bug fix:continuously sending emails when can't read config info
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apollo/cron/download_sa_manager.py            | 43 ++++++-------------
 apollo/cron/manager/__init__.py               | 11 +++--
 apollo/cron/timed_correct_manager.py          |  2 +-
 apollo/database/proxy/cve.py                  |  2 +-
 .../task_handler/manager/scan_manager.py      |  9 ++--
 conf/apollo_crontab.ini                       |  3 +-
 6 files changed, 28 insertions(+), 42 deletions(-)

diff --git a/apollo/cron/download_sa_manager.py b/apollo/cron/download_sa_manager.py
index 7b3cea2..0bb6ef8 100644
--- a/apollo/cron/download_sa_manager.py
+++ b/apollo/cron/download_sa_manager.py
@@ -17,7 +17,6 @@ import shutil
 import sqlalchemy
 import requests
 import retrying
-from lxml import etree
 from retrying import retry
 
 from apollo.conf import configuration
@@ -37,16 +36,10 @@ class TimedDownloadSATask(TimedTaskBase):
     Timed download sa tasks
     """
     config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH)
-    security_base_url = config_info.get(
-        "download_sa", dict()).get("base_url", "")
-    if not security_base_url:
-        LOGGER.error("Please add base_url in configuration file")
-    advisory_years = config_info.get("download_sa", dict()).get("sa_years", "")
-    if advisory_years:
-        advisory_years = re.findall(r"\d{4}", advisory_years)
-    else:
-        LOGGER.error("Please add sa_years in configuration file")
-        raise KeyError
+    cvrf_url = config_info.get(
+        "download_sa", dict()).get("cvrf_url", "")
+    if not cvrf_url:
+        LOGGER.error("Please add cvrf_url in configuration file")
 
     save_sa_record = []
 
@@ -152,7 +145,7 @@ class TimedDownloadSATask(TimedTaskBase):
 
         for sa_name in sa_name_list:
             advisory_year, advisory_serial_number = re.findall("\d+", sa_name)
-            sa_url = f"{TimedDownloadSATask.security_base_url}/{advisory_year}/{sa_name}"
+            sa_url = f"{TimedDownloadSATask.cvrf_url}/{advisory_year}/{sa_name}"
             try:
                 response = TimedDownloadSATask.get_response(sa_url)
                 if response:
@@ -170,22 +163,21 @@ class TimedDownloadSATask(TimedTaskBase):
                                                            "download_status": False})
 
     @staticmethod
-    def get_advisory_url_list(url: str) -> list:
+    def get_advisory_url_list() -> list:
         """
         Send a request and parse the data on the page to get all the security bulletins url and store them in the list
 
-        Args:
-            url(str): the url of all security announcements
-
         Returns:
             list: security url list
         """
         try:
-            response = TimedDownloadSATask.get_response(url)
+            response = TimedDownloadSATask.get_response(TimedDownloadSATask.cvrf_url + "/index.txt")
             if response:
-                html = etree.HTML(response.text)
+                html = response.text
+                sa_list = html.replace("\r", "").split("\n")
                 security_files = [
-                    file for file in html.xpath("//*[@id='list']/tbody/tr/td[1]/a/@title")
+                    # 2021/cvrf-openEuler-SA-2021-1022.xml, we don't need the first five characters
+                    sa_name[5:] for sa_name in sa_list
                 ]
                 return security_files
             else:
@@ -206,18 +198,7 @@ class TimedDownloadSATask(TimedTaskBase):
         Returns:
             list: The name of the sa that needs to be downloaded
         """
-        today = datetime.datetime.today()
-        current_year = str(today.year)
-        TimedDownloadSATask.advisory_years.append(current_year)
-        all_sa_name_list = []
-
-        for year in set(TimedDownloadSATask.advisory_years):
-            try:
-                sa_name_list = TimedDownloadSATask.get_advisory_url_list(
-                    TimedDownloadSATask.security_base_url + "/" + str(year) + "/")
-                all_sa_name_list.extend(sa_name_list)
-            except retrying.RetryError:
-                LOGGER.error(f"Failed to get the {year} sa file")
+        all_sa_name_list = TimedDownloadSATask.get_advisory_url_list()
 
         succeed_sa_name_list = []
         for succeed_record in download_succeed_record:
diff --git a/apollo/cron/manager/__init__.py b/apollo/cron/manager/__init__.py
index 1ec9b72..1f3a855 100644
--- a/apollo/cron/manager/__init__.py
+++ b/apollo/cron/manager/__init__.py
@@ -91,9 +91,12 @@ class TimedTaskManager():
                 return
             timed_task_parameters['id'] = task_id
 
-        if "base_url" in timed_task_parameters and "sa_years" in timed_task_parameters:
-            timed_task_parameters.pop("base_url")
-            timed_task_parameters.pop("sa_years")
+        if "day_of_week" not in timed_task_parameters or "hour" not in timed_task_parameters:
+            LOGGER.error("Create scheduled task is missing required  fields.")
+            return
+
+        if "cvrf_url" in timed_task_parameters:
+            timed_task_parameters.pop("cvrf_url")
         if "service_timeout_threshold_min" in timed_task_parameters:
             timed_task_parameters.pop("service_timeout_threshold_min")
 
@@ -102,7 +105,7 @@ class TimedTaskManager():
         if "auto_start" in timed_task_parameters:
             auto_start = timed_task_parameters['auto_start']
             timed_task_parameters.pop("auto_start")
-            if not auto_start:
+            if auto_start == "false":
                 LOGGER.info(f"{task_id}, This task is configured to not start.")
                 return
         TimedTaskManager._APscheduler.add_job(**timed_task_parameters)
diff --git a/apollo/cron/timed_correct_manager.py b/apollo/cron/timed_correct_manager.py
index 4f1459e..fc2f62a 100644
--- a/apollo/cron/timed_correct_manager.py
+++ b/apollo/cron/timed_correct_manager.py
@@ -33,7 +33,7 @@ class TimedCorrectTask(TimedTaskBase):
     """
     config_info = get_timed_task_config_info(TIMED_TASK_CONFIG_PATH)
     SERVICE_TIMEOUT_THRESHOLD_MIN = config_info.get(
-        "correct_data").get("service_timeout_threshold_min")
+        "correct_data").get("service_timeout_threshold_min", 15)
 
     @staticmethod
     def task_enter():
diff --git a/apollo/database/proxy/cve.py b/apollo/database/proxy/cve.py
index 1a77211..75ab71c 100644
--- a/apollo/database/proxy/cve.py
+++ b/apollo/database/proxy/cve.py
@@ -951,7 +951,7 @@ class CveProxy(CveMysqlProxy, CveEsProxy):
         """
         Save the record of download sa
         Args:
-            sa_record_rows(list): aach element is a record of the AdvisoryDownloadRecord table,e.g.
+            sa_record_rows(list): each element is a record of the AdvisoryDownloadRecord table,e.g.
                 [{"advisory_year": 2022,
                 "advisory_serial_number": 1230,
                 "download_status": 1}]
diff --git a/apollo/handler/task_handler/manager/scan_manager.py b/apollo/handler/task_handler/manager/scan_manager.py
index efd6356..48ee71b 100644
--- a/apollo/handler/task_handler/manager/scan_manager.py
+++ b/apollo/handler/task_handler/manager/scan_manager.py
@@ -195,16 +195,19 @@ class ScanManager(Manager):
         tmp = {}
         chart_data = []
         file_content = "序号,CVE_ID,主机IP,主机名称,CVSS评分,评分级别\n"
-        for num, row in enumerate(rows, 1):
+        excel_row_num = 1
+        for row in rows:
             if row.host_ip in tmp:
                 tmp[row.host_ip]["count"] += 1
-                file_content += f"{num},{row.cve_id},{row.host_ip}," \
+                file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \
                                 f"{row.host_name},{row.cvss_score},{row.severity}\n"
+                excel_row_num += 1
             else:
                 if row.cve_id is not None:
                     tmp[row.host_ip] = {"count": 1, "host_name": row.host_name}
-                    file_content += f"{num},{row.cve_id},{row.host_ip}," \
+                    file_content += f"{excel_row_num},{row.cve_id},{row.host_ip}," \
                                     f"{row.host_name},{row.cvss_score},{row.severity}\n"
+                    excel_row_num += 1
                 else:
                     tmp[row.host_ip] = {"count": 0, "host_name": row.host_name}
 
diff --git a/conf/apollo_crontab.ini b/conf/apollo_crontab.ini
index 3ac8eeb..7ab1bfa 100644
--- a/conf/apollo_crontab.ini
+++ b/conf/apollo_crontab.ini
@@ -13,8 +13,7 @@ id = download sa
 day_of_week = 0-6
 hour = 3
 auto_start = true
-base_url = https://repo.openeuler.org/security/data/cvrf
-sa_years = 2021,2022,2023
+cvrf_url = https://repo.openeuler.org/security/data/cvrf
 
 [correct_data]
 id = correct data
-- 
Gitee


