Apply limit time filter.

Signed-off-by: Shriya Deshmukh <[email protected]>

Author: shriya-deshmukh

src/rgw/support/rgw_support_bundle

@@ -32,7 +32,8 @@ from cortx.utils.conf_store.conf_store import Conf, MappedConf
 from cortx.utils.support_framework.log_filters import FilterLog
 from cortx.rgw.const import (
     CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE,
-    SVC_FILE_PERCENTAGE)
+    SVC_FILE_PERCENTAGE, LOG_DATE_REGEX, LOG_TIME_REGEX,
+    DATETIME_DATE_REGEX, DATETIME_TIME_REGEX)
 
 
 class SupportBundleError(BaseError):
@@ -98,34 +99,22 @@ class RGWSupportBundle:
             # files in delim_T use timestamp syntax as
             # 2022-07-10T05:28:35.570+0000
             from cortx.utils.support_framework.log_filters import FilterLog
-            from cortx.rgw.const import (RGW_STARTUP_LOG, RGW_SETUP_LOG,
-                RGW_SUPPORT_BUNDLE_LOG, RADOSGW_ADMIN_LOG, RGW_1_LOG,
-                LOG_DATE_REGEX, LOG_TIME_REGEX, DATETIME_DATE_REGEX,
-                DATETIME_TIME_REGEX)
-
-            # log files with timestamp syntax "YYYY-mm-DD H:M:S"
-            delim_space = [RGW_STARTUP_LOG, RGW_SETUP_LOG, RGW_SUPPORT_BUNDLE_LOG]
-            # log files with timestamp syntax "YYYY-mm-DDTH:M:S"
-            delim_T = [RADOSGW_ADMIN_LOG, RGW_1_LOG]
+            temp_path_src = os.path.join('/tmp', 'rgw_src')
+            os.makedirs(temp_path_src, exist_ok=True)
+            # Move text logs and rotated log files to temp dir.
             for file in os.listdir(RGWSupportBundle._tmp_src):
-                if file in delim_space:
-                    delim = ' '
-                elif file in delim_T:
-                    delim = 'T'
-                else:
+                if file.endswith('.conf') or file == "installed-cortx-rpms.txt":
                     continue
-                log_timestamp_regex = LOG_DATE_REGEX + delim + LOG_TIME_REGEX
-                datetime_format = DATETIME_DATE_REGEX + delim + DATETIME_TIME_REGEX
-                # apply log filter to individual file by passing exact filename
-                # to file_name_reg_ex
-                FilterLog.limit_time(
-                    src_dir=RGWSupportBundle._tmp_src,
-                    dest_dir=RGWSupportBundle._tmp_src,
-                    duration=duration,
-                    file_name_reg_ex=file,
-                    log_timestamp_regex=log_timestamp_regex,
-                    datetime_format=datetime_format)
-
+                shutil.move(os.path.join(RGWSupportBundle._tmp_src, file),
+                    os.path.join(temp_path_src, file))
+            temp_path = os.path.join('/tmp', 'rgw_time_based_log')
+            RGWSupportBundle._apply_limit_time(temp_path_src, temp_path, 'rgw-*', duration)
+            RGWSupportBundle._apply_limit_time(temp_path_src, temp_path, 'rgw_*', duration)
+            RGWSupportBundle._apply_limit_time(temp_path_src, temp_path, 'radosgw*', duration)
+            # delete temp paths.
+            shutil.rmtree(temp_path)
+            shutil.rmtree(temp_path_src)
+            Log.info('Collected log files based on time duration.')
         else:
             Log.error("RGW log file does not exists hence skipping log file collection.")
 
@@ -233,6 +222,31 @@ class RGWSupportBundle:
         RGWSupportBundle._generate_tar(bundle_id, target_path)
         RGWSupportBundle._cleanup()
 
+    @staticmethod
+    def _apply_limit_time(temp_path_src: str, path: str, reg_ex: str, duration: str):
+        """Apply limit_time filter."""
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        os.makedirs(path, exist_ok=True)
+        for file in os.listdir(temp_path_src):
+            regex = re.compile(f'({reg_ex})')
+            if regex.match(file) and file.startswith(reg_ex[:-1]):
+                shutil.move(os.path.join(temp_path_src, file),
+                    os.path.join(path, file))
+        if reg_ex == 'rgw-*' or reg_ex == 'radosgw*':
+            delim = 'T'
+        else:
+            delim = ' '
+        log_timestamp_regex = LOG_DATE_REGEX + delim + LOG_TIME_REGEX
+        datetime_format = DATETIME_DATE_REGEX + delim + DATETIME_TIME_REGEX
+        Log.info(f'apply limit-time filter on {reg_ex} files.')
+        # apply limit_time log filter to file names started by reg_ex
+        FilterLog.limit_time(src_dir=path, dest_dir=RGWSupportBundle._tmp_src,
+            duration=duration, file_name_reg_ex=reg_ex[:-1],
+            log_timestamp_regex=log_timestamp_regex,
+            datetime_format=datetime_format)
+
+
     @staticmethod
     def _get_dir_size(dir_path: str):
         """Calculate total directory size."""
@@ -285,7 +299,7 @@ class RGWSupportBundle:
             if size_limit:
                 remaining_size_limit = RGWSupportBundle._get_remaining_folder_size(
                     RGWSupportBundle._tmp_src, size_limit)
-            if (os.path.getsize(addb_dir) > int(remaining_size_limit[:-1])):
+            if (RGWSupportBundle._get_dir_size(addb_dir) > int(remaining_size_limit[:-1])):
                 Log.warn('Exhausted support bundle size limit while '
                     'collecting addb files.')
                 break