Addressed review comments.

Signed-off-by: Shriya Deshmukh <[email protected]>

Author: shriya-deshmukh

py-utils/src/utils/support_framework/binary_search.py

@@ -0,0 +1,43 @@
+import mmap
+import os
+
+
+def binarySearch(f, pos, total_size, start_time):
+    while pos <= total_size:
+        mid = int(pos + (total_size - 1)/2)
+        # Check if x is present at mid
+        # mid_slice = mm[mid:(mid+1)]
+        # mid_slice = mid_slice.decode('utf-8').strip()
+        mid = int(total_size/2)
+        f.seek(mid, 0)
+        line = f.readline()
+        print(mid)
+        if mid_slice == start_time:
+            return mid
+        # If x is greater, ignore left half
+        elif mid_slice < start_time:
+            pos = mid + 1
+            print(pos)
+        # If x is smaller, ignore right half
+        else:
+            total_size = mid - 1
+            print(total_size)
+
+    # If we reach here, then the element
+    # was not present
+    return -1
+
+# text=f.readlines()
+# data=text
+#print data
+start_time = ''
+end_time = ''
+with open('rgw-1.log', 'r') as f:
+    # mm = mmap.mmap(f.fileno(), 0)
+    f.seek(0, os.SEEK_END)
+    size = f.tell()
+    result = binarySearch(f, 0, size, start_time)
+    if result != -1:
+        print("Element is present at index % d" % result)
+    else:
+        print("Element is not present in array")

py-utils/src/utils/support_framework/log_filters.py

@@ -183,32 +183,30 @@ def limit_time(src_dir, dest_dir, duration, file_name_reg_ex,
             + f"unexpected characters: {invalid_chars}")
 
         include_lines_without_timestamp = False
-        supported_file_types = ['.log', '.gz']
+        supported_file_types = ['.log', '.txt']
         start_time, end_time = FilterLog._parse_duration(duration)
         Log.info(f'start_time = {start_time}, end_time = {end_time}')
         # sort files based on timestamp
-        list_of_files = filter(lambda f: os.path.isfile(os.path.join(src_dir, f)),
-            os.listdir(src_dir))
+        list_of_files = filter(lambda f: os.path.isfile(
+            os.path.join(src_dir, f)), os.listdir(src_dir))
         # sort the files based on last modification time in descending order
         list_of_files = sorted(list_of_files,
             key=lambda f: os.path.getmtime(os.path.join(src_dir, f)),
             reverse=True)
         for file in list_of_files:
             log_scope_exceeded = False
-            is_log_written_to_file = False
-            file_extension = pathlib.Path(file).suffix
-            # Ignore processing of other file format.
-            if file_extension not in supported_file_types:
-                Log.warn(f'{file} file is skipped..')
-                continue
+            log_written_to_file = False
             op_file = os.path.join(dest_dir, 'tmp_' + file)
             if file.startswith(file_name_reg_ex):
                 in_file = os.path.join(src_dir, file)
-                if file.endswith('.gz'):
-                    # File modification time is in
+                file_extension = pathlib.Path(file).suffix
+                if file_extension not in supported_file_types:
                     FilterLog._collect_rotated_log_file(
                         in_file, dest_dir, start_time, end_time)
                     continue
+                # TODO: Instead of processing the file line by line,
+                # apply binary search on file so that we can get log lines
+                # in Log(N) complexity.
                 with open(in_file, 'r') as fd_in, open(op_file, 'a') as fd_out:
                     line = fd_in.readline()
                     while (line):
@@ -220,7 +218,7 @@ def limit_time(src_dir, dest_dir, duration, file_name_reg_ex,
                                 log_time = datetime.strptime(log_duration, datetime_format)
                                 if start_time <= log_time and log_time <= end_time:
                                     include_lines_without_timestamp = True
-                                    is_log_written_to_file = True
+                                    log_written_to_file = True
                                     fd_out.write(line)
                                 elif log_time > end_time and include_lines_without_timestamp:
                                     include_lines_without_timestamp = False
@@ -233,11 +231,11 @@ def limit_time(src_dir, dest_dir, duration, file_name_reg_ex,
                             # flag include_lines_without_timestamp = True
                             except ValueError:
                                 if include_lines_without_timestamp:
-                                    is_log_written_to_file = True
+                                    log_written_to_file = True
                                     fd_out.write(line)
                         line = fd_in.readline()
                 try:
-                    if is_log_written_to_file:
+                    if log_written_to_file:
                         final_op_file = os.path.join(dest_dir, file)
                         os.rename(op_file, final_op_file)
                     else: