CORTX-34147: Add time stamp for consul process state updated in bq (#2163)

Problem:
HARE bq messages in Support Bundle do not contain timestamps.
This is needed for effective offline debug using logs from Support bundle.

Solution:
Add timestamps to bq publisher; also add the consul kv dump so generated to
Support Bundle.

Signed-off-by: pavankrishnat <[email protected]>

Author: pavankrishnat

hax/hax/handler.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 Seagate Technology LLC and/or its Affiliates
+# Copyright (c) 2020-2022 Seagate Technology LLC and/or its Affiliates
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 # please email [email protected] or [email protected].
 #
 
+import datetime
 import logging
 from typing import List, Any
 from hax.message import (BroadcastHAStates, Die, EntrypointRequest,
@@ -78,7 +79,10 @@ def broadcast_process_state(self, event: ConfHaProcess):
                 'state': ha_process_events[event.chp_event],
                 'type': event_type}
         payload = dump_json(data)
-        res = self.bq_publisher.publish('PROCESS-STATE-UPDATE', payload)
+        now_time = str(datetime.datetime.now())
+        res = self.bq_publisher.publish(now_time,
+                                        'PROCESS-STATE-UPDATE',
+                                        payload)
         LOG.debug('PROCESS-STATE-UPDATE event JSON: %s res: %d',
                   payload, res)
 

hax/hax/queue/publish.py

@@ -6,7 +6,8 @@
 # XXX do we want to make payload definition more strict?
 # E.g. there could be a type hierarchy for payload objects that depends
 # on the type name.
-Message = NamedTuple('Message', [('message_type', str),
+Message = NamedTuple('Message', [('timestamp', str),
+                                 ('message_type', str),
                                  ('payload', Dict[str, Any])])
 
 
@@ -22,10 +23,12 @@ def __init__(self,
         self.epoch_key = epoch_key
 
     @repeat_if_fails(wait_seconds=0.1)
-    def publish(self, message_type: str, payload: str) -> int:
+    def publish(self, timestamp: str, message_type: str, payload: str) -> int:
         """Publishes the given message to the queue."""
         data = simplejson.loads(payload)
-        message = Message(message_type=message_type, payload=data)
+        message = Message(timestamp=timestamp,
+                          message_type=message_type,
+                          payload=data)
         data = simplejson.dumps(message)
 
         while True:

utils/hare-reportbug

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2020 Seagate Technology LLC and/or its Affiliates
+# Copyright (c) 2020-2022 Seagate Technology LLC and/or its Affiliates
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -130,6 +130,9 @@ sudo systemctl --all --full --no-pager status {hare,m0,motr,s3}\* \
 
 running_process_list
 
+# consul kv output
+consul kv get --recurse > consul_kv.log
+
 # cluster status
 sudo timeout --kill-after 30 15 hctl status \
      > hctl-status.txt || true