update logic for setting metric

Author: juanhotencoding

receiver/splunkenterprisereceiver/documentation.md

@@ -417,8 +417,8 @@ The status of a rolling restart.
 
 | Name | Description | Values |
 | ---- | ----------- | ------ |
-| splunk.searchable.restart | String that indicates if a searchable rolling restart/upgrade in progress.t | Any Str |
-| splunk.rollingorrestart | String that indicates if there is a rolling restart or rolling upgrade in progress. | Any Str |
+| splunk.searchable.restart | Boolean that indicates if a searchable rolling restart/upgrade in progress.t | Any Bool |
+| splunk.rollingorrestart | Boolean that indicates if there is a rolling restart or rolling upgrade in progress. | Any Bool |
 
 ### splunk.indexer.throughput
 

receiver/splunkenterprisereceiver/internal/metadata/generated_metrics.go

@@ -46,11 +46,11 @@ attributes:
     description: The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint
     type: string
   splunk.searchable.restart:
-    description: String that indicates if a searchable rolling restart/upgrade in progress.t
-    type: string
+    description: Boolean that indicates if a searchable rolling restart/upgrade in progress.t
+    type: bool
   splunk.rollingorrestart:
-    description: String that indicates if there is a rolling restart or rolling upgrade in progress.
-    type: string
+    description: Boolean that indicates if there is a rolling restart or rolling upgrade in progress.
+    type: bool
 
 metrics:
   splunk.license.index.usage:

receiver/splunkenterprisereceiver/internal/metadata/generated_metrics_test.go

@@ -102,6 +102,7 @@ func (s *splunkScraper) scrape(ctx context.Context) (pmetric.Metrics, error) {
 		s.scrapeKVStoreStatus,
 		s.scrapeSearchArtifacts,
 		s.scrapeHealth,
+		s.scrapeIndexerClusterManagerStatus,
 	}
 	errChan := make(chan error, len(metricScrapes))
 
@@ -1789,7 +1790,7 @@ func (s *splunkScraper) traverseHealthDetailFeatures(details healthDetails, now
 
 // Scrape Indexer Cluster Manger Status Endpoint
 func (s *splunkScraper) scrapeIndexerClusterManagerStatus(ctx context.Context, now pcommon.Timestamp, errs chan error) {
-	if !s.conf.MetricsBuilderConfig.Metrics.SplunkIndexerRollingRestartStatus.Enabled {
+	if !s.conf.MetricsBuilderConfig.Metrics.SplunkIndexerRollingrestartStatus.Enabled {
 		return
 	}
 
@@ -1816,27 +1817,11 @@ func (s *splunkScraper) scrapeIndexerClusterManagerStatus(ctx context.Context, n
 		return
 	}
 
-	restartOrUpgrade := "false"
-	if icms.Entry.Content.RollingRestartOrUpgrade {
-		restartOrUpgrade = "true"
-	}
-
-	searchable := "false"
-	if icms.Entry.Content.SearchableRolling {
-		searchable = "true"
-	}
 
-	restarting := icms.Entry.Content.RollingRestartOrUpgrade || icms.Entry.Content.SearchableRolling || icms.Entry.Content.RollingRestartFlag
-
-	for _, mtsRestartOrUpgrade := range []string{"false", "true"} {
-		for _, mtsSearchable := range []string{"false", "true"} {
-			if restartOrUpgrade == mtsRestartOrUpgrade && searchable == mtsSearchable && restarting {
-				// If we're restarting, put a 1 in the gauge that matches the current state
-				s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 1, mtsRestartOrUpgrade, mtsSearchable)
-			} else {
-				// Put a zero in all the other gauges
-				s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 0, mtsRestartOrUpgrade, mtsSearchable)
-			}
+	for _, ic := range icms.Entries {
+		if ic.Content.RollingRestartOrUpgrade {
+			s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 1, ic.Content.SearchableRolling, ic.Content.RollingRestartFlag)
 		}
+		s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 0, ic.Content.SearchableRolling, ic.Content.RollingRestartFlag)
 	}
 }

receiver/splunkenterprisereceiver/metadata.yaml

@@ -54,6 +54,14 @@ func mockDispatchArtifacts(w http.ResponseWriter, _ *http.Request) {
 	_, _ = w.Write([]byte(`{"links":{},"origin":"https://somehost:8089/services/server/status/dispatch-artifacts","updated":"2024-10-24T04:46:47+00:00","generator":{"build":"05775df3af30","version":"9.2.2406.108"},"entry":[{"name":"result","id":"https://somehost:8089/services/server/status/dispatch-artifacts/result","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/server/status/dispatch-artifacts/result","list":"/services/server/status/dispatch-artifacts/result"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["*"],"write":[]},"removable":false,"sharing":"system"},"content":{"adhoc_count":"7","adhoc_size_mb":"1","adhoc_subsearch_count":"0","adhoc_subsearch_size_mb":"0","cached_job_status_info_csv_size_mb":"0","cached_job_status_status_csv_size_mb":"0","cached_job_status_total_entries":"20","completed_count":"20","completed_size_mb":"2","count_summary":"1","disk_usage_MB":"2","eai:acl":null,"incomple_count":"0","incomple_size_mb":"0","invalid_count":"1","remote_count":"0","remote_mb":"0","rsa_count":"0","rsa_scheduled_count":"0","rsa_scheduled_size_mb":"0","rsa_size_mb":"0","scheduled_count":"13","scheduled_size_mb":"1","scheduled_subsearch_count":"0","scheduled_subsearch_size_mb":"0","ss_count":"7","status_cache_info_csv_size_mb":"0","status_cache_status_csv_size_mb":"0","status_cache_total_entries":"20","temp_dispatch_count":"0","temp_dispatch_size_mb":"0","top_apps":{"0":{"splunk_instrumentation":"6"},"1":{"search":"1"}},"top_named_searches":null,"top_users":{"0":{"splunk-system-user":"6"},"1":{"internal_observability":"1"}},"total_count":"7"}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}`))
 }
 
+func mockIndexerClusterMangerStatus(w http.ResponseWriter, _ *http.Request) {
+	status := http.StatusOK
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_, _ = w.Write([]byte(`{"links":{},"origin":"https://somehost:8089/services/cluster/manager/status","updated":"2025-03-27T22:21:40+00:00","generator":{"build":"b10ab43f821f","version":"9.3.2408.109"},"entry":[{"name":"master","id":"https://somehost:8089/services/cluster/manager/status/master","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/cluster/manager/status/master","list":"/services/cluster/manager/status/master"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["admin","index-manager","internal_ops_admin","splunk-system-role"],"write":["admin","index-manager","internal_ops_admin","splunk-system-role"]},"removable":false,"sharing":"system"},"content":{"available_sites":"[site11, site12, site13]","decommission_force_timeout":"0","eai:acl":null,"ha_mode":"Disabled","maintenance_mode":false,"messages":"","multisite":true,"peers":{},"restart_inactivity_timeout":"0","restart_progress":{"done":[],"failed":[],"in_progress":[],"skipped":[],"to_be_restarted":[]},"rolling_restart_flag":false,"rolling_restart_or_upgrade":false,"rolling_restart_type":"None","searchable_rolling":false,"service_ready_flag":true}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}
+`))
+}
+
 // mock server create
 func createMockServer() *httptest.Server {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -66,6 +74,8 @@ func createMockServer() *httptest.Server {
 			mockIntrospectionQueues(w, r)
 		case "/services/server/status/dispatch-artifacts?output_mode=json&count=-1":
 			mockDispatchArtifacts(w, r)
+		case "/services/cluster/manager/status?output_mode=json":
+			mockIndexerClusterMangerStatus(w, r)
 		default:
 			http.NotFoundHandler().ServeHTTP(w, r)
 		}
@@ -90,6 +100,7 @@ func TestScraper(t *testing.T) {
 	metricsettings.Metrics.SplunkDataIndexesExtendedBucketWarmCount.Enabled = true
 	metricsettings.Metrics.SplunkServerIntrospectionQueuesCurrent.Enabled = true
 	metricsettings.Metrics.SplunkServerIntrospectionQueuesCurrentBytes.Enabled = true
+	metricsettings.Metrics.SplunkIndexerRollingrestartStatus.Enabled = true
 
 	cfg := &Config{
 		IdxEndpoint: confighttp.ClientConfig{

receiver/splunkenterprisereceiver/scraper.go

@@ -159,7 +159,7 @@ type dispatchArtifactContent struct {
 	CacheTotalEntries  string `json:"cached_job_status_total_entries"`
 }
 
-// '/services/server/health/splunkd/details'
+// '/services/server/health/splunkd/details
 type healthArtifacts struct {
 	Entries []healthArtifactEntry `json:"entry"`
 }
@@ -175,7 +175,7 @@ type healthDetails struct {
 
 // '/services/cluster/manager/status' 
 type indexersClusterManagerStatus struct {
-	Entry idxClusterManagerStatusEntry
+	Entries []idxClusterManagerStatusEntry `json:"entry"`
 }
 type idxClusterManagerStatusEntry struct {
 	Name    string      `json:"name"`

receiver/splunkenterprisereceiver/scraper_test.go

@@ -95,6 +95,21 @@ resourceMetrics:
                   timeUnixNano: "2000000"
             name: splunk.data.indexes.extended.total.size
             unit: By
+          - description: The status of a rolling restart.
+            gauge:
+              dataPoints:
+                - asInt: "0"
+                  attributes:
+                    - key: splunk.rollingorrestart
+                      value:
+                        boolValue: false
+                    - key: splunk.searchable.restart
+                      value:
+                        boolValue: false
+                  startTimeUnixNano: "1000000"
+                  timeUnixNano: "2000000"
+            name: splunk.indexer.rollingrestart.status
+            unit: '{status}'
           - description: Gauge tracking average bytes per second throughput of indexer. *Note:** Must be pointed at specific indexer `endpoint` and gathers metrics from only that indexer.
             gauge:
               dataPoints: