[receiver/splunkenterprisereceiver] Add splunk.indexer.rollingrestart.status metric

receiver/splunkenterprisereceiver/documentation.md

@@ -405,6 +405,21 @@ The status ('red', 'yellow', or 'green') of the Splunk server. Health of 'red' p
 | splunk.feature | The Feature name from the Splunk Health Introspection Endpoint | Any Str |
 | splunk.feature.health | The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint | Any Str |
 
+### splunk.indexer.rollingrestart.status
+
+The status of a rolling restart.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| {status} | Gauge | Int |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| splunk.searchable.restart | String that indicates if a searchable rolling restart/upgrade in progress.t | Any Str |
+| splunk.rollingorrestart | String that indicates if there is a rolling restart or rolling upgrade in progress. | Any Str |
+
 ### splunk.indexer.throughput
 
 Gauge tracking average bytes per second throughput of indexer. *Note:** Must be pointed at specific indexer `endpoint` and gathers metrics from only that indexer.

receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml

@@ -29,6 +29,8 @@ all_set:
       enabled: true
     splunk.indexer.raw.write.time:
       enabled: true
+    splunk.indexer.rollingrestart.status:
+      enabled: true
     splunk.indexer.throughput:
       enabled: true
     splunk.indexes.avg.size:
@@ -113,6 +115,8 @@ none_set:
       enabled: false
     splunk.indexer.raw.write.time:
       enabled: false
+    splunk.indexer.rollingrestart.status:
+      enabled: false
     splunk.indexer.throughput:
       enabled: false
     splunk.indexes.avg.size:

receiver/splunkenterprisereceiver/metadata.yaml

@@ -45,6 +45,12 @@ attributes:
   splunk.feature.health:
     description: The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint
     type: string
+  splunk.searchable.restart:
+    description: String that indicates if a searchable rolling restart/upgrade in progress.t
+    type: string
+  splunk.rollingorrestart:
+    description: String that indicates if there is a rolling restart or rolling upgrade in progress.
+    type: string
 
 metrics:
   splunk.license.index.usage:
@@ -359,6 +365,14 @@ metrics:
     gauge:
       value_type: int
     attributes: [splunk.feature, splunk.feature.health]
+  # `services/cluster/manager/status`
+  splunk.indexer.rollingrestart.status:
+    enabled: false
+    description: The status of a rolling restart.
+    unit: "{status}"
+    gauge:
+      value_type: int
+    attributes: [splunk.searchable.restart, splunk.rollingorrestart]
 
 tests:
   config:

receiver/splunkenterprisereceiver/scraper.go

@@ -1786,3 +1786,57 @@ func (s *splunkScraper) traverseHealthDetailFeatures(details healthDetails, now
 		s.traverseHealthDetailFeatures(feature, now)
 	}
 }
+
+// Scrape Indexer Cluster Manger Status Endpoint
+func (s *splunkScraper) scrapeIndexerClusterManagerStatus(ctx context.Context, now pcommon.Timestamp, errs chan error) {
+	if !s.conf.MetricsBuilderConfig.Metrics.SplunkIndexerRollingRestartStatus.Enabled {
+		return
+	}
+
+	ctx = context.WithValue(ctx, endpointType("type"), typeCm)
+
+	ept := apiDict[`SplunkIndexerClusterManagerStatus`]
+	var icms indexersClusterManagerStatus
+
+	req, err := s.splunkClient.createAPIRequest(ctx, ept)
+	if err != nil {
+		errs <- err
+		return
+	}
+
+	res, err := s.splunkClient.makeRequest(req)
+	if err != nil {
+		errs <- err
+		return
+	}
+	defer res.Body.Close()
+
+	if err := json.NewDecoder(res.Body).Decode(&icms); err != nil {
+		errs <- err
+		return
+	}
+
+	restartOrUpgrade := "false"
+	if icms.Entry.Content.RollingRestartOrUpgrade {
+		restartOrUpgrade = "true"
+	}
+
+	searchable := "false"
+	if icms.Entry.Content.SearchableRolling {
+		searchable = "true"
+	}
+
+	restarting := icms.Entry.Content.RollingRestartOrUpgrade || icms.Entry.Content.SearchableRolling || icms.Entry.Content.RollingRestartFlag
+
+	for _, mtsRestartOrUpgrade := range []string{"false", "true"} {
+		for _, mtsSearchable := range []string{"false", "true"} {
+			if restartOrUpgrade == mtsRestartOrUpgrade && searchable == mtsSearchable && restarting {
+				// If we're restarting, put a 1 in the gauge that matches the current state
+				s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 1, mtsRestartOrUpgrade, mtsSearchable)
+			} else {
+				// Put a zero in all the other gauges
+				s.mb.RecordSplunkIndexerRollingrestartStatusDataPoint(now, 0, mtsRestartOrUpgrade, mtsSearchable)
+			}
+		}
+	}
+}

receiver/splunkenterprisereceiver/search_result.go

@@ -26,6 +26,7 @@ var apiDict = map[string]string{
 	`SplunkKVStoreStatus`:       `/services/kvstore/status?output_mode=json`,
 	`SplunkDispatchArtifacts`:   `/services/server/status/dispatch-artifacts?output_mode=json&count=-1`,
 	`SplunkHealth`:              `/services/server/health/splunkd/details?output_mode=json`,
+	`SplunkIndexerClusterManagerStatus`: `/services/cluster/manager/status?output_mode=json`,
 }
 
 type searchResponse struct {
@@ -158,7 +159,7 @@ type dispatchArtifactContent struct {
 	CacheTotalEntries  string `json:"cached_job_status_total_entries"`
 }
 
-// '/services/server/health/splunkd/details
+// '/services/server/health/splunkd/details'
 type healthArtifacts struct {
 	Entries []healthArtifactEntry `json:"entry"`
 }
@@ -171,3 +172,20 @@ type healthDetails struct {
 	Health   string                   `json:"health"`
 	Features map[string]healthDetails `json:"features,omitempty"`
 }
+
+// '/services/cluster/manager/status' 
+type indexersClusterManagerStatus struct {
+	Entry idxClusterManagerStatusEntry
+}
+type idxClusterManagerStatusEntry struct {
+	Name    string      `json:"name"`
+	Content idxClusterManagerStatusContent `json:"content"`
+}
+
+type idxClusterManagerStatusContent struct {
+	RollingRestartFlag      bool   `json:"rolling_restart_flag,omitempty"`
+	RollingRestartOrUpgrade bool   `json:"rolling_restart_or_upgrade,omitempty"`
+	RollingRestartType      string `json:"rolling_restart_type,omitempty"`
+	SearchableRolling       bool   `json:"searchable_rolling,omitempty"`
+	ServiceReadyFlag        bool   `json:"service_ready_flag,omitempty"`
+}