Splunkenterprisereceiver add health metric (#36695)

.chloggen/changes.yaml

@@ -0,0 +1,22 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: 'splunkenterprisereceiver'
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Added a new `splunk.health` metric."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36695]
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

receiver/splunkenterprisereceiver/documentation.md

@@ -41,6 +41,21 @@ Gauge tracking the number of buckets and their searchable status. *Note:** Searc
 | splunk.host | The name of the splunk host | Any Str |
 | splunk.indexer.searchable | The searchability status reported for a specific object | Any Str |
 
+### splunk.health
+
+The status (color) of the Splunk server. Health of 'red' produces a 0 while all other colors produce a 1.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| {status} | Gauge | Int |
+
+#### Attributes
+
+| Name | Description | Values |
+| ---- | ----------- | ------ |
+| splunk.feature | The Feature name from the Splunk Health Introspection Endpoint | Any Str |
+| splunk.feature.health | The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint | Any Str |
+
 ### splunk.indexer.avg.rate
 
 Gauge tracking the average rate of indexed data. **Note:** Search is best run against a Cluster Manager.

receiver/splunkenterprisereceiver/internal/metadata/testdata/config.yaml

@@ -19,6 +19,8 @@ all_set:
       enabled: true
     splunk.data.indexes.extended.total.size:
       enabled: true
+    splunk.health:
+      enabled: true
     splunk.indexer.avg.rate:
       enabled: true
     splunk.indexer.cpu.time:
@@ -101,6 +103,8 @@ none_set:
       enabled: false
     splunk.data.indexes.extended.total.size:
       enabled: false
+    splunk.health:
+      enabled: false
     splunk.indexer.avg.rate:
       enabled: false
     splunk.indexer.cpu.time:

receiver/splunkenterprisereceiver/metadata.yaml

@@ -39,6 +39,12 @@ attributes:
   splunk.searchartifacts.cache.type:
     description: The search artifacts cache type
     type: string
+  splunk.feature:
+    description: The Feature name from the Splunk Health Introspection Endpoint
+    type: string
+  splunk.feature.health:
+    description: The Health (in color form) of a Splunk Feature from the Splunk Health Introspection Endpoint
+    type: string
 
 metrics:
   splunk.license.index.usage:
@@ -345,6 +351,14 @@ metrics:
       aggregation_temporality: cumulative
       value_type: int
     attributes: [splunk.host]
+  #`services/server/health/splunkd/details`
+  splunk.health:
+    enabled: True
+    description: The status (color) of the Splunk server. Health of 'red' produces a 0 while all other colors produce a 1.
+    unit: "{status}"
+    gauge:
+      value_type: int
+    attributes: [splunk.feature, splunk.feature.health]
 
 tests:
   config:

receiver/splunkenterprisereceiver/scraper.go

@@ -101,6 +101,7 @@ func (s *splunkScraper) scrape(ctx context.Context) (pmetric.Metrics, error) {
 		s.scrapeIndexerAvgRate,
 		s.scrapeKVStoreStatus,
 		s.scrapeSearchArtifacts,
+		s.scrapeHealth,
 	}
 	errChan := make(chan error, len(metricScrapes))
 
@@ -1075,12 +1076,12 @@ func unmarshallSearchReq(res *http.Response, sr *searchResponse) error {
 
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
-		return fmt.Errorf("Failed to read response: %w", err)
+		return fmt.Errorf("failed to read response: %w", err)
 	}
 
 	err = xml.Unmarshal(body, &sr)
 	if err != nil {
-		return fmt.Errorf("Failed to unmarshall response: %w", err)
+		return fmt.Errorf("failed to unmarshall response: %w", err)
 	}
 
 	return nil
@@ -1733,3 +1734,55 @@ func (s *splunkScraper) scrapeSearchArtifacts(ctx context.Context, now pcommon.T
 		}
 	}
 }
+
+// Scrape Health Introspection Endpoint
+func (s *splunkScraper) scrapeHealth(ctx context.Context, now pcommon.Timestamp, errs chan error) {
+	if !s.conf.MetricsBuilderConfig.Metrics.SplunkHealth.Enabled {
+		return
+	}
+
+	ctx = context.WithValue(ctx, endpointType("type"), typeCm)
+
+	ept := apiDict[`SplunkHealth`]
+	var ha HealthArtifacts
+
+	req, err := s.splunkClient.createAPIRequest(ctx, ept)
+	if err != nil {
+		errs <- err
+		return
+	}
+
+	res, err := s.splunkClient.makeRequest(req)
+	if err != nil {
+		errs <- err
+		return
+	}
+	defer res.Body.Close()
+
+	if err := json.NewDecoder(res.Body).Decode(&ha); err != nil {
+		errs <- err
+		return
+	}
+
+	s.settings.Logger.Debug(fmt.Sprintf("Features: %s", ha.Entries))
+	for _, details := range ha.Entries {
+		s.traverseHealthDetailFeatures(details.Content, now)
+	}
+}
+
+func (s *splunkScraper) traverseHealthDetailFeatures(details HealthDetails, now pcommon.Timestamp) {
+	if details.Features == nil {
+		return
+	}
+
+	for k, feature := range details.Features {
+		if feature.Health != "red" {
+			s.settings.Logger.Debug(feature.Health)
+			s.mb.RecordSplunkHealthDataPoint(now, 1, k, feature.Health)
+		} else {
+			s.settings.Logger.Debug(feature.Health)
+			s.mb.RecordSplunkHealthDataPoint(now, 0, k, feature.Health)
+		}
+		s.traverseHealthDetailFeatures(feature, now)
+	}
+}

receiver/splunkenterprisereceiver/search_result.go

@@ -25,6 +25,7 @@ var apiDict = map[string]string{
 	`SplunkIntrospectionQueues`: `/services/server/introspection/queues?output_mode=json&count=-1`,
 	`SplunkKVStoreStatus`:       `/services/kvstore/status?output_mode=json`,
 	`SplunkDispatchArtifacts`:   `/services/server/status/dispatch-artifacts?output_mode=json&count=-1`,
+	`SplunkHealth`:              `/services/server/health/splunkd/details?output_mode=json`,
 }
 
 type searchResponse struct {
@@ -156,3 +157,17 @@ type DispatchArtifactContent struct {
 	StatusCacheSize    string `json:"cached_job_status_status_csv_size_mb"`
 	CacheTotalEntries  string `json:"cached_job_status_total_entries"`
 }
+
+// '/services/server/health/splunkd/details
+type HealthArtifacts struct {
+	Entries []HealthArtifactEntry `json:"entry"`
+}
+
+type HealthArtifactEntry struct {
+	Content HealthDetails `json:"content"`
+}
+
+type HealthDetails struct {
+	Health   string                   `json:"health"`
+	Features map[string]HealthDetails `json:"features,omitempty"`
+}