[receiver/windowsperfcounters] fix: include instance index for multiple matches. (#32321)

**Description:** This PR adds the counter's instance index for counters
that include multiple instances with the same name. A simple example of
this case is for monitoring process ID when there are multiple instances
of the same process running, but this can technically happen for any
counter that uses instance names.

This can potentially increase cardinality so I am open to adding a
configuration to the receiver to toggle this behavior on and leave it
disabled as a default.

**Link to tracking Issue:** #32319

**Testing:** 
- See issue for details. Short version: two notepad processes running,
scraping `Process\ID Process` to `debug` and `prometheusremotewrite` to
validate that `instance` label now matches what is seen in `perfmon.exe`
- Added unit test to watcher.go to validate instance name indexing

**Documentation:** Added enhancement CHANGELOG

Fixes #32319

---------

Co-authored-by: Paulo Janotti <[email protected]>

Author: alxbl

.chloggen/windowsperfcounters-instance-indexing.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: windowsperfcountersreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "`windowsperfcountersreceiver` now appends an index number to additional instance names that share a name. An example of this is when scraping `\Process(*)` counters with multiple running instances of the same executable."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [32319]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  **NOTES**
+  - This change can expose cardinality issues where the counters were previously collapsed under the non-indexed instance name.
+  - The change mimics Windows Performance Monitor behavior: The first instance name remains unchanged, additional instances are suffixed with `#<N>` where `N=1` and is increased for each duplicate.
+    - e.g. Given 3 powershell instances, this will return `powershell`, `powershell#1` and `powershell#2`.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

pkg/winperfcounters/watcher.go

@@ -142,7 +142,7 @@ func (pc *perfCounter) ScrapeData() ([]CounterValue, error) {
 		return nil, fmt.Errorf("failed to format data for performance counter '%s': %w", pc.path, err)
 	}
 
-	vals = removeTotalIfMultipleValues(vals)
+	vals = cleanupScrapedValues(vals)
 	return vals, nil
 }
 
@@ -151,24 +151,42 @@ func ExpandWildCardPath(counterPath string) ([]string, error) {
 	return win_perf_counters.ExpandWildCardPath(counterPath)
 }
 
-func removeTotalIfMultipleValues(vals []CounterValue) []CounterValue {
+// cleanupScrapedValues handles instance name collisions and standardizes names.
+// It cleans up the list in-place to avoid unnecessary copies.
+func cleanupScrapedValues(vals []CounterValue) []CounterValue {
 	if len(vals) == 0 {
 		return vals
 	}
 
-	if len(vals) == 1 {
-		// if there is only one item & the instance name is "_Total", clear the instance name
-		if vals[0].InstanceName == totalInstanceName {
-			vals[0].InstanceName = ""
-		}
+	// If there is only one "_Total" instance, clear the instance name.
+	if len(vals) == 1 && vals[0].InstanceName == totalInstanceName {
+		vals[0].InstanceName = ""
 		return vals
 	}
 
-	// if there is more than one item, remove an item that has the instance name "_Total"
-	for i, val := range vals {
-		if val.InstanceName == totalInstanceName {
-			return removeItemAt(vals, i)
+	occurrences := map[string]int{}
+	totalIndex := -1
+
+	for i := range vals {
+		instanceName := vals[i].InstanceName
+
+		if instanceName == totalInstanceName {
+			// Remember if a "_Total" instance was present.
+			totalIndex = i
 		}
+
+		if n, ok := occurrences[instanceName]; ok {
+			// Append indices to duplicate instance names.
+			occurrences[instanceName]++
+			vals[i].InstanceName = fmt.Sprintf("%s#%d", instanceName, n)
+		} else {
+			occurrences[instanceName] = 1
+		}
+	}
+
+	// Remove the "_Total" instance, as it can be computed with a sum aggregation.
+	if totalIndex >= 0 {
+		return removeItemAt(vals, totalIndex)
 	}
 
 	return vals

pkg/winperfcounters/watcher_test.go

@@ -176,3 +176,142 @@ func TestPerfCounter_ScrapeData(t *testing.T) {
 		})
 	}
 }
+
+func Test_InstanceNameIndexing(t *testing.T) {
+	type testCase struct {
+		name     string
+		vals     []CounterValue
+		expected []CounterValue
+	}
+
+	testCases := []testCase{
+		{
+			name: "Multiple distinct instances",
+			vals: []CounterValue{
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "C",
+					Value:        1.0,
+				},
+			},
+			expected: []CounterValue{
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "C",
+					Value:        1.0,
+				},
+			},
+		},
+		{
+			name: "Single repeated instance name",
+			vals: []CounterValue{
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+			},
+			expected: []CounterValue{
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "A#1",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "A#2",
+					Value:        1.0,
+				},
+			},
+		},
+		{
+			name: "Multiple repeated instance name",
+			vals: []CounterValue{
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "C",
+					Value:        1.0,
+				},
+			},
+			expected: []CounterValue{
+				{
+					InstanceName: "A",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "A#1",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B#1",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "B#2",
+					Value:        1.0,
+				},
+				{
+					InstanceName: "C",
+					Value:        1.0,
+				},
+			},
+		},
+	}
+
+	for _, test := range testCases {
+		actual := cleanupScrapedValues(test.vals)
+		t.Run(test.name, func(t *testing.T) {
+			compareCounterValues(t, test.expected, actual)
+		})
+	}
+}
+
+func compareCounterValues(t *testing.T, expected []CounterValue, actual []CounterValue) {
+	assert.EqualValues(t, expected, actual)
+}