[chore] prom translation rw2 add support for metadata (#40469)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description
Adds support for translation of OTEL metadata to prometheus metadata in
the RW2 translation path.

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes

Partially implements #33661 (when merging PR please don't close the
tracing issue)

<!--Describe what testing was performed and which tests were added.-->
#### Testing

* [x] Updated existing unit tests
* [x] Added new ones
* [x] e2e run with prometheus

When debugging prometheus locally I can see it receives the metadata,
but I can't see it in the frontend, because at least in prometheus
metadata is not stored when coming via remote write for now
([source](https://github.com/jmichalek132/prometheus/blob/de23a9667cc670b26d8afdba8e3fa48a00e74ec0/storage/remote/write.go#L332)).

![image](https://github.com/user-attachments/assets/09781972-425f-45b1-9595-32d267da49cd)
 

<!--Please delete paragraphs that you did not use before submitting.-->

Author: jmichalek132

pkg/translator/prometheusremotewrite/metrics_to_prw_test.go

@@ -119,6 +119,8 @@ func createExportRequest(resourceAttributeCount int, histogramCount int, nonHist
 	for i := 1; i <= nonHistogramCount; i++ {
 		m := metrics.AppendEmpty()
 		m.SetEmptySum()
+		m.SetDescription("test sum description")
+		m.SetUnit("By")
 		m.SetName(fmt.Sprintf("sum-%v", i))
 		m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
 		point := m.Sum().DataPoints().AppendEmpty()
@@ -131,6 +133,8 @@ func createExportRequest(resourceAttributeCount int, histogramCount int, nonHist
 	for i := 1; i <= nonHistogramCount; i++ {
 		m := metrics.AppendEmpty()
 		m.SetEmptyGauge()
+		m.SetDescription("test gauge description")
+		m.SetUnit("By")
 		m.SetName(fmt.Sprintf("gauge-%v", i))
 		point := m.Gauge().DataPoints().AppendEmpty()
 		point.SetTimestamp(timestamp)

pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go

@@ -38,6 +38,12 @@ type prometheusConverterV2 struct {
 	symbolTable writev2.SymbolsTable
 }
 
+type metadata struct {
+	Type writev2.Metadata_MetricType
+	Help string
+	Unit string
+}
+
 func newPrometheusConverterV2() *prometheusConverterV2 {
 	return &prometheusConverterV2{
 		unique:      map[uint64]*writev2.TimeSeries{},
@@ -69,6 +75,11 @@ func (c *prometheusConverterV2) fromMetrics(md pmetric.Metrics, settings Setting
 				}
 
 				promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
+				m := metadata{
+					Type: otelMetricTypeToPromMetricTypeV2(metric),
+					Help: metric.Description(),
+					Unit: prometheustranslator.BuildCompliantPrometheusUnit(metric.Unit()),
+				}
 
 				// handle individual metrics based on type
 				//exhaustive:enforce
@@ -78,16 +89,16 @@ func (c *prometheusConverterV2) fromMetrics(md pmetric.Metrics, settings Setting
 					if dataPoints.Len() == 0 {
 						break
 					}
-					c.addGaugeNumberDataPoints(dataPoints, resource, settings, promName)
+					c.addGaugeNumberDataPoints(dataPoints, resource, settings, promName, m)
 				case pmetric.MetricTypeSum:
 					dataPoints := metric.Sum().DataPoints()
 					if dataPoints.Len() == 0 {
 						break
 					}
 					if !metric.Sum().IsMonotonic() {
-						c.addGaugeNumberDataPoints(dataPoints, resource, settings, promName)
+						c.addGaugeNumberDataPoints(dataPoints, resource, settings, promName, m)
 					} else {
-						c.addSumNumberDataPoints(dataPoints, resource, metric, settings, promName)
+						c.addSumNumberDataPoints(dataPoints, resource, metric, settings, promName, m)
 					}
 				case pmetric.MetricTypeHistogram:
 					// TODO implement
@@ -116,7 +127,8 @@ func (c *prometheusConverterV2) timeSeries() []writev2.TimeSeries {
 	return allTS
 }
 
-func (c *prometheusConverterV2) addSample(sample *writev2.Sample, lbls []prompb.Label) {
+func (c *prometheusConverterV2) addSample(sample *writev2.Sample, lbls []prompb.Label, metadata metadata) {
+	// TODO consider how to accommodate metadata in the symbol table when allocating the buffer, given not all metrics might have metadata.
 	buf := make([]uint32, 0, len(lbls)*2)
 
 	// TODO: Read the PRW spec to see if labels need to be sorted. If it is, then we need to sort in export code. If not, we can sort in the test. (@dashpole have more context on this)
@@ -134,6 +146,11 @@ func (c *prometheusConverterV2) addSample(sample *writev2.Sample, lbls []prompb.
 	ts := writev2.TimeSeries{
 		LabelsRefs: buf,
 		Samples:    []writev2.Sample{*sample},
+		Metadata: writev2.Metadata{
+			Type:    metadata.Type,
+			HelpRef: c.symbolTable.Symbolize(metadata.Help),
+			UnitRef: c.symbolTable.Symbolize(metadata.Unit),
+		},
 	}
 	c.unique[timeSeriesSignature(lbls)] = &ts
 }

pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go

@@ -27,19 +27,29 @@ func TestFromMetricsV2(t *testing.T) {
 	payload := createExportRequest(5, 0, 1, 3, 0, pcommon.Timestamp(ts))
 	want := []*writev2.TimeSeries{
 		{
-			LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8},
+			LabelsRefs: []uint32{1, 11, 3, 4, 5, 6, 7, 8},
 			Samples: []writev2.Sample{
 				{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 1.23},
 			},
+			Metadata: writev2.Metadata{
+				Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+				HelpRef: 12,
+				UnitRef: 10,
+			},
 		},
 		{
-			LabelsRefs: []uint32{1, 9, 3, 4, 5, 6, 7, 8},
+			LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8},
 			Samples: []writev2.Sample{
 				{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 1.23},
 			},
+			Metadata: writev2.Metadata{
+				Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+				HelpRef: 9,
+				UnitRef: 10,
+			},
 		},
 	}
-	wantedSymbols := []string{"", "series_name_2", "value-2", "series_name_3", "value-3", "__name__", "gauge_1", "series_name_1", "value-1", "sum_1"}
+	wantedSymbols := []string{"", "series_name_2", "value-2", "series_name_3", "value-3", "__name__", "gauge_1", "series_name_1", "value-1", "sum_1", "test gauge description", "test sum description", "bytes"}
 	tsMap, symbolsTable, err := FromMetricsV2(payload.Metrics(), settings)
 	require.NoError(t, err)
 	require.ElementsMatch(t, want, slices.Collect(maps.Values(tsMap)))

pkg/translator/prometheusremotewrite/number_data_points_v2.go

@@ -15,7 +15,7 @@ import (
 )
 
 func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.NumberDataPointSlice,
-	resource pcommon.Resource, settings Settings, name string,
+	resource pcommon.Resource, settings Settings, name string, metadata metadata,
 ) {
 	for x := 0; x < dataPoints.Len(); x++ {
 		pt := dataPoints.At(x)
@@ -43,12 +43,12 @@ func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.Numb
 		if pt.Flags().NoRecordedValue() {
 			sample.Value = math.Float64frombits(value.StaleNaN)
 		}
-		c.addSample(sample, labels)
+		c.addSample(sample, labels, metadata)
 	}
 }
 
 func (c *prometheusConverterV2) addSumNumberDataPoints(dataPoints pmetric.NumberDataPointSlice,
-	resource pcommon.Resource, _ pmetric.Metric, settings Settings, name string,
+	resource pcommon.Resource, _ pmetric.Metric, settings Settings, name string, metadata metadata,
 ) {
 	for x := 0; x < dataPoints.Len(); x++ {
 		pt := dataPoints.At(x)
@@ -76,7 +76,7 @@ func (c *prometheusConverterV2) addSumNumberDataPoints(dataPoints pmetric.Number
 			sample.Value = math.Float64frombits(value.StaleNaN)
 		}
 		// TODO: properly add exemplars to the TimeSeries
-		c.addSample(sample, lbls)
+		c.addSample(sample, lbls, metadata)
 	}
 }
 

pkg/translator/prometheusremotewrite/number_data_points_v2_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
+
+	prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
 )
 
 func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) {
@@ -47,6 +49,11 @@ func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) {
 						Samples: []writev2.Sample{
 							{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 1},
 						},
+						Metadata: writev2.Metadata{
+							Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+							HelpRef: 3,
+							UnitRef: 4,
+						},
 					},
 				}
 			},
@@ -73,6 +80,11 @@ func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) {
 						Samples: []writev2.Sample{
 							{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 1.5},
 						},
+						Metadata: writev2.Metadata{
+							Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+							HelpRef: 3,
+							UnitRef: 4,
+						},
 					},
 				}
 			},
@@ -99,6 +111,11 @@ func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) {
 						Samples: []writev2.Sample{
 							{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: math.Float64frombits(value.StaleNaN)},
 						},
+						Metadata: writev2.Metadata{
+							Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+							HelpRef: 3,
+							UnitRef: 4,
+						},
 					},
 				}
 			},
@@ -115,7 +132,12 @@ func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) {
 				SendMetadata:      false,
 			}
 			converter := newPrometheusConverterV2()
-			converter.addGaugeNumberDataPoints(metric.Gauge().DataPoints(), pcommon.NewResource(), settings, metric.Name())
+			m := metadata{
+				Type: otelMetricTypeToPromMetricTypeV2(metric),
+				Help: metric.Description(),
+				Unit: prometheustranslator.BuildCompliantPrometheusUnit(metric.Unit()),
+			}
+			converter.addGaugeNumberDataPoints(metric.Gauge().DataPoints(), pcommon.NewResource(), settings, metric.Name(), m)
 			w := tt.want()
 
 			diff := cmp.Diff(w, converter.unique, cmpopts.EquateNaNs())
@@ -151,6 +173,11 @@ func TestPrometheusConverterV2_addGaugeNumberDataPointsDuplicate(t *testing.T) {
 				Samples: []writev2.Sample{
 					{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 2},
 				},
+				Metadata: writev2.Metadata{
+					Type:    writev2.Metadata_METRIC_TYPE_GAUGE,
+					HelpRef: 3,
+					UnitRef: 4,
+				},
 			},
 		}
 	}
@@ -163,8 +190,19 @@ func TestPrometheusConverterV2_addGaugeNumberDataPointsDuplicate(t *testing.T) {
 	}
 
 	converter := newPrometheusConverterV2()
-	converter.addGaugeNumberDataPoints(metric1.Gauge().DataPoints(), pcommon.NewResource(), settings, metric1.Name())
-	converter.addGaugeNumberDataPoints(metric2.Gauge().DataPoints(), pcommon.NewResource(), settings, metric2.Name())
+	m1 := metadata{
+		Type: otelMetricTypeToPromMetricTypeV2(metric1),
+		Help: metric1.Description(),
+		Unit: prometheustranslator.BuildCompliantPrometheusUnit(metric1.Unit()),
+	}
+	converter.addGaugeNumberDataPoints(metric1.Gauge().DataPoints(), pcommon.NewResource(), settings, metric1.Name(), m1)
+
+	m2 := metadata{
+		Type: otelMetricTypeToPromMetricTypeV2(metric2),
+		Help: metric2.Description(),
+		Unit: prometheustranslator.BuildCompliantPrometheusUnit(metric2.Unit()),
+	}
+	converter.addGaugeNumberDataPoints(metric2.Gauge().DataPoints(), pcommon.NewResource(), settings, metric2.Name(), m2)
 
 	assert.Equal(t, want(), converter.unique)
 }

pkg/translator/prometheusremotewrite/otlp_to_openmetrics_metadata_v2.go

@@ -0,0 +1,43 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package prometheusremotewrite // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite"
+
+import (
+	"github.com/prometheus/common/model"
+	writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
+	"go.opentelemetry.io/collector/pdata/pmetric"
+
+	prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
+)
+
+func otelMetricTypeToPromMetricTypeV2(otelMetric pmetric.Metric) writev2.Metadata_MetricType {
+	// metric metadata can be used to support Prometheus types that don't exist
+	// in OpenTelemetry.
+	typeFromMetadata, hasTypeFromMetadata := otelMetric.Metadata().Get(prometheustranslator.MetricMetadataTypeKey)
+	switch otelMetric.Type() {
+	case pmetric.MetricTypeGauge:
+		if hasTypeFromMetadata && typeFromMetadata.Str() == string(model.MetricTypeUnknown) {
+			return writev2.Metadata_METRIC_TYPE_UNSPECIFIED
+		}
+		return writev2.Metadata_METRIC_TYPE_GAUGE
+	case pmetric.MetricTypeSum:
+		if otelMetric.Sum().IsMonotonic() {
+			return writev2.Metadata_METRIC_TYPE_COUNTER
+		}
+		if hasTypeFromMetadata && typeFromMetadata.Str() == string(model.MetricTypeInfo) {
+			return writev2.Metadata_METRIC_TYPE_INFO
+		}
+		if hasTypeFromMetadata && typeFromMetadata.Str() == string(model.MetricTypeStateset) {
+			return writev2.Metadata_METRIC_TYPE_STATESET
+		}
+		return writev2.Metadata_METRIC_TYPE_GAUGE
+	case pmetric.MetricTypeHistogram:
+		return writev2.Metadata_METRIC_TYPE_HISTOGRAM
+	case pmetric.MetricTypeSummary:
+		return writev2.Metadata_METRIC_TYPE_SUMMARY
+	case pmetric.MetricTypeExponentialHistogram:
+		return writev2.Metadata_METRIC_TYPE_HISTOGRAM
+	}
+	return writev2.Metadata_METRIC_TYPE_UNSPECIFIED
+}