[exporter][prometheusremotewrite] Additional metrics for the exporter

Often, we have to do anylsis of the current state of the exporter which includes the current level of concurrency and throughput.

For the concurrency, we need to rely on revising each individual configuration which is time consuming, here I'm proposing to have it as a metric that we can keep a record of and see it through observability.

For the throughput, while tecnically `otelcol_processor_batch_batch_send_size_count` exists, this is not divisible by type (e.g. metrics vs traces) and is not specific to exporter. Whilst I'm also interested in splitting `batch_send_size_count` by type (because that also includes OTLP batches and this metric is only for RW), I think this is part of a separate conversation and it doesn't hurt to have the single counter that's specific for remote write.

There's one more thing here (and please advise if you'd rather see a different PR for it) - I'm also adding the `url` as attribute to _ALL_ the metrics  as I think this is key to identify where the exporter is pointing to.

This is tecnically my first contribution to the repo so I'd appreciate any guidiance on what else can I need to do. Telemetry tests appear to be auto generated so I didn't add any to this PR but please let me know otherwise.

Signed-off-by: gotjosh <[email protected]>

Author: gotjosh

exporter/prometheusremotewriteexporter/documentation.md

@@ -6,6 +6,14 @@
 
 The following telemetry is emitted by this component.
 
+### otelcol_exporter_prometheusremotewrite_consumers
+
+Number of configured workers to use to fan out the outgoing requests
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| 1 | Gauge | Int |
+
 ### otelcol_exporter_prometheusremotewrite_failed_translations
 
 Number of translation operations that failed to translate metrics from Otel to Prometheus
@@ -14,6 +22,14 @@ Number of translation operations that failed to translate metrics from Otel to P
 | ---- | ----------- | ---------- | --------- |
 | 1 | Sum | Int | true |
 
+### otelcol_exporter_prometheusremotewrite_sent_batch_count
+
+Number of remote write request batches sent to the remote write endpoint
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
 ### otelcol_exporter_prometheusremotewrite_translated_time_series
 
 Number of Prometheus time series that were translated from OTel metrics

exporter/prometheusremotewriteexporter/exporter.go

@@ -38,13 +38,23 @@ import (
 type prwTelemetry interface {
 	recordTranslationFailure(ctx context.Context)
 	recordTranslatedTimeSeries(ctx context.Context, numTS int)
+	recordRemoteWriteSentBatch(ctx context.Context)
+	setNumberConsumer(ctx context.Context, n int64)
 }
 
 type prwTelemetryOtel struct {
 	telemetryBuilder *metadata.TelemetryBuilder
 	otelAttrs        []attribute.KeyValue
 }
 
+func (p *prwTelemetryOtel) setNumberConsumer(ctx context.Context, n int64) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteConsumers.Record(ctx, n, metric.WithAttributes(p.otelAttrs...))
+}
+
+func (p *prwTelemetryOtel) recordRemoteWriteSentBatch(ctx context.Context) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteSentBatchCount.Add(ctx, 1, metric.WithAttributes(p.otelAttrs...))
+}
+
 func (p *prwTelemetryOtel) recordTranslationFailure(ctx context.Context) {
 	p.telemetryBuilder.ExporterPrometheusremotewriteFailedTranslations.Add(ctx, 1, metric.WithAttributes(p.otelAttrs...))
 }
@@ -91,7 +101,7 @@ type prwExporter struct {
 	batchStatePool sync.Pool
 }
 
-func newPRWTelemetry(set exporter.Settings) (prwTelemetry, error) {
+func newPRWTelemetry(set exporter.Settings, endpointURL *url.URL) (prwTelemetry, error) {
 	telemetryBuilder, err := metadata.NewTelemetryBuilder(set.TelemetrySettings)
 	if err != nil {
 		return nil, err
@@ -101,6 +111,7 @@ func newPRWTelemetry(set exporter.Settings) (prwTelemetry, error) {
 		telemetryBuilder: telemetryBuilder,
 		otelAttrs: []attribute.KeyValue{
 			attribute.String("exporter", set.ID.String()),
+			attribute.String("endpoint", endpointURL.String()),
 		},
 	}, nil
 }
@@ -117,7 +128,7 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
 		return nil, errors.New("invalid endpoint")
 	}
 
-	prwTelemetry, err := newPRWTelemetry(set)
+	telemetry, err := newPRWTelemetry(set, endpointURL)
 	if err != nil {
 		return nil, err
 	}
@@ -132,6 +143,9 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
 		concurrency = *cfg.MaxBatchRequestParallelism
 	}
 
+	// Set the desired number of consumers as a metric for the exporter.
+	telemetry.setNumberConsumer(context.Background(), int64(concurrency))
+
 	prwe := &prwExporter{
 		endpointURL:       endpointURL,
 		wg:                new(sync.WaitGroup),
@@ -151,7 +165,7 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
 			AddMetricSuffixes:   cfg.AddMetricSuffixes,
 			SendMetadata:        cfg.SendMetadata,
 		},
-		telemetry:      prwTelemetry,
+		telemetry:      telemetry,
 		batchStatePool: sync.Pool{New: func() any { return newBatchTimeServicesState() }},
 	}
 
@@ -350,6 +364,7 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ
 		req.Header.Set("User-Agent", prwe.userAgentHeader)
 
 		resp, err := prwe.client.Do(req)
+		prwe.telemetry.recordRemoteWriteSentBatch(ctx)
 		if err != nil {
 			return err
 		}

exporter/prometheusremotewriteexporter/internal/metadata/generated_telemetry.go

@@ -27,3 +27,17 @@ telemetry:
       sum:
         value_type: int
         monotonic: true
+    exporter_prometheusremotewrite_consumers:
+      enabled: true
+      description: Number of configured workers to use to fan out the outgoing requests
+      unit: "1"
+      gauge:
+        value_type: int
+        monotonic: false
+    exporter_prometheusremotewrite_sent_batch_count:
+      enabled: true
+      description: Number of remote write request batches sent to the remote write endpoint
+      unit: "1"
+      sum:
+        value_type: int
+        monotonic: true