chore(otel): add support for http/protobuf OTLP metrics export (#15061)

Signed-off-by: Milas Bowman <[email protected]>

Author: milas

docs/metrics.md

@@ -31,6 +31,8 @@ It will not be enabled if left blank, unlike some other implementations.
 
 You can configure the protocol using the environment variables documented in [standard environment variables](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/).
 
+By default, GRPC is used; to switch to HTTP, set either the `OTEL_EXPORTER_OTLP_PROTOCOL` or `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL` environment variable to `http/protobuf`.
+
 The [configuration options](#common) in the controller ConfigMap `metricsTTL`, `modifiers` and `temporality` affect the OpenTelemetry behavior, but the other parameters do not.
 
 To use the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) you can configure it

go.mod

@@ -60,6 +60,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/runtime v0.61.0
 	go.opentelemetry.io/otel v1.36.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.58.0
 	go.opentelemetry.io/otel/metric v1.36.0
 	go.opentelemetry.io/otel/sdk v1.36.0

go.sum

@@ -872,6 +872,8 @@ go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
 go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 h1:zwdo1gS2eH26Rg+CoqVQpEK1h8gvt5qyU5Kk5Bixvow=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0/go.mod h1:rUKCPscaRWWcqGT6HnEmYrK+YNe5+Sw64xgQTOJ5b30=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 h1:gAU726w9J8fwr4qRDqu1GYMNNs4gXrU+Pv20/N1UpB4=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0/go.mod h1:RboSDkp7N292rgu+T0MgVt2qgFGu6qa1RpZDOtpL76w=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM=

util/telemetry/metrics.go

@@ -3,13 +3,14 @@ package telemetry
 import (
 	"context"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
-	"go.opentelemetry.io/otel"
-
 	"go.opentelemetry.io/contrib/instrumentation/runtime"
+	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
 	"go.opentelemetry.io/otel/metric"
 	metricsdk "go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/resource"
@@ -74,13 +75,32 @@ func NewMetrics(ctx context.Context, serviceName, prometheusName string, config
 	_, otlpEnabled := os.LookupEnv(`OTEL_EXPORTER_OTLP_ENDPOINT`)
 	_, otlpMetricsEnabled := os.LookupEnv(`OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`)
 	logger := logging.RequireLoggerFromContext(ctx)
+
 	if otlpEnabled || otlpMetricsEnabled {
 		logger.Info(ctx, "Starting OTLP metrics exporter")
-		otelExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithTemporalitySelector(config.Temporality))
-		if err != nil {
-			return nil, err
+
+		// NOTE: The OTel SDK default changed from gRPC to http/protobuf. For backwards compatibility,
+		// gRPC is preserved as the default in workflows controller, but http/protobuf can be opted-in
+		// to by setting the _PROTOCOL env var explicitly.
+		// These env vars match the official SDK: https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#otel_exporter_otlp_metrics_protocol.
+		otlpProtocol := os.Getenv(`OTEL_EXPORTER_OTLP_METRICS_PROTOCOL`)
+		if otlpProtocol == "" {
+			otlpProtocol = os.Getenv(`OTEL_EXPORTER_OTLP_PROTOCOL`)
+		}
+
+		if otlpProtocol == "" || otlpProtocol == "grpc" {
+			httpExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithTemporalitySelector(config.Temporality))
+			if err != nil {
+				return nil, err
+			}
+			options = append(options, metricsdk.WithReader(metricsdk.NewPeriodicReader(httpExporter)))
+		} else if strings.HasPrefix(otlpProtocol, "http/") {
+			grpcExporter, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTemporalitySelector(config.Temporality))
+			if err != nil {
+				return nil, err
+			}
+			options = append(options, metricsdk.WithReader(metricsdk.NewPeriodicReader(grpcExporter)))
 		}
-		options = append(options, metricsdk.WithReader(metricsdk.NewPeriodicReader(otelExporter)))
 	}
 
 	if config.Enabled {