Distribute internal metrics across different levels (#9767)

**Description:**

This change distributes the reported internal metrics across available
levels and updates the level set by default:

1. The default level is changed from `basic` to `normal`, which can be
overridden with `service::telmetry::metrics::level` configuration.

2. The following batch processor metrics are updated to be reported
starting from `normal` level instead of `basic` level:
  - `processor_batch_batch_send_size`
  - `processor_batch_metadata_cardinality` 
  - `processor_batch_timeout_trigger_send` 
  - `processor_batch_size_trigger_send` 

3. The following GRPC/HTTP server and client metrics are updated to be
reported starting from `detailed` level:
  - `http.client.*` metrics 
  - `http.server.*` metrics 
  - `rpc.server.*` metrics 
  - `rpc.client.*` metrics

**Link to tracking Issue:**
#7890

Author: dmitryax

.chloggen/update-internal-metric-levels.yaml

@@ -0,0 +1,37 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: telemetry
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Distributed internal metrics across different levels.
+
+# One or more tracking issues or pull requests related to the change
+issues: [7890]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+    The internal metrics levels are updated along with reported metrics:
+    - The default level is changed from `basic` to `normal`, which can be overridden with `service::telmetry::metrics::level` configuration.
+    - Batch processor metrics are updated to be reported starting from `normal` level:
+      - `processor_batch_batch_send_size` 
+      - `processor_batch_metadata_cardinality`
+      - `processor_batch_timeout_trigger_send`
+      - `processor_batch_size_trigger_send`
+    - GRPC/HTTP server and client metrics are updated to be reported starting from `detailed` level:
+      - http.client.* metrics
+      - http.server.* metrics
+      - rpc.server.* metrics
+      - rpc.client.* metrics
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

config/configgrpc/configgrpc.go

@@ -30,6 +30,7 @@ import (
 	"go.opentelemetry.io/collector/config/configcompression"
 	"go.opentelemetry.io/collector/config/confignet"
 	"go.opentelemetry.io/collector/config/configopaque"
+	"go.opentelemetry.io/collector/config/configtelemetry"
 	"go.opentelemetry.io/collector/config/configtls"
 	"go.opentelemetry.io/collector/config/internal"
 	"go.opentelemetry.io/collector/extension/auth"
@@ -253,9 +254,11 @@ func (gcs *ClientConfig) toDialOptions(host component.Host, settings component.T
 
 	otelOpts := []otelgrpc.Option{
 		otelgrpc.WithTracerProvider(settings.TracerProvider),
-		otelgrpc.WithMeterProvider(settings.MeterProvider),
 		otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
 	}
+	if settings.MetricsLevel >= configtelemetry.LevelDetailed {
+		otelOpts = append(otelOpts, otelgrpc.WithMeterProvider(settings.MeterProvider))
+	}
 
 	// Enable OpenTelemetry observability plugin.
 	opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelOpts...)))
@@ -356,9 +359,11 @@ func (gss *ServerConfig) toServerOption(host component.Host, settings component.
 
 	otelOpts := []otelgrpc.Option{
 		otelgrpc.WithTracerProvider(settings.TracerProvider),
-		otelgrpc.WithMeterProvider(settings.MeterProvider),
 		otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
 	}
+	if settings.MetricsLevel >= configtelemetry.LevelDetailed {
+		otelOpts = append(otelOpts, otelgrpc.WithMeterProvider(settings.MeterProvider))
+	}
 
 	// Enable OpenTelemetry observability plugin.
 

config/configgrpc/go.mod

@@ -11,6 +11,7 @@ require (
 	go.opentelemetry.io/collector/config/configcompression v1.5.0
 	go.opentelemetry.io/collector/config/confignet v0.98.0
 	go.opentelemetry.io/collector/config/configopaque v1.5.0
+	go.opentelemetry.io/collector/config/configtelemetry v0.98.0
 	go.opentelemetry.io/collector/config/configtls v0.98.0
 	go.opentelemetry.io/collector/config/internal v0.98.0
 	go.opentelemetry.io/collector/extension/auth v0.98.0
@@ -48,7 +49,6 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.52.3 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	go.opentelemetry.io/collector/config/configtelemetry v0.98.0 // indirect
 	go.opentelemetry.io/collector/confmap v0.98.0 // indirect
 	go.opentelemetry.io/collector/extension v0.98.0 // indirect
 	go.opentelemetry.io/collector/featuregate v1.5.0 // indirect

config/confighttp/confighttp.go

@@ -23,6 +23,7 @@ import (
 	"go.opentelemetry.io/collector/config/configauth"
 	"go.opentelemetry.io/collector/config/configcompression"
 	"go.opentelemetry.io/collector/config/configopaque"
+	"go.opentelemetry.io/collector/config/configtelemetry"
 	"go.opentelemetry.io/collector/config/configtls"
 	"go.opentelemetry.io/collector/config/internal"
 	"go.opentelemetry.io/collector/extension/auth"
@@ -206,14 +207,17 @@ func (hcs *ClientConfig) ToClient(ctx context.Context, host component.Host, sett
 		}
 	}
 
+	otelOpts := []otelhttp.Option{
+		otelhttp.WithTracerProvider(settings.TracerProvider),
+		otelhttp.WithPropagators(otel.GetTextMapPropagator()),
+	}
+	if settings.MetricsLevel >= configtelemetry.LevelDetailed {
+		otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider))
+	}
+
 	// wrapping http transport with otelhttp transport to enable otel instrumentation
 	if settings.TracerProvider != nil && settings.MeterProvider != nil {
-		clientTransport = otelhttp.NewTransport(
-			clientTransport,
-			otelhttp.WithTracerProvider(settings.TracerProvider),
-			otelhttp.WithMeterProvider(settings.MeterProvider),
-			otelhttp.WithPropagators(otel.GetTextMapPropagator()),
-		)
+		clientTransport = otelhttp.NewTransport(clientTransport, otelOpts...)
 	}
 
 	if hcs.CustomRoundTripper != nil {
@@ -383,18 +387,20 @@ func (hss *ServerConfig) ToServer(_ context.Context, host component.Host, settin
 		handler = responseHeadersHandler(handler, hss.ResponseHeaders)
 	}
 
-	// Enable OpenTelemetry observability plugin.
-	// TODO: Consider to use component ID string as prefix for all the operations.
-	handler = otelhttp.NewHandler(
-		handler,
-		"",
+	otelOpts := []otelhttp.Option{
 		otelhttp.WithTracerProvider(settings.TracerProvider),
-		otelhttp.WithMeterProvider(settings.MeterProvider),
 		otelhttp.WithPropagators(otel.GetTextMapPropagator()),
 		otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string {
 			return r.URL.Path
 		}),
-	)
+	}
+	if settings.MetricsLevel >= configtelemetry.LevelDetailed {
+		otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider))
+	}
+
+	// Enable OpenTelemetry observability plugin.
+	// TODO: Consider to use component ID string as prefix for all the operations.
+	handler = otelhttp.NewHandler(handler, "", otelOpts...)
 
 	// wrap the current handler in an interceptor that will add client.Info to the request's context
 	handler = &clientInfoHandler{

otelcol/unmarshaler.go

@@ -59,7 +59,7 @@ func unmarshal(v *confmap.Conf, factories Factories) (*configSettings, error) {
 					InitialFields:     map[string]any(nil),
 				},
 				Metrics: telemetry.MetricsConfig{
-					Level:   configtelemetry.LevelBasic,
+					Level:   configtelemetry.LevelNormal,
 					Address: ":8888",
 				},
 			},

processor/batchprocessor/metrics.go

@@ -8,6 +8,7 @@ import (
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
+	noopmetric "go.opentelemetry.io/otel/metric/noop"
 	"go.uber.org/multierr"
 
 	"go.opentelemetry.io/collector/component"
@@ -56,8 +57,17 @@ func newBatchProcessorTelemetry(set processor.CreateSettings, currentMetadataCar
 }
 
 func (bpt *batchProcessorTelemetry) createOtelMetrics(set component.TelemetrySettings, currentMetadataCardinality func() int) error {
-	var errors, err error
-	meter := metadata.Meter(set)
+	var (
+		errors, err error
+		meter       metric.Meter
+	)
+
+	// BatchProcessor are emitted starting from Normal level only.
+	if bpt.level >= configtelemetry.LevelNormal {
+		meter = metadata.Meter(set)
+	} else {
+		meter = noopmetric.Meter{}
+	}
 
 	bpt.batchSizeTriggerSend, err = meter.Int64Counter(
 		processorhelper.BuildCustomMetricName(typeStr, "batch_size_trigger_send"),