[receiver/receiver_creator] Add support for enabling receivers/scrapers from K8s hints (open-telemetry#35617)

Author: ChrsMark

.chloggen/hints.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: receivercreator
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for starting receivers/scrapers based on provided annotations' hints for metrics' collection
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34427]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

extension/observer/endpoints.go

@@ -223,11 +223,11 @@ func (p *Pod) Type() EndpointType {
 // PodContainer is a discovered k8s pod's container
 type PodContainer struct {
 	// Name of the container
-	Name string
+	Name string `mapstructure:"container_name"`
 	// Image of the container
-	Image string
+	Image string `mapstructure:"container_image"`
 	// ContainerID is the id of the container exposing the Endpoint
-	ContainerID string
+	ContainerID string `mapstructure:"container_id"`
 	// Pod is the k8s pod in which the container is running
 	Pod Pod
 }

receiver/receivercreator/README.md

@@ -439,3 +439,191 @@ service:
 
 The full list of settings exposed for this receiver are documented [here](./config.go)
 with detailed sample configurations [here](./testdata/config.yaml).
+
+
+## Generate receiver configurations from provided Hints
+
+Note: When hints feature is enabled if hints are present for an endpoint no receiver templates will be evaluated.
+
+Currently this feature is only supported for K8s environments and the `k8sobserver`.
+
+The discovery feature for K8s is enabled with the following setting:
+
+```yaml
+receiver_creator/metrics:
+  watch_observers: [ k8s_observer ]
+  discovery:
+     enabled: true
+     # Define which receivers should be ignored when provided through annotations
+     # ignore_receivers: []
+```
+
+Find bellow the supported annotations that user can define to automatically enable receivers to start collecting metrics signals from the target Pods/containers.
+
+### Supported metrics annotations
+
+#### Enable/disable discovery
+
+`io.opentelemetry.discovery.metrics/enabled` (Required. `"true"` or `"false"`)
+
+#### Define scraper
+
+`io.opentelemetry.discovery.metrics/scraper` (example: `"nginx"`)
+
+
+#### Define configuration
+
+`io.opentelemetry.discovery.metrics/config`
+
+For `"endpoint"` setting specifically, it sticks to urls that include
+```"`endpoint`"``` as it comes from the Port endpoint which is
+in form of `pod_ip:container_port`. This is to ensure that each Pod can only
+generate configuration that targets itself and not others.
+If no endpoint is provided the Pod's endpoint will be used (in form of `pod_ip:container_port`).
+
+**Example:**
+
+```yaml
+io.opentelemetry.discovery.metrics/config: |
+  endpoint: "http://`endpoint`/nginx_status"
+  collection_interval: "20s"
+  initial_delay: "20s"
+  read_buffer_size: "10"
+  xyz: "abc"
+```
+
+
+#### Support multiple target containers
+
+Users can target the annotation to a specific container by suffixing it with the name of the port that container exposes:
+`io.opentelemetry.discovery.metrics.<container_port>/config`.
+For example:
+```yaml
+io.opentelemetry.discovery.metrics.80/config: |
+  endpoint: "http://`endpoint`/nginx_status"
+```
+where `80` is the port that the target container exposes.
+
+If a Pod is annotated with both container level hints and pod level hints the container level hints have priority and
+the Pod level hints are used as a fallback (see detailed example bellow).
+
+The current implementation relies on the implementation of `k8sobserver` extension and specifically
+the [pod_endpoint](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.111.0/extension/observer/k8sobserver/pod_endpoint.go).
+The hints are evaluated per container by extracting the annotations from each [`Port` endpoint](#Port) that is emitted. 
+
+
+
+### Examples
+
+#### Metrics example
+
+Collector's configuration:
+```yaml
+receivers:
+  receiver_creator/metrics:
+    watch_observers: [ k8s_observer ]
+    discovery:
+      enabled: true
+    receivers:
+
+service:
+  extensions: [ k8s_observer]
+  pipelines:
+    metrics:
+      receivers: [ receiver_creator ]
+      processors: []
+      exporters: [ debug ]
+```
+
+Target Pod annotated with hints:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-conf
+data:
+  nginx.conf: |
+    user  nginx;
+    worker_processes  1;
+    error_log  /dev/stderr warn;
+    pid        /var/run/nginx.pid;
+    events {
+      worker_connections  1024;
+    }
+    http {
+      include       /etc/nginx/mime.types;
+      default_type  application/octet-stream;
+
+      log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                        '$status $body_bytes_sent "$http_referer" '
+                        '"$http_user_agent" "$http_x_forwarded_for"';
+      access_log  /dev/stdout main;
+      server {
+          listen 80;
+          server_name localhost;
+
+          location /nginx_status {
+              stub_status on;
+          }
+      }
+      include /etc/nginx/conf.d/*;
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis-deployment
+  labels:
+    app: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+      annotations:
+        # redis container port metrics hints
+        io.opentelemetry.discovery.metrics.6379/enabled: "true"
+        io.opentelemetry.discovery.metrics.6379/scraper: redis
+        io.opentelemetry.discovery.metrics.6379/config: |
+          collection_interval: "20s"
+          timeout: "10s"
+
+        # nginx container port metrics hints
+        io.opentelemetry.discovery.metrics.80/enabled: "true"
+        io.opentelemetry.discovery.metrics.80/scraper: nginx
+        io.opentelemetry.discovery.metrics.80/config: |
+          endpoint: "http://`endpoint`/nginx_status"
+          collection_interval: "30s"
+          timeout: "20s"
+    spec:
+      volumes:
+        - name: nginx-conf
+          configMap:
+            name: nginx-conf
+            items:
+              - key: nginx.conf
+                path: nginx.conf
+      containers:
+        - name: webserver
+          image: nginx:latest
+          ports:
+            - containerPort: 80
+              name: webserver
+          volumeMounts:
+            - mountPath: /etc/nginx/nginx.conf
+              readOnly: true
+              subPath: nginx.conf
+              name: nginx-conf
+        - image: redis
+          imagePullPolicy: IfNotPresent
+          name: redis
+          ports:
+            - name: redis
+              containerPort: 6379
+              protocol: TCP
+```

receiver/receivercreator/config.go

@@ -35,6 +35,12 @@ type receiverConfig struct {
 // userConfigMap is an arbitrary map of string keys to arbitrary values as specified by the user
 type userConfigMap map[string]any
 
+type receiverSignals struct {
+	metrics bool
+	logs    bool
+	traces  bool
+}
+
 // receiverTemplate is the configuration of a single subreceiver.
 type receiverTemplate struct {
 	receiverConfig
@@ -46,6 +52,7 @@ type receiverTemplate struct {
 	// It can contain expr expressions for endpoint env value expansion
 	ResourceAttributes map[string]any `mapstructure:"resource_attributes"`
 	rule               rule
+	signals            receiverSignals
 }
 
 // resourceAttributes holds a map of default resource attributes for each Endpoint type.
@@ -60,6 +67,7 @@ func newReceiverTemplate(name string, cfg userConfigMap) (receiverTemplate, erro
 	}
 
 	return receiverTemplate{
+		signals: receiverSignals{metrics: true, logs: true, traces: true},
 		receiverConfig: receiverConfig{
 			id:         id,
 			config:     cfg,
@@ -78,6 +86,12 @@ type Config struct {
 	// ResourceAttributes is a map of default resource attributes to add to each resource
 	// object received by this receiver from dynamically created receivers.
 	ResourceAttributes resourceAttributes `mapstructure:"resource_attributes"`
+	Discovery          DiscoveryConfig    `mapstructure:"discovery"`
+}
+
+type DiscoveryConfig struct {
+	Enabled         bool     `mapstructure:"enabled"`
+	IgnoreReceivers []string `mapstructure:"ignore_receivers"`
 }
 
 func (cfg *Config) Unmarshal(componentParser *confmap.Conf) error {

receiver/receivercreator/config_test.go

@@ -90,6 +90,7 @@ func TestLoadConfig(t *testing.T) {
 						Rule:               `type == "port"`,
 						ResourceAttributes: map[string]any{"one": "two"},
 						rule:               portRule,
+						signals:            receiverSignals{true, true, true},
 					},
 					"nop/1": {
 						receiverConfig: receiverConfig{
@@ -102,6 +103,7 @@ func TestLoadConfig(t *testing.T) {
 						Rule:               `type == "port"`,
 						ResourceAttributes: map[string]any{"two": "three"},
 						rule:               portRule,
+						signals:            receiverSignals{true, true, true},
 					},
 				},
 				WatchObservers: []component.ID{