[datadogexporter] Improve support of semantic conventions for K8s, Azure and ECS (#2623)

TL;DR:
- Add support for the new Azure resource detector
- Improve support for Kubernetes semantic conventions
- Improve support for Kubernetes labels
- Improve support for ECS semantic conventions

### Azure

- The `host.id` (Azure VM ID) is sent as a host alias now, to replicate the Datadog Agent behavior.
- The cluster name is taken from the resource group ID.

### ECS

- Task family, cluster ARN and task revision are mapped to Datadog conventions. `container.id` priority to be taken as a hostname is lowered since we only want this as a last resort.

### Kubernetes

- OpenTelemetry Kubernetes semantic conventions are mapped to Datadog conventions.
- Kubernetes labels conventions (both Datadog-specific and general recommendations) are mapped to Datadog conventions.
- We now take the Kubernetes cluster name on Azure (see above) and AWS.

Author: mx-psi

exporter/datadogexporter/attributes/attributes.go

@@ -15,10 +15,53 @@
 package attributes
 
 import (
+	"fmt"
+
 	"go.opentelemetry.io/collector/consumer/pdata"
 	"go.opentelemetry.io/collector/translator/conventions"
 )
 
+var (
+	// conventionsMappings defines the mapping between OpenTelemetry semantic conventions
+	// and Datadog Agent conventions
+	conventionsMapping = map[string]string{
+		// ECS conventions
+		// https://github.com/DataDog/datadog-agent/blob/e081bed/pkg/tagger/collectors/ecs_extract.go
+		conventions.AttributeAWSECSTaskFamily: "task_family",
+		conventions.AttributeAWSECSClusterARN: "ecs_cluster_name",
+		"aws.ecs.task.revision":               "task_version",
+
+		// Kubernetes resource name (via semantic conventions)
+		// https://github.com/DataDog/datadog-agent/blob/e081bed/pkg/util/kubernetes/const.go
+		conventions.AttributeK8sPod:         "pod_name",
+		conventions.AttributeK8sDeployment:  "kube_deployment",
+		conventions.AttributeK8sReplicaSet:  "kube_replica_set",
+		conventions.AttributeK8sStatefulSet: "kube_stateful_set",
+		conventions.AttributeK8sDaemonSet:   "kube_daemon_set",
+		conventions.AttributeK8sJob:         "kube_job",
+		conventions.AttributeK8sCronJob:     "kube_cronjob",
+	}
+
+	// Kubernetes mappings defines the mapping between Kubernetes conventions (both general and Datadog specific)
+	// and Datadog Agent conventions. The Datadog Agent conventions can be found at
+	// https://github.com/DataDog/datadog-agent/blob/e081bed/pkg/tagger/collectors/const.go and
+	// https://github.com/DataDog/datadog-agent/blob/e081bed/pkg/util/kubernetes/const.go
+	kubernetesMapping = map[string]string{
+		// Standard Datadog labels
+		"tags.datadoghq.com/env":     "env",
+		"tags.datadoghq.com/service": "service",
+		"tags.datadoghq.com/version": "version",
+
+		// Standard Kubernetes labels
+		"app.kubernetes.io/name":       "kube_app_name",
+		"app.kubernetes.io/instance":   "kube_app_instance",
+		"app.kubernetes.io/version":    "kube_app_version",
+		"app.kuberenetes.io/component": "kube_app_component",
+		"app.kubernetes.io/part-of":    "kube_app_part_of",
+		"app.kubernetes.io/managed-by": "kube_app_managed_by",
+	}
+)
+
 // TagsFromAttributes converts a selected list of attributes
 // to a tag list that can be added to metrics.
 func TagsFromAttributes(attrs pdata.AttributeMap) []string {
@@ -47,6 +90,16 @@ func TagsFromAttributes(attrs pdata.AttributeMap) []string {
 		case conventions.AttributeOSType:
 			systemAttributes.OSType = value.StringVal()
 		}
+
+		// conventions mapping
+		if datadogKey, found := conventionsMapping[key]; found && value.StringVal() != "" {
+			tags = append(tags, fmt.Sprintf("%s:%s", datadogKey, value.StringVal()))
+		}
+
+		// Kubernetes labels mapping
+		if datadogKey, found := kubernetesMapping[key]; found && value.StringVal() != "" {
+			tags = append(tags, fmt.Sprintf("%s:%s", datadogKey, value.StringVal()))
+		}
 	})
 
 	tags = append(tags, processAttributes.extractTags()...)

exporter/datadogexporter/attributes/attributes_test.go

@@ -32,12 +32,18 @@ func TestTagsFromAttributes(t *testing.T) {
 		conventions.AttributeProcessID:             pdata.NewAttributeValueInt(1),
 		conventions.AttributeProcessOwner:          pdata.NewAttributeValueString("root"),
 		conventions.AttributeOSType:                pdata.NewAttributeValueString("LINUX"),
+		conventions.AttributeK8sDaemonSet:          pdata.NewAttributeValueString("daemon_set_name"),
+		conventions.AttributeAWSECSClusterARN:      pdata.NewAttributeValueString("cluster_arn"),
+		"tags.datadoghq.com/service":               pdata.NewAttributeValueString("service_name"),
 	}
 	attrs := pdata.NewAttributeMap().InitFromMap(attributeMap)
 
-	assert.Equal(t, []string{
+	assert.ElementsMatch(t, []string{
 		fmt.Sprintf("%s:%s", conventions.AttributeProcessExecutableName, "otelcol"),
 		fmt.Sprintf("%s:%s", conventions.AttributeOSType, "LINUX"),
+		fmt.Sprintf("%s:%s", "kube_daemon_set", "daemon_set_name"),
+		fmt.Sprintf("%s:%s", "ecs_cluster_name", "cluster_arn"),
+		fmt.Sprintf("%s:%s", "service", "service_name"),
 	}, TagsFromAttributes(attrs))
 }
 

exporter/datadogexporter/metadata/azure/azure.go

@@ -0,0 +1,68 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package azure
+
+import (
+	"strings"
+
+	"go.opentelemetry.io/collector/consumer/pdata"
+	"go.opentelemetry.io/collector/translator/conventions"
+)
+
+const (
+	// AttributeResourceGroupName is the Azure resource group name attribute
+	AttributeResourceGroupName = "azure.resourcegroup.name"
+)
+
+// HostInfo has the Azure host information
+type HostInfo struct {
+	HostAliases []string
+}
+
+// HostInfoFromAttributes gets Azure host info from attributes following
+// OpenTelemetry semantic conventions
+func HostInfoFromAttributes(attrs pdata.AttributeMap) (hostInfo *HostInfo) {
+	hostInfo = &HostInfo{}
+
+	// Add Azure VM ID as a host alias if available for compatibility with Azure integration
+	if vmID, ok := attrs.Get(conventions.AttributeHostID); ok {
+		hostInfo.HostAliases = append(hostInfo.HostAliases, vmID.StringVal())
+	}
+
+	return
+}
+
+// HostnameFromAttributes gets the Azure hostname from attributes
+func HostnameFromAttributes(attrs pdata.AttributeMap) (string, bool) {
+	if hostname, ok := attrs.Get(conventions.AttributeHostName); ok {
+		return hostname.StringVal(), true
+	}
+
+	return "", false
+}
+
+// ClusterNameFromAttributes gets the Azure cluster name from attributes
+func ClusterNameFromAttributes(attrs pdata.AttributeMap) (string, bool) {
+	// Get cluster name from resource group
+	// https://github.com/DataDog/datadog-agent/blob/aad29b8/pkg/util/azure/azure.go#L51
+	if resourceGroup, ok := attrs.Get(AttributeResourceGroupName); ok {
+		splitAll := strings.Split(resourceGroup.StringVal(), "_")
+		if len(splitAll) < 4 || strings.ToLower(splitAll[0]) != "mc" {
+			return "", false // Failed to parse
+		}
+		return splitAll[len(splitAll)-2], true
+	}
+
+	return "", false
+}

exporter/datadogexporter/metadata/azure/azure_test.go

@@ -0,0 +1,79 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/translator/conventions"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/testutils"
+)
+
+var (
+	testVMID     = "02aab8a4-74ef-476e-8182-f6d2ba4166a6"
+	testHostname = "test-hostname"
+	testAttrs    = testutils.NewAttributeMap(map[string]string{
+		conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure,
+		conventions.AttributeHostName:      testHostname,
+		conventions.AttributeCloudRegion:   "location",
+		conventions.AttributeHostID:        testVMID,
+		conventions.AttributeCloudAccount:  "subscriptionID",
+		AttributeResourceGroupName:         "resourceGroup",
+	})
+	testEmpty = testutils.NewAttributeMap(map[string]string{})
+)
+
+func TestHostInfoFromAttributes(t *testing.T) {
+	hostInfo := HostInfoFromAttributes(testAttrs)
+	require.NotNil(t, hostInfo)
+	assert.ElementsMatch(t, hostInfo.HostAliases, []string{testVMID})
+
+	emptyHostInfo := HostInfoFromAttributes(testEmpty)
+	require.NotNil(t, emptyHostInfo)
+	assert.ElementsMatch(t, emptyHostInfo.HostAliases, []string{})
+}
+
+func TestHostnameFromAttributes(t *testing.T) {
+	hostname, ok := HostnameFromAttributes(testAttrs)
+	assert.True(t, ok)
+	assert.Equal(t, hostname, testHostname)
+
+	_, ok = HostnameFromAttributes(testEmpty)
+	assert.False(t, ok)
+}
+
+func TestClusterNameFromAttributes(t *testing.T) {
+	cluster, ok := ClusterNameFromAttributes(testutils.NewAttributeMap(map[string]string{
+		AttributeResourceGroupName: "MC_aks-kenafeh_aks-kenafeh-eu_westeurope",
+	}))
+	assert.True(t, ok)
+	assert.Equal(t, cluster, "aks-kenafeh-eu")
+
+	cluster, ok = ClusterNameFromAttributes(testutils.NewAttributeMap(map[string]string{
+		AttributeResourceGroupName: "mc_foo-bar-aks-k8s-rg_foo-bar-aks-k8s_westeurope",
+	}))
+	assert.True(t, ok)
+	assert.Equal(t, cluster, "foo-bar-aks-k8s")
+
+	_, ok = ClusterNameFromAttributes(testutils.NewAttributeMap(map[string]string{
+		AttributeResourceGroupName: "unexpected-resource-group-name-format",
+	}))
+	assert.False(t, ok)
+
+	_, ok = ClusterNameFromAttributes(testutils.NewAttributeMap(map[string]string{}))
+	assert.False(t, ok)
+}

exporter/datadogexporter/metadata/ec2/ec2.go

@@ -25,8 +25,9 @@ import (
 )
 
 var (
-	defaultPrefixes = [3]string{"ip-", "domu", "ec2amaz-"}
-	ec2TagPrefix    = "ec2.tag."
+	defaultPrefixes  = [3]string{"ip-", "domu", "ec2amaz-"}
+	ec2TagPrefix     = "ec2.tag."
+	clusterTagPrefix = ec2TagPrefix + "kubernetes.io/cluster/"
 )
 
 type HostInfo struct {
@@ -123,3 +124,16 @@ func HostInfoFromAttributes(attrs pdata.AttributeMap) (hostInfo *HostInfo) {
 
 	return
 }
+
+// ClusterNameFromAttributes gets the AWS cluster name from attributes
+func ClusterNameFromAttributes(attrs pdata.AttributeMap) (clusterName string, ok bool) {
+	// Get cluster name from tag keys
+	// https://github.com/DataDog/datadog-agent/blob/1c94b11/pkg/util/ec2/ec2.go#L238
+	attrs.ForEach(func(k string, _ pdata.AttributeValue) {
+		if strings.HasPrefix(k, clusterTagPrefix) {
+			clusterName = strings.Split(k, "/")[2]
+			ok = true
+		}
+	})
+	return
+}

exporter/datadogexporter/metadata/ec2/ec2_test.go

@@ -81,3 +81,14 @@ func TestHostInfoFromAttributes(t *testing.T) {
 	assert.Equal(t, hostInfo.EC2Hostname, testIP)
 	assert.ElementsMatch(t, hostInfo.EC2Tags, []string{"tag1:val1", "tag2:val2"})
 }
+
+func TestClusterNameFromAttributes(t *testing.T) {
+	cluster, ok := ClusterNameFromAttributes(testutils.NewAttributeMap(map[string]string{
+		"ec2.tag.kubernetes.io/cluster/clustername": "dummy_value",
+	}))
+	assert.True(t, ok)
+	assert.Equal(t, cluster, "clustername")
+
+	_, ok = ClusterNameFromAttributes(testutils.NewAttributeMap(map[string]string{}))
+	assert.False(t, ok)
+}

exporter/datadogexporter/metadata/host.go

@@ -20,6 +20,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/azure"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/ec2"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/gcp"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/metadata/system"
@@ -66,6 +67,21 @@ func GetHost(logger *zap.Logger, cfg *config.Config) *string {
 	return &hostname
 }
 
+func getClusterName(attrs pdata.AttributeMap) (string, bool) {
+	if k8sClusterName, ok := attrs.Get(conventions.AttributeK8sCluster); ok {
+		return k8sClusterName.StringVal(), true
+	}
+
+	cloudProvider, ok := attrs.Get(conventions.AttributeCloudProvider)
+	if ok && cloudProvider.StringVal() == conventions.AttributeCloudProviderAzure {
+		return azure.ClusterNameFromAttributes(attrs)
+	} else if ok && cloudProvider.StringVal() == conventions.AttributeCloudProviderAWS {
+		return ec2.ClusterNameFromAttributes(attrs)
+	}
+
+	return "", false
+}
+
 // HostnameFromAttributes tries to get a valid hostname from attributes by checking, in order:
 //
 //   1. a custom Datadog hostname provided by the "datadog.host.name" attribute
@@ -84,23 +100,19 @@ func HostnameFromAttributes(attrs pdata.AttributeMap) (string, bool) {
 
 	// Kubernetes: node-cluster if cluster name is available, else node
 	if k8sNodeName, ok := attrs.Get(AttributeK8sNodeName); ok {
-		if k8sClusterName, ok := attrs.Get(conventions.AttributeK8sCluster); ok {
-			return k8sNodeName.StringVal() + "-" + k8sClusterName.StringVal(), true
+		if k8sClusterName, ok := getClusterName(attrs); ok {
+			return k8sNodeName.StringVal() + "-" + k8sClusterName, true
 		}
 		return k8sNodeName.StringVal(), true
 	}
 
-	// container id (e.g. from Docker)
-	if containerID, ok := attrs.Get(conventions.AttributeContainerID); ok {
-		return containerID.StringVal(), true
-	}
-
-	// handle AWS case separately to have similar behavior to the Datadog Agent
 	cloudProvider, ok := attrs.Get(conventions.AttributeCloudProvider)
 	if ok && cloudProvider.StringVal() == conventions.AttributeCloudProviderAWS {
 		return ec2.HostnameFromAttributes(attrs)
 	} else if ok && cloudProvider.StringVal() == conventions.AttributeCloudProviderGCP {
 		return gcp.HostnameFromAttributes(attrs)
+	} else if ok && cloudProvider.StringVal() == conventions.AttributeCloudProviderAzure {
+		return azure.HostnameFromAttributes(attrs)
 	}
 
 	// host id from cloud provider
@@ -113,5 +125,10 @@ func HostnameFromAttributes(attrs pdata.AttributeMap) (string, bool) {
 		return hostName.StringVal(), true
 	}
 
+	// container id (e.g. from Docker)
+	if containerID, ok := attrs.Get(conventions.AttributeContainerID); ok {
+		return containerID.StringVal(), true
+	}
+
 	return "", false
 }