Switch to gcp from gce/gke (#2488)

* Switch to gcp from gce as specified in open-telemetry/opentelemetry-collector-contrib#10348
Tracking issue: https://github.com/signalfx/splunk-otel-collector/issues/2474

* Update cmd/otelcol/config/collector/full_config_linux.yaml

Co-authored-by: Antoine Toulme <[email protected]>

Co-authored-by: Antoine Toulme <[email protected]>

Author: hughesjj

cmd/otelcol/config/collector/agent_config.yaml

@@ -110,7 +110,7 @@ processors:
   # libraries can send wrong values from container environments.
   # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
   resourcedetection:
-    detectors: [gce, ecs, ec2, azure, system]
+    detectors: [gcp, ecs, ec2, azure, system]
     override: true
 
   # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and 

cmd/otelcol/config/collector/full_config_linux.yaml

@@ -204,8 +204,8 @@ processors:
   # Enables the resource detection processor with default settings
   # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor
   # NOTE: This processor settings need to be change when using this processor
-  #resouredection:
-    # # a list of resource detectors to run, valid options are: "env", "system",  "gce", "ec2", "ecs", "elastic_beanstalk"
+  #resourcedetection:
+    # # a list of resource detectors to run, valid options are: "env", "system",  "gcp", "ec2", "ecs", "elastic_beanstalk"
     #detectors: [ <string> ]
     # # determines if existing resource attributes should be overridden or preserved, defaults to true
     #override: <bool>
@@ -214,7 +214,7 @@ processors:
   # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info.
   # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
   resourcedetection/internal:
-    detectors: [gce, ecs, ec2, azure, system]
+    detectors: [gcp, ecs, ec2, azure, system]
     override: true
 
   # Enables the memory limiter processor with default settings

cmd/otelcol/config/collector/gateway_config.yaml

@@ -83,7 +83,7 @@ processors:
   # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info.
   # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
   resourcedetection/internal:
-    detectors: [gce, ecs, ec2, azure, system]
+    detectors: [gcp, ecs, ec2, azure, system]
     override: true
 
 exporters:

cmd/otelcol/config/collector/otlp_config_linux.yaml

@@ -70,7 +70,7 @@ processors:
   # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info.
   # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
   resourcedetection/internal:
-    detectors: [gce, ecs, ec2, azure, system]
+    detectors: [gcp, ecs, ec2, azure, system]
     override: true
 
 exporters:

cmd/otelcol/config/collector/upstream_agent_config.yaml

@@ -100,7 +100,7 @@ processors:
   # libraries can send wrong values from container environments.
   # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
   resourcedetection:
-    detectors: [gce, ecs, ec2, azure, system]
+    detectors: [gcp, ecs, ec2, azure, system]
     override: true
 
   # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and 

cmd/translatesfx/README.md

@@ -396,7 +396,7 @@ by the `signalfx` exporter to generate cloud resource IDs.
   resourcedetection:
     detectors:
       - env
-      - gce
+      - gcp
       - ecs
       - ec2
       - azure

cmd/translatesfx/translatesfx/otel.go

@@ -63,7 +63,7 @@ func newOtelCfg() *otelCfg {
 		Receivers: map[string]map[string]any{},
 		Processors: map[string]map[string]any{
 			resourceDetection: {
-				"detectors": []string{"system", "env", "gce", "ecs", "ec2", "azure"},
+				"detectors": []string{"system", "env", "gcp", "ecs", "ec2", "azure"},
 			},
 		},
 		Extensions: map[string]map[string]any{},

cmd/translatesfx/translatesfx/otel_test.go

@@ -159,7 +159,7 @@ func TestInfoToOtelConfig_ResourceDetectionProcessor(t *testing.T) {
 	assert.NotNil(t, oc.Processors)
 	rdProc := oc.Processors["resourcedetection"]
 	assert.Equal(t, map[string]any{
-		"detectors": []string{"system", "env", "gce", "ecs", "ec2", "azure"},
+		"detectors": []string{"system", "env", "gcp", "ecs", "ec2", "azure"},
 	}, rdProc)
 	assert.Equal(t, []string{"resourcedetection"}, oc.Service.Pipelines["metrics"].Processors)
 }

cmd/translatesfx/translatesfx/testdata/otel-e2e-expected.yaml

@@ -56,7 +56,7 @@ processors:
     detectors:
       - system
       - env
-      - gce
+      - gcp
       - ecs
       - ec2
       - azure

docs/apm-infra-correlation.md

@@ -67,7 +67,7 @@ receivers:
 ...
 processors:
   resourcedetection:
-    detectors: [system,env,gce,ec2]
+    detectors: [system,env,gcp,ec2]
     override: true
   resource/add_environment:
     attributes:
@@ -164,7 +164,7 @@ receivers:
 ...
 processors:
   resourcedetection:
-    detectors: [system,env,gce,ec2]
+    detectors: [system,env,gcp,ec2]
     override: true
   resource/add_environment:
     attributes:

internal/configconverter/README.md

@@ -0,0 +1,74 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configconverter
+
+import (
+	"context"
+	"log"
+	"regexp"
+
+	"go.opentelemetry.io/collector/confmap"
+)
+
+type NormalizeGcp struct{}
+
+func (NormalizeGcp) Convert(_ context.Context, in *confmap.Conf) error {
+	if in == nil {
+		return nil
+	}
+
+	const resourceDetector = "processors::resourcedetection(?P<processor_name>/(?:[^:]|:[^:])+)?::detectors"
+	resourceDetectorRE := regexp.MustCompile(resourceDetector)
+	out := map[string]any{}
+	nonNormalizedGcpDetectorFound := false
+
+	for _, k := range in.AllKeys() {
+		v := in.Get(k)
+		match := resourceDetectorRE.FindStringSubmatch(k)
+		if match != nil {
+			if vArr, ok := v.([]interface{}); ok {
+				normalizedV := make([]interface{}, 0, len(vArr))
+				found := false
+				for _, item := range vArr {
+					switch item.(type) {
+					case string:
+						if item == "gce" || item == "gke" {
+							if !found {
+								normalizedV = append(normalizedV, "gcp")
+							}
+							found = true
+							nonNormalizedGcpDetectorFound = true
+						} else if item != nil {
+							normalizedV = append(normalizedV, item)
+						}
+					default:
+						if item != nil {
+							normalizedV = append(normalizedV, item)
+						}
+					}
+				}
+				out[k] = normalizedV
+			}
+		}
+	}
+	if nonNormalizedGcpDetectorFound {
+		log.Println("[WARNING] `processors` -> `resourcedetection` -> `detectors` parameter " +
+			"contains a deprecated configuration. Please update the config according to the guideline: " +
+			"https://github.com/signalfx/splunk-otel-collector#from-0680-to-0690.")
+	}
+
+	in.Merge(confmap.NewFromStringMap(out))
+	return nil
+}

internal/configconverter/normalize_gcp.go

@@ -0,0 +1,113 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configconverter
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/confmap/confmaptest"
+)
+
+func TestNormalizeGcp(t *testing.T) {
+	expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml")
+	require.NotNil(t, expectedCfgMap)
+	require.NoError(t, err)
+
+	cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config.yaml")
+	require.NoError(t, err)
+	require.NotNil(t, cfgMap)
+
+	err = NormalizeGcp{}.Convert(context.Background(), cfgMap)
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedCfgMap, cfgMap)
+}
+
+func TestNormalizeGcpMany(t *testing.T) {
+	expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml")
+	require.NotNil(t, expectedCfgMap)
+	require.NoError(t, err)
+
+	cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_many.yaml")
+	require.NoError(t, err)
+	require.NotNil(t, cfgMap)
+
+	err = NormalizeGcp{}.Convert(context.Background(), cfgMap)
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedCfgMap, cfgMap)
+}
+
+func TestNormalizeGcpSame(t *testing.T) {
+	expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml")
+	require.NotNil(t, expectedCfgMap)
+	require.NoError(t, err)
+
+	cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_post_migration.yaml")
+	require.NoError(t, err)
+	require.NotNil(t, cfgMap)
+
+	err = NormalizeGcp{}.Convert(context.Background(), cfgMap)
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedCfgMap, cfgMap)
+}
+
+func TestNormalizeGcpNoop(t *testing.T) {
+	expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_no_op.yaml")
+	require.NotNil(t, expectedCfgMap)
+	require.NoError(t, err)
+
+	cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_no_op.yaml")
+	require.NoError(t, err)
+	require.NotNil(t, cfgMap)
+
+	err = NormalizeGcp{}.Convert(context.Background(), cfgMap)
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedCfgMap, cfgMap)
+}
+
+func TestNormalizeGcpSubresources(t *testing.T) {
+	expectedCfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml")
+	require.NotNil(t, expectedCfgMap)
+	require.NoError(t, err)
+
+	cfgMap, err := confmaptest.LoadConf("testdata/normalize_gcp/upstream_agent_config_subresources.yaml")
+	require.NoError(t, err)
+	require.NotNil(t, cfgMap)
+
+	err = NormalizeGcp{}.Convert(context.Background(), cfgMap)
+	require.NoError(t, err)
+
+	assert.Equal(t, expectedCfgMap, cfgMap)
+}

internal/configconverter/normalize_gcp_test.go

@@ -0,0 +1,9 @@
+processors:
+  # Detect if the collector is running on a cloud system, which is important for creating unique cloud provider dimensions.
+  # Detector order is important: the `system` detector goes last so it can't preclude cloud detectors from setting host/os info.
+  # Resource detection processor is configured to override all host and cloud attributes because instrumentation
+  # libraries can send wrong values from container environments.
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#ordering
+  resourcedetection:
+    detectors: [gce, ecs, ec2, azure, system]
+    override: true

internal/configconverter/testdata/normalize_gcp/upstream_agent_config.yaml

@@ -0,0 +1,4 @@
+processors:
+  resourcedetection:
+    detectors: [gce, ecs, gke, ec2, azure, system]
+    override: true

internal/configconverter/testdata/normalize_gcp/upstream_agent_config_many.yaml

@@ -0,0 +1,4 @@
+processors:
+  resourcedetection:
+    detectors: [ecs, ec2, azure, system]
+    override: true

internal/configconverter/testdata/normalize_gcp/upstream_agent_config_no_op.yaml

@@ -0,0 +1,4 @@
+processors:
+  resourcedetection:
+    detectors: [gcp, ecs, ec2, azure, system]
+    override: true

internal/configconverter/testdata/normalize_gcp/upstream_agent_config_post_migration.yaml

@@ -0,0 +1,9 @@
+processors:
+  resourcedetection:
+    detectors: [gce, ecs, gke, ec2, azure, system]
+  resourcedetection/1:
+    detectors: [gce, ecs, gke, ec2, azure, system]
+  resourcedetection/abc:
+    detectors: [gce, ecs, gke, ec2, azure, system]
+  resourcedetection/abc123_+!@#%^&*():
+    detectors: [gce, ecs, gke, ec2, azure, system]

internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources.yaml

@@ -0,0 +1,9 @@
+processors:
+  resourcedetection:
+    detectors: [gcp, ecs, ec2, azure, system]
+  resourcedetection/1:
+    detectors: [gcp, ecs, ec2, azure, system]
+  resourcedetection/abc:
+    detectors: [gcp, ecs, ec2, azure, system]
+  resourcedetection/abc123_+!@#%^&*():
+    detectors: [gcp, ecs, ec2, azure, system]

internal/configconverter/testdata/normalize_gcp/upstream_agent_config_subresources_post_migration.yaml

@@ -147,6 +147,7 @@ func (s *Settings) ConfMapConverters() []confmap.Converter {
 			configconverter.MoveOTLPInsecureKey{},
 			configconverter.MoveHecTLS{},
 			configconverter.RenameK8sTagger{},
+			configconverter.NormalizeGcp{},
 		)
 	}
 	return confMapConverters

internal/settings/settings.go

@@ -133,6 +133,7 @@ func TestNewSettingsConvertConfig(t *testing.T) {
 		configconverter.MoveOTLPInsecureKey{},
 		configconverter.MoveHecTLS{},
 		configconverter.RenameK8sTagger{},
+		configconverter.NormalizeGcp{},
 	}, settings.ConfMapConverters())
 	require.Equal(t, []string{"--feature-gates", "foo", "--feature-gates", "-bar"}, settings.ColCoreArgs())
 }