Merge branch 'main' into remote-eol-ubuntu

Author: pjanotti

.github/workflows/splunk-ta-otel.yml

@@ -96,7 +96,7 @@ jobs:
   distribute-ta:
     name: "distribute-ta" # what gets run to package in gitlab
     runs-on: ubuntu-24.04
-    needs: [setup-environment, check_changes]
+    needs: [setup-environment, check_changes, test-discovery]
     if: needs.check_changes.outputs.ta_packaging_change == 'true' || needs.check_changes.outputs.ta_workflow_change == 'true'
     steps:
       - name: Check out the codebase.
@@ -106,3 +106,27 @@ jobs:
         run: |
           set -o pipefail
           PLATFORM="all" make -e distribute-ta
+
+  test-discovery:
+    name: test-discovery
+    runs-on: ubuntu-24.04
+    needs: [setup-environment, test]
+    strategy:
+      matrix:
+        PLATFORM: ["all"]
+    steps:
+      - name: Check out the codebase.
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: '**/go.sum'
+
+      - name: Build & Package TA
+        run: |
+          set -o pipefail
+          make -e bundle.d otelcol generate-technical-addon copy-local-build-to-ta package-ta
+          make -e discovery-test-ta
+

cmd/otelcol/config/collector/fargate_config.yaml

@@ -1,10 +1,5 @@
 # This collector config file is designed for use within an ECS task.
 # The collector should run in a sidecar container within an ECS task.
-config_sources:
-  env:
-    defaults:
-      METRICS_TO_EXCLUDE: []
-
 extensions:
   health_check:
     endpoint: 0.0.0.0:13133
@@ -73,14 +68,6 @@ processors:
   resourcedetection:
     detectors: [ecs]
     override: false
-  # Enables the filter processor with example settings
-  # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor
-  # NOTE: These settings need to be change when using this processor
-  filter/1:
-    metrics:
-      exclude:
-        match_type: regexp
-        metric_names: ${env:METRICS_TO_EXCLUDE}
   # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and
   # traces when it's not populated by instrumentation libraries.
   # If enabled, make sure to enable this processor in the pipeline below.

cmd/otelcol/fips/config/fargate_config.yaml

@@ -1,10 +1,5 @@
 # This collector config file is designed for use within an ECS task.
 # The collector should run in a sidecar container within an ECS task.
-config_sources:
-  env:
-    defaults:
-      METRICS_TO_EXCLUDE: []
-
 extensions:
   health_check:
     endpoint: 0.0.0.0:13133
@@ -70,14 +65,7 @@ processors:
   resourcedetection:
     detectors: [ecs]
     override: false
-  # Enables the filter processor with example settings
-  # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor
-  # NOTE: These settings need to be change when using this processor
-  filter/1:
-    metrics:
-      exclude:
-        match_type: regexp
-        metric_names: ${env:METRICS_TO_EXCLUDE}
+
   # Optional: The following processor can be used to add a default "deployment.environment" attribute to the logs and
   # traces when it's not populated by instrumentation libraries.
   # If enabled, make sure to enable this processor in the pipeline below.

deployments/fargate/README.md

@@ -53,8 +53,7 @@ In summary, the default Collector container definition does the following:
 - Sets the realm using environment variable `SPLUNK_REALM`.
 - Sets the default configuration file path using environment variable `SPLUNK_CONFIG`.
 
-Assign a stringified array of metrics you want excluded to environment variable
-`METRICS_TO_EXCLUDE`. You can set the memory limit for the memory limiter processor using
+You can set the memory limit for the memory limiter processor using
 environment variable `SPLUNK_MEMORY_LIMIT_MIB`. The default memory limit is 512 MiB. For
 more information about the memory limiter processor, see
 [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md)

packaging/technical-addon/Makefile

@@ -108,6 +108,15 @@ smoketest-ta:
 	PLATFORM="$(PLATFORM)" \
 	$(SOURCE_DIR)/packaging-scripts/cicd-tests/smoketests/smoketest.sh
 
+.PHONY: discovery-test-ta
+discovery-test-ta:
+	SOURCE_DIR="$(SOURCE_DIR)" \
+	BUILD_DIR="$(BUILD_DIR)" \
+	ARCH="$(ARCH)" \
+	PLATFORM="$(PLATFORM)" \
+	$(SOURCE_DIR)/packaging-scripts/cicd-tests/discovery/discovery-docker-test.sh
+
+
 .PHONY: update-ta-deps
 update-ta-deps:
 	SOURCE_DIR="$(SOURCE_DIR)" \

packaging/technical-addon/Splunk_TA_otel/README/inputs.conf.spec

@@ -25,7 +25,7 @@ splunk_config = <value>
 * Value to use for the `SPLUNK_CONFIG` environment variable (default `$SPLUNK_OTEL_TA_HOME/config/ta_agent_config.yaml`)
 
 splunk_config_dir = <value>
-* Value to use for the `SPLUNK_CONFIG_DIR` environment variable (default `$SPLUNK_OTEL_TA_HOME/config/`)
+* Value to use for the `SPLUNK_CONFIG_DIR` environment variable (default `$SPLUNK_OTEL_TA_HOME/config/`).  Same as `--configd` parameter as defined in https://github.com/signalfx/splunk-otel-collector/blob/v0.114.0/internal/confmapprovider/discovery/README.md?plain=1#L47
 
 splunk_debug_config_server = <value>
 * Value to use for the `SPLUNK_DEBUG_CONFIG_SERVER` environment variable
@@ -50,3 +50,12 @@ splunk_ingest_url = <value>
 
 splunk_realm = <value>
 * Splunk Observability realm to use for the `SPLUNK_REALM` environment variable (ex us0)
+
+discovery = <value>
+* Boolean, if `true` will enable `--discovery`
+
+discovery_properties = <value>
+* String, same as `--discovery-properties` parameter.  See [properties reference](https://github.com/signalfx/splunk-otel-collector/blob/v0.114.0/internal/confmapprovider/discovery/README.md?plain=1#L175)
+
+configd = <value>
+* Boolean, if `true` will enable `--configd`

packaging/technical-addon/Splunk_TA_otel/linux_x86_64/bin/Splunk_TA_otel.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 # script args related vars
 splunk_TA_otel_num_args="$#"
 splunk_TA_otel_args=("$@")
@@ -17,8 +16,16 @@ if [ -z "$SPLUNK_HOME" ]; then
     SPLUNK_HOME="$(realpath "$SPLUNK_OTEL_TA_HOME/../../../")"
 fi
 
+SPLUNK_OTEL_FLAGS=""
+#SPLUNK_OTEL_TA_DEBUG="DEBUG"
 
 # Begin autogenerated code
+configd_name="configd"
+configd_value=""
+discovery_name="discovery"
+discovery_value=""
+discovery_properties_name="discovery_properties"
+discovery_properties_value=""
 gomemlimit_name="gomemlimit"
 gomemlimit_value=""
 splunk_api_url_name="splunk_api_url"
@@ -112,6 +119,27 @@ splunk_TA_otel_read_configs() {
         splunk_TA_otel_log_msg "DEBUG" "line: $line"
 
         # Begin autogenerated code
+        has_configd="$(echo "$line" | grep "$configd_name")"
+        if [ "$has_configd" ] ; then
+            splunk_TA_otel_log_msg "DEBUG" "reading $configd_name from line $has_configd"
+            configd_value="$(echo "$has_configd" | grep -Eo ">(.*?)<" | sed 's/^>\(.*\)<$/\1/')"
+            splunk_TA_otel_log_msg "DEBUG" "Set $configd_name to $configd_value"
+        fi
+
+        has_discovery="$(echo "$line" | grep "$discovery_name")"
+        if [ "$has_discovery" ] ; then
+            splunk_TA_otel_log_msg "DEBUG" "reading $discovery_name from line $has_discovery"
+            discovery_value="$(echo "$has_discovery" | grep -Eo ">(.*?)<" | sed 's/^>\(.*\)<$/\1/')"
+            splunk_TA_otel_log_msg "DEBUG" "Set $discovery_name to $discovery_value"
+        fi
+
+        has_discovery_properties="$(echo "$line" | grep "$discovery_properties_name")"
+        if [ "$has_discovery_properties" ] ; then
+            splunk_TA_otel_log_msg "DEBUG" "reading $discovery_properties_name from line $has_discovery_properties"
+            discovery_properties_value="$(echo "$has_discovery_properties" | grep -Eo ">(.*?)<" | sed 's/^>\(.*\)<$/\1/')"
+            splunk_TA_otel_log_msg "DEBUG" "Set $discovery_properties_name to $discovery_properties_value"
+        fi
+
         has_gomemlimit="$(echo "$line" | grep "$gomemlimit_name")"
         if [ "$has_gomemlimit" ] ; then
             splunk_TA_otel_log_msg "DEBUG" "reading $gomemlimit_name from line $has_gomemlimit"
@@ -274,6 +302,8 @@ splunk_TA_otel_run_agent() {
     splunk_collectd_dir_value="$(Splunk_TA_otel_expand_config_param "$splunk_collectd_dir_value")"
     splunk_otel_log_file_value="$(Splunk_TA_otel_expand_config_param "$splunk_otel_log_file_value")"
     splunk_access_token_file_value="$(Splunk_TA_otel_expand_config_param "$splunk_access_token_file_value")"
+    discovery_properties_value="$(Splunk_TA_otel_expand_config_param "$discovery_properties_value")"
+    splunk_config_dir_value="$(Splunk_TA_otel_expand_config_param "$splunk_config_dir_value")"
 
     get_access_token
     if [[ $? == 1 ]] ; then
@@ -307,11 +337,13 @@ splunk_TA_otel_run_agent() {
     fi
     if [ "$splunk_config_value" ] ; then
       export SPLUNK_CONFIG="$splunk_config_value"
+      SPLUNK_OTEL_FLAGS="$SPLUNK_OTEL_FLAGS --config=$splunk_config_value"
     else
       splunk_TA_otel_log_msg "DEBUG" "NOT SET: $splunk_config_name"
     fi
     if [ "$splunk_config_dir_value" ] ; then
       export SPLUNK_CONFIG_DIR="$splunk_config_dir_value"
+      SPLUNK_OTEL_FLAGS="$SPLUNK_OTEL_FLAGS --config-dir=$splunk_config_dir_value"
     else
       splunk_TA_otel_log_msg "DEBUG" "NOT SET: $splunk_config_dir_name"
     fi
@@ -370,6 +402,21 @@ splunk_TA_otel_run_agent() {
     else
       splunk_TA_otel_log_msg "DEBUG" "NOT SET: $splunk_realm_name"
     fi
+    if [ "$configd_value" ] ; then
+      SPLUNK_OTEL_FLAGS="$SPLUNK_OTEL_FLAGS --configd"
+    else
+      splunk_TA_otel_log_msg "INFO" "BOOLEAN PARAM NOT SET: $configd_name"
+    fi
+    if [ "$discovery_value" ] ; then
+      SPLUNK_OTEL_FLAGS="$SPLUNK_OTEL_FLAGS --discovery"
+    else
+      splunk_TA_otel_log_msg "INFO" "BOOLEAN PARAM NOT SET: $discovery_name"
+    fi
+    if [ "$discovery_properties_value" ] ; then
+      SPLUNK_OTEL_FLAGS="$SPLUNK_OTEL_FLAGS --discovery-properties=$discovery_properties_value"
+    else
+      splunk_TA_otel_log_msg "INFO" "STRING PARAM NOT SET: $discovery_properties_name"
+    fi
     if [ "$splunk_access_token_file_value" ] ; then
       export SPLUNK_ACCESS_TOKEN_FILE="$splunk_access_token_file_value"
     else
@@ -410,7 +457,7 @@ splunk_TA_otel_run_agent() {
     LD_PRELOAD="$LD_PRELOAD" \
     LD_LIBRARY_PATH="$LD_LIBRARY_PATH" \
     SPLUNK_OTEL_TA_PLATFORM_HOME="$SPLUNK_OTEL_TA_PLATFORM_HOME" \
-        "$otel_path" > "$splunk_otel_log_file_value" 2>&1 &
+        "$otel_path" "$SPLUNK_OTEL_FLAGS" > "$splunk_otel_log_file_value" 2>&1 &
     splunk_TA_otel_pid="$!"
     is_splunk_TA_otel_agent_running "$splunk_TA_otel_pid"
     if [[ $? == 1 ]] ; then

packaging/technical-addon/Splunk_TA_otel/windows_x86_64/bin/Splunk_TA_otel.cmd

@@ -7,8 +7,14 @@ set "splunk_otel_common_ps_name=Splunk_TA_otel_utils.ps1"
 set "SPLUNK_OTEL_TA_HOME="
 set "SPLUNK_OTEL_TA_PLATFORM_HOME="
 
-
+set "SPLUNK_OTEL_FLAGS="
 :: BEGIN AUTOGENERATED CODE
+set "configd_name=configd"
+set "configd_value="
+set "discovery_name=discovery"
+set "discovery_value="
+set "discovery_properties_name=discovery_properties"
+set "discovery_properties_value="
 set "gomemlimit_name=gomemlimit"
 set "gomemlimit_value="
 set "splunk_api_url_name=splunk_api_url"
@@ -109,11 +115,13 @@ if "%splunk_bundle_dir_value%" == "" (
 if "%splunk_config_value%" == "" (
     call :splunk_TA_otel_log_msg "DEBUG" "Param %splunk_config_name% not set"
 ) else (
+    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --config=%splunk_config_value%"
     set "SPLUNK_CONFIG=%splunk_config_value%"
 )
 if "%splunk_config_dir_value%" == "" (
     call :splunk_TA_otel_log_msg "DEBUG" "Param %splunk_config_dir_name% not set"
 ) else (
+    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --config-dir=%splunk_config_dir_value%"
     set "SPLUNK_CONFIG_DIR=%splunk_config_dir_value%"
 )
 if "%splunk_collectd_dir_value%" == "" (
@@ -176,13 +184,28 @@ if "%splunk_access_token_file_value%" == "" (
 ) else (
     set "SPLUNK_ACCESS_TOKEN_FILE=%splunk_access_token_file_value%"
 )
+if "%configd_value%" == "true" (
+    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --configd"
+) else (
+    call :splunk_TA_otel_log_msg "DEBUG" "Optional flag %configd_name% not set"
+)
+if "%discovery_value%" == "true" (
+    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --discovery"
+) else (
+    call :splunk_TA_otel_log_msg "DEBUG" "Optional flag %discovery_name% not set"
+)
+if "%discovery_properties_value%" != "" (
+    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --discovery-properties=%discovery_properties_value%"
+) else (
+    call :splunk_TA_otel_log_msg "DEBUG" "Optional flag %discovery_properties_name% not set"
+)
 :: END AUTOGENERATED CODE
 
 :: Extract agent bundle
 call :extract_bundle
 
 set "command_line=%splunk_TA_otel_app_directory%%splunk_otel_process_name%"
-start /B "" "%command_line%" > "%splunk_otel_log_file_value%" 2>&1
+start /B "" "%command_line%" "%SPLUNK_OTEL_FLAGS%" > "%splunk_otel_log_file_value%" 2>&1
 set "splunk_otel_common_ps=%splunk_TA_otel_app_directory%%splunk_otel_common_ps_name%"
 start /B "" /I /WAIT powershell "& '%splunk_otel_common_ps%' '%splunk_otel_process_name%' '%splunk_TA_otel_log_file%'"
 
@@ -212,7 +235,12 @@ exit /B 0
 :splunk_TA_otel_read_configs
 echo "INFO grabbing config from stdin..."
 
-for /F "tokens=1,2 delims==" %%I in ('powershell -noninteractive -noprofile -command "$input | Select-String -Pattern '.*?(%gomemlimit_name%|%splunk_api_url_name%|%splunk_bundle_dir_name%|%splunk_config_name%|%splunk_config_dir_name%|%splunk_collectd_dir_name%|%splunk_debug_config_server_name%|%splunk_config_yaml_name%|%splunk_gateway_url_name%|%splunk_hec_url_name%|%splunk_listen_interface_name%|%splunk_memory_limit_mib_name%|%splunk_memory_total_mib_name%|%splunk_otel_log_file_name%|%splunk_ingest_url_name%|%splunk_realm_name%|%splunk_access_token_file_name%|session_key).*?>(.*?)<' | ForEach-Object { $_.Matches.Groups[1].Value + '=' + $_.Matches.Groups[2].Value }"') do (
+
+
+for /F "tokens=1,2 delims==" %%I in ('powershell -noninteractive -noprofile -command "$input | Select-String -Pattern '.*?(%configd_name%|%discovery_name%|%discovery_properties_name%|%gomemlimit_name%|%splunk_access_token_file_name%|%splunk_api_url_name%|%splunk_ballast_size_mib_name%|%splunk_bundle_dir_name%|%splunk_collectd_dir_name%|%splunk_config_name%|%splunk_config_dir_name%|%splunk_config_yaml_name%|%splunk_debug_config_server_name%|%splunk_gateway_url_name%|%splunk_hec_url_name%|%splunk_ingest_url_name%|%splunk_listen_interface_name%|%splunk_memory_limit_mib_name%|%splunk_memory_total_mib_name%|%splunk_otel_log_file_name%|%splunk_realm_name%).*?>(.*?)<' | ForEach-Object { $_.Matches.Groups[1].Value + '=' + $_.Matches.Groups[2].Value }"') do (
+    if "%%I"=="%configd_name%" set "configd_value=%%J"
+    if "%%I"=="%discovery_name%" set "discovery_value=%%J"
+    if "%%I"=="%discovery_properties_name%" set "discovery_properties_value=%%J"
     if "%%I"=="%gomemlimit_name%" set "gomemlimit_value=%%J"
     if "%%I"=="%splunk_api_url_name%" set "splunk_api_url_value=%%J"
     if "%%I"=="%splunk_bundle_dir_name%" set "splunk_bundle_dir_value=%%J"
@@ -339,6 +367,16 @@ for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%splu
     set "splunk_otel_log_file_value=%%i"
 )
 
+:: expand params in discovery_properties_value
+for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%discovery_properties_value%' -replace '\$SPLUNK_OTEL_TA_PLATFORM_HOME', '%SPLUNK_OTEL_TA_PLATFORM_HOME%'"') do (
+    set "discovery_properties_value=%%i"
+)
+for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%discovery_properties_value%' -replace '\$SPLUNK_OTEL_TA_HOME', '%SPLUNK_OTEL_TA_HOME%'"') do (
+    set "discovery_properties_value=%%i"
+)
+for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%discovery_properties_value%' -replace '\$SPLUNK_HOME', '%SPLUNK_HOME%'"') do (
+    set "discovery_properties_value=%%i"
+)
 :: END AUTOGENERATED CODE
 
 :get_access_token