Remove ballast extension entirely from components. (#5429)

* Remove ballast extension entirely from components.

* Update CHANGELOG.md

* Remove more references

* Remove more references

Author: atoulme

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### 🛑 Breaking changes 🛑
+
+- (Splunk) Remove deprecated memory ballast extension ([#5429](https://github.com/open-telemetry/opentelemetry-collector/pull/5429))
+
 ### 🚩Deprecations 🚩
 
 - (Splunk) Deprecate the jaegergrpc monitor ([#5428](https://github.com/signalfx/splunk-otel-collector/pull/5428))

cmd/otelcol/config/collector/logs_config_linux.yaml

@@ -7,7 +7,6 @@
 # - SPLUNK_HEC_URL: The Splunk HEC endpoint URL, e.g. https://ingest.us0.signalfx.com/v1/log
 # - SPLUNK_INGEST_URL: The Splunk ingest URL, e.g. https://ingest.us0.signalfx.com
 # - SPLUNK_LISTEN_INTERFACE: The network interface the agent receivers listen on.
-# - SPLUNK_BALLAST_SIZE_MIB: This size of the ballast which should be 1/3 to 1/2 of memory allocated
 # - SPLUNK_MEMORY_LIMIT_MIB: 90% of memory allocated
 
 receivers:

cmd/otelcol/config/collector/upstream_agent_config.yaml

@@ -8,7 +8,6 @@
 # configured below:
 # - SPLUNK_ACCESS_TOKEN: The Splunk access token to authenticate requests
 # - SPLUNK_API_URL: The Splunk API URL, e.g. https://api.us0.signalfx.com
-# - SPLUNK_BALLAST_SIZE_MIB: This size of the ballast which should be 1/3 to 1/2 of memory allocated
 # - SPLUNK_MEMORY_LIMIT_MIB: 90% of memory allocated
 # - SPLUNK_HEC_TOKEN: The Splunk HEC authentication token
 # - SPLUNK_HEC_URL: The Splunk HEC endpoint URL, e.g. https://ingest.us0.signalfx.com/v1/log
@@ -95,7 +94,7 @@ processors:
   # Enabling the memory_limiter is strongly recommended for every pipeline.
   # Configuration is based on the amount of memory allocated to the collector.
   # In general, the limit should be 90% of the collector's memory. The simplest way to specify the
-  # ballast size is set the value of SPLUNK_BALLAST_SIZE_MIB env variable.
+  # memory allocation is set the value of GOMEMLIMIT env variable.
   # For more information about memory limiter, see
   # https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiter/README.md
   memory_limiter:

deployments/chef/templates/splunk-otel-collector.conf.erb

@@ -23,9 +23,7 @@ SPLUNK_HEC_URL=<%= node['splunk_otel_collector']['splunk_hec_url'] %>
 SPLUNK_HEC_TOKEN=<%= node['splunk_otel_collector']['splunk_hec_token'] %>
 
 # Total memory in MIB to allocate to the collector.
-# Automatically configures the ballast and memory limit.
-# If `SPLUNK_BALLAST_SIZE_MIB` is also defined, it will override
-# the value calculated by `SPLUNK_MEMORY_TOTAL_MIB`.
+# Automatically configures the memory limit.
 SPLUNK_MEMORY_TOTAL_MIB=<%= node['splunk_otel_collector']['splunk_memory_total_mib'] %>
 
 <% unless node['splunk_otel_collector']['splunk_listen_interface'].to_s.strip.empty? -%>

deployments/chef/test/integration/default/test.rb

@@ -29,6 +29,7 @@
     { name: 'SPLUNK_REALM', type: :string, data: splunk_realm },
     { name: 'SPLUNK_TRACE_URL', type: :string, data: splunk_trace_url },
   ]
+
   collector_env_vars_strings = []
   collector_env_vars.each do |item|
     collector_env_vars_strings |= [ "#{item[:name]}=#{item[:data]}" ]

deployments/puppet/README.md

@@ -14,14 +14,6 @@
         'SPLUNK_TRACE_URL' => $splunk_otel_collector::splunk_trace_url,
     }
 
-    $ballast_size_mib = if $splunk_otel_collector::collector_version != 'latest' and
-        versioncmp($splunk_otel_collector::collector_version, '0.97.0') < 0 and
-        !$splunk_otel_collector::splunk_ballast_size_mib.strip().empty() {
-            { 'SPLUNK_BALLAST_SIZE_MIB' => $splunk_otel_collector::splunk_ballast_size_mib }
-        } else {
-            {}
-        }
-
     $gomemlimit = if ($splunk_otel_collector::collector_version == 'latest' or
         versioncmp($splunk_otel_collector::collector_version, '0.97.0') >= 0) and
         !$splunk_otel_collector::gomemlimit.strip().empty() {
@@ -36,5 +28,5 @@
             {}
         }
 
-    $collector_env_vars = stdlib::merge($base_env_vars, $ballast_size_mib, $gomemlimit, $listen_interface)
+    $collector_env_vars = stdlib::merge($base_env_vars, $gomemlimit, $listen_interface)
 }

deployments/puppet/manifests/collector_win_config_options.pp

@@ -10,7 +10,6 @@
   $splunk_bundle_dir       = $splunk_otel_collector::params::splunk_bundle_dir,
   $splunk_collectd_dir     = $splunk_otel_collector::params::splunk_collectd_dir,
   $splunk_memory_total_mib = '512',
-  $splunk_ballast_size_mib = '',
   $splunk_listen_interface = '',
   $collector_version       = $splunk_otel_collector::params::collector_version,
   $collector_config_source = $splunk_otel_collector::params::collector_config_source,

deployments/puppet/manifests/init.pp

@@ -1,6 +1,5 @@
 SPLUNK_ACCESS_TOKEN=<%= @splunk_access_token %>
 SPLUNK_API_URL=<%= @splunk_api_url %>
-SPLUNK_BALLAST_SIZE_MIB=<%= @splunk_ballast_size_mib %>
 SPLUNK_BUNDLE_DIR=<%= @splunk_bundle_dir %>
 SPLUNK_COLLECTD_DIR=<%= @splunk_collectd_dir %>
 SPLUNK_CONFIG=<%= @collector_config_dest %>

deployments/puppet/templates/splunk-otel-collector.conf.erb

@@ -140,7 +140,6 @@ The distribution offers support for the following components.
 | [host_observer](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/observer/hostobserver)        | [beta]    |
 | [http_forwarder](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/httpforwarderextension)      | [beta]    |
 | [k8s_observer](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/observer/k8sobserver)          | [beta]    |
-| [memory_ballast](https://github.com/open-telemetry/opentelemetry-collector/tree/main/extension/ballastextension)                    | [beta]    |
 | [oauth2client](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/oauth2clientauthextension)     | [beta]    |
 | [pprof](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/pprofextension)                       | [beta]    |
 | [smartagent](../pkg/extension/smartagentextension)                                                                                  | [beta]    |

docs/components.md

@@ -145,7 +145,6 @@ require (
 	go.opentelemetry.io/collector/exporter/otlpexporter v0.110.0
 	go.opentelemetry.io/collector/exporter/otlphttpexporter v0.110.0
 	go.opentelemetry.io/collector/extension v0.110.0
-	go.opentelemetry.io/collector/extension/ballastextension v0.108.1
 	go.opentelemetry.io/collector/extension/zpagesextension v0.110.0
 	go.opentelemetry.io/collector/otelcol v0.110.0
 	go.opentelemetry.io/collector/pdata v1.16.0

go.mod

@@ -1933,8 +1933,6 @@ go.opentelemetry.io/collector/extension v0.110.0 h1:AYFk57W25f7xOo3I6pV0rWNWVtOL
 go.opentelemetry.io/collector/extension v0.110.0/go.mod h1:zD/pw9o83SFyn/DCbBdBcH0eUPyGtYgpMSAOqotFYRc=
 go.opentelemetry.io/collector/extension/auth v0.110.0 h1:9SHC2sF/KR99LciHABDXRIsXLiujzIjTJpWHO0V8Bqg=
 go.opentelemetry.io/collector/extension/auth v0.110.0/go.mod h1:NjpHds6mjeT8Zn2KJVxZtV9c59AoIr6RlBha1RpmScQ=
-go.opentelemetry.io/collector/extension/ballastextension v0.108.1 h1:HmuvkGN2s5hnviVZ62CRM7XbGOWcVjlp3L6VOZEvaTc=
-go.opentelemetry.io/collector/extension/ballastextension v0.108.1/go.mod h1:Yu2jtjK7RD8Ie13N06rQMtGbLqWySmPEPEZoJdS9DT4=
 go.opentelemetry.io/collector/extension/experimental/storage v0.110.0 h1:G1xkNGiBkdSrdhhU5VLE9+y7sZ5fU1/CHps92KSYDLc=
 go.opentelemetry.io/collector/extension/experimental/storage v0.110.0/go.mod h1:0XFrIUcbqjsSycNI6Vu7ndMnjSkglMnD2YtUl2ZrzIU=
 go.opentelemetry.io/collector/extension/extensioncapabilities v0.110.0 h1:VVqGHUh1A5ljEoQVeMCMUs1i7oLMKRQRpLCAim4WwpU=

go.sum

@@ -47,14 +47,9 @@ SPLUNK_HEC_URL=https://ingest.us0.signalfx.com/v1/log
 SPLUNK_HEC_TOKEN=12345
 
 # Total memory in MIB to allocate to the collector.
-# Automatically configures the ballast and memory limit.
-# If `SPLUNK_BALLAST_SIZE_MIB` is also defined, it will override
-# the value calculated by `SPLUNK_MEMORY_TOTAL_MIB`.
+# Automatically configures the memory limit.
 SPLUNK_MEMORY_TOTAL_MIB=1024
 
-# How much memory to allocate to the ballast. This should be set to 1/3 to 1/2 of configured memory.
-# SPLUNK_BALLAST_SIZE_MIB=683
-
 # The path to the Smart Agent bundle.
 SPLUNK_BUNDLE_DIR=/usr/lib/splunk-otel-collector/agent-bundle
 

internal/buildscripts/packaging/fpm/etc/otel/collector/splunk-otel-collector.conf.example

@@ -902,8 +902,6 @@ Collector:
   --api-url <url>                       Set the api endpoint URL explicitly instead of the endpoint inferred from the
                                         specified realm.
                                         (default: https://api.REALM.signalfx.com)
-  --ballast <ballast size>              Set the ballast size explicitly instead of the value calculated from the
-                                        '--memory' option. This should be set to 1/3 to 1/2 of configured memory.
   --beta                                Use the beta package repo instead of the primary.
   --collector-config <path>             Set the path to an existing custom config file for the collector service instead
                                         of the default config file provided by the collector package based on the
@@ -923,8 +921,7 @@ Collector:
   --ingest-url <url>                    Set the ingest endpoint URL explicitly instead of the endpoint inferred from the
                                         specified realm.
                                         (default: https://ingest.REALM.signalfx.com)
-  --memory <memory size>                Total memory in MIB to allocate to the collector; automatically calculates the
-                                        ballast size.
+  --memory <memory size>                Total memory in MIB to allocate to the collector
                                         (default: "$default_memory_size")
   --mode <agent|gateway>                Configure the collector service to run in agent or gateway mode.
                                         (default: "agent")
@@ -1236,7 +1233,6 @@ check_support() {
 parse_args_and_install() {
   local access_token=
   local api_url=
-  local ballast=
   local collector_version="$default_collector_version"
   local hec_token=
   local hec_url=
@@ -1272,10 +1268,6 @@ parse_args_and_install() {
         api_url="$2"
         shift 1
         ;;
-      --ballast)
-        ballast="$2"
-        shift 1
-        ;;
       --beta)
         stage="beta"
         ;;
@@ -1585,9 +1577,6 @@ parse_args_and_install() {
   ensure_not_installed "$with_fluentd" "$with_instrumentation" "$with_systemd_instrumentation" "$npm_path"
 
   echo "Splunk OpenTelemetry Collector Version: ${collector_version}"
-  if [ -n "$ballast" ]; then
-    echo "Ballast Size in MIB: $ballast"
-  fi
   echo "Memory Size in MIB: $memory"
 
   if [ -n "$listen_interface" ]; then
@@ -1742,9 +1731,6 @@ parse_args_and_install() {
   configure_env_file "SPLUNK_HEC_URL" "$hec_url" "$collector_env_path"
   configure_env_file "SPLUNK_HEC_TOKEN" "$hec_token" "$collector_env_path"
   configure_env_file "SPLUNK_MEMORY_TOTAL_MIB" "$memory" "$collector_env_path"
-  if [ -n "$ballast" ]; then
-    configure_env_file "SPLUNK_BALLAST_SIZE_MIB" "$ballast" "$collector_env_path"
-  fi
   if [ -d "$collector_bundle_dir" ]; then
     configure_env_file "SPLUNK_BUNDLE_DIR" "$collector_bundle_dir" "$collector_env_path"
     # ensure the collector service owner has access to the bundle dir

internal/buildscripts/packaging/installer/install.sh

@@ -23,8 +23,6 @@ gomemlimit_name="gomemlimit"
 gomemlimit_value=""
 splunk_api_url_name="splunk_api_url"
 splunk_api_url_value=""
-splunk_ballast_size_mib_name="splunk_ballast_size_mib"
-splunk_ballast_size_mib_value=""
 splunk_bundle_dir_name="splunk_bundle_dir"
 splunk_bundle_dir_value=""
 splunk_config_name="splunk_config"
@@ -130,13 +128,6 @@ splunk_TA_otel_read_configs() {
             splunk_TA_otel_log_msg "INFO" "Set $splunk_api_url_name to $splunk_api_url_value"
         fi
 
-        has_splunk_ballast_size_mib="$(echo "$line" | grep "$splunk_ballast_size_mib_name")"
-        if [ "$has_splunk_ballast_size_mib" ] ; then
-            splunk_TA_otel_log_msg "DEBUG" "reading $splunk_ballast_size_mib_name from line $has_splunk_ballast_size_mib"
-            splunk_ballast_size_mib_value="$(echo "$has_splunk_ballast_size_mib" | grep -Eo ">(.*?)<" | sed 's/^>\(.*\)<$/\1/')"
-            splunk_TA_otel_log_msg "INFO" "Set $splunk_ballast_size_mib_name to $splunk_ballast_size_mib_value"
-        fi
-
         has_splunk_bundle_dir="$(echo "$line" | grep "$splunk_bundle_dir_name")"
         if [ "$has_splunk_bundle_dir" ] ; then
             splunk_TA_otel_log_msg "DEBUG" "reading $splunk_bundle_dir_name from line $has_splunk_bundle_dir"
@@ -318,11 +309,6 @@ splunk_TA_otel_run_agent() {
     else
       splunk_TA_otel_log_msg "INFO" "NOT SET: $splunk_api_url_name"
     fi
-    if [ "$splunk_ballast_size_mib_value" ] ; then
-      export SPLUNK_BALLAST_SIZE_MIB="$splunk_ballast_size_mib_value"
-    else
-      splunk_TA_otel_log_msg "INFO" "NOT SET: $splunk_ballast_size_mib_name"
-    fi
     if [ "$splunk_bundle_dir_value" ] ; then
       export SPLUNK_BUNDLE_DIR="$splunk_bundle_dir_value"
     else

internal/buildscripts/packaging/technical-addon/Splunk_TA_otel/linux_x86_64/bin/Splunk_TA_otel.sh

@@ -13,8 +13,6 @@ set "gomemlimit_name=gomemlimit"
 set "gomemlimit_value="
 set "splunk_api_url_name=splunk_api_url"
 set "splunk_api_url_value="
-set "splunk_ballast_size_mib_name=splunk_ballast_size_mib"
-set "splunk_ballast_size_mib_value="
 set "splunk_bundle_dir_name=splunk_bundle_dir"
 set "splunk_bundle_dir_value="
 set "splunk_config_name=splunk_config"
@@ -105,11 +103,6 @@ if "%splunk_api_url_value%" == "" (
 ) else (
     set "SPLUNK_API_URL=%splunk_api_url_value%"
 )
-if "%splunk_ballast_size_mib_value%" == "" (
-    call :splunk_TA_otel_log_msg "DEBUG" "Param %splunk_ballast_size_mib_name% not set"
-) else (
-    set "SPLUNK_BALLAST_SIZE_MIB=%splunk_ballast_size_mib_value%"
-)
 if "%splunk_bundle_dir_value%" == "" (
     call :splunk_TA_otel_log_msg "DEBUG" "Param %splunk_bundle_dir_name% not set"
 ) else (
@@ -226,10 +219,9 @@ exit /B 0
 :splunk_TA_otel_read_configs
 echo "INFO grabbing config from stdin..."
 
-for /F "tokens=1,2 delims==" %%I in ('powershell -noninteractive -noprofile -command "$input | Select-String -Pattern '.*?(%gomemlimit_name%|%splunk_api_url_name%|%splunk_ballast_size_mib_name%|%splunk_bundle_dir_name%|%splunk_config_name%|%splunk_config_dir_name%|%splunk_collectd_dir_name%|%splunk_debug_config_server_name%|%splunk_config_yaml_name%|%splunk_gateway_url_name%|%splunk_hec_url_name%|%splunk_listen_interface_name%|%splunk_memory_limit_mib_name%|%splunk_memory_total_mib_name%|%splunk_trace_url_name%|%splunk_otel_log_file_name%|%splunk_ingest_url_name%|%splunk_realm_name%|%splunk_access_token_file_name%|session_key).*?>(.*?)<' | ForEach-Object { $_.Matches.Groups[1].Value + '=' + $_.Matches.Groups[2].Value }"') do (
+for /F "tokens=1,2 delims==" %%I in ('powershell -noninteractive -noprofile -command "$input | Select-String -Pattern '.*?(%gomemlimit_name%|%splunk_api_url_name%|%splunk_bundle_dir_name%|%splunk_config_name%|%splunk_config_dir_name%|%splunk_collectd_dir_name%|%splunk_debug_config_server_name%|%splunk_config_yaml_name%|%splunk_gateway_url_name%|%splunk_hec_url_name%|%splunk_listen_interface_name%|%splunk_memory_limit_mib_name%|%splunk_memory_total_mib_name%|%splunk_trace_url_name%|%splunk_otel_log_file_name%|%splunk_ingest_url_name%|%splunk_realm_name%|%splunk_access_token_file_name%|session_key).*?>(.*?)<' | ForEach-Object { $_.Matches.Groups[1].Value + '=' + $_.Matches.Groups[2].Value }"') do (
     if "%%I"=="%gomemlimit_name%" set "gomemlimit_value=%%J"
     if "%%I"=="%splunk_api_url_name%" set "splunk_api_url_value=%%J"
-    if "%%I"=="%splunk_ballast_size_mib_name%" set "splunk_ballast_size_mib_value=%%J"
     if "%%I"=="%splunk_bundle_dir_name%" set "splunk_bundle_dir_value=%%J"
     if "%%I"=="%splunk_config_name%" set "splunk_config_value=%%J"
     if "%%I"=="%splunk_config_dir_name%" set "splunk_config_dir_value=%%J"

internal/buildscripts/packaging/technical-addon/Splunk_TA_otel/windows_x86_64/bin/Splunk_TA_otel.cmd

@@ -133,7 +133,7 @@ def verify_config_file(container, path, key, value, exists=True):
         assert not match, f"'{line}' found in {path}:\n{config}"
 
 
-def verify_env_file(container, mode="agent", config_path=None, memory=TOTAL_MEMORY, listen_addr="", ballast=None):
+def verify_env_file(container, mode="agent", config_path=None, memory=TOTAL_MEMORY, listen_addr=""):
     env_path = SPLUNK_ENV_PATH
     if container_file_exists(container, OLD_SPLUNK_ENV_PATH):
         env_path = OLD_SPLUNK_ENV_PATH
@@ -162,9 +162,6 @@ def verify_env_file(container, mode="agent", config_path=None, memory=TOTAL_MEMO
     else:
         verify_config_file(container, env_path, "SPLUNK_LISTEN_INTERFACE", False, False)
 
-    if ballast:
-        verify_config_file(container, env_path, "SPLUNK_BALLAST_SIZE_MIB", ballast)
-
 
 def verify_support_bundle(container):
     run_container_cmd(container, "/etc/otel/collector/splunk-support-bundle.sh -t /tmp/splunk-support-bundle")
@@ -280,7 +277,6 @@ def test_installer_custom(distro, arch):
         "--with-fluentd",
         "--listen-interface 10.0.0.1",
         "--memory 256",
-        "--ballast 64",
         f"--service-user {service_owner} --service-group {service_owner}",
         f"--collector-config {custom_config}",
         f"--collector-version {collector_version}",
@@ -301,7 +297,7 @@ def test_installer_custom(distro, arch):
             assert output.decode("utf-8").strip() == f"otelcol version v{collector_version}"
 
             # verify env file created with configured parameters
-            verify_env_file(container, config_path=custom_config, memory="256", listen_addr="10.0.0.1", ballast="64")
+            verify_env_file(container, config_path=custom_config, memory="256", listen_addr="10.0.0.1")
 
             # verify collector service status
             assert wait_for(lambda: service_is_running(container, service_owner=service_owner))

internal/buildscripts/packaging/tests/installer_test.py

@@ -119,7 +119,6 @@ import (
 	"go.opentelemetry.io/collector/exporter/otlpexporter"
 	"go.opentelemetry.io/collector/exporter/otlphttpexporter"
 	"go.opentelemetry.io/collector/extension"
-	"go.opentelemetry.io/collector/extension/ballastextension"
 	"go.opentelemetry.io/collector/extension/zpagesextension"
 	"go.opentelemetry.io/collector/otelcol"
 	"go.opentelemetry.io/collector/processor"
@@ -144,7 +143,6 @@ func Get() (otelcol.Factories, error) {
 	var errs []error
 	extensions, err := extension.MakeFactoryMap(
 		ackextension.NewFactory(),
-		ballastextension.NewFactory(),
 		basicauthextension.NewFactory(),
 		dockerobserver.NewFactory(),
 		ecsobserver.NewFactory(),

internal/components/components.go

@@ -37,7 +37,6 @@ func TestDefaultComponents(t *testing.T) {
 		"host_observer",
 		"http_forwarder",
 		"k8s_observer",
-		"memory_ballast",
 		"oauth2client",
 		"pprof",
 		"smartagent",

internal/components/components_test.go

@@ -15,7 +15,6 @@ receivers:
 
 processors:
   memory_limiter:
-    ballast_size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     check_interval: 2s
     limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}
 

internal/configconverter/testdata/logging_loglevel-after.yaml

@@ -15,7 +15,6 @@ receivers:
 
 processors:
   memory_limiter:
-    ballast_size_mib: ${SPLUNK_BALLAST_SIZE_MIB}
     check_interval: 2s
     limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB}