K8s: Resilient Helm config to override SE_DRAIN_AFTER_SESSION_COUNT (#2902)

Signed-off-by: Viet Nguyen Duc <[email protected]>

[skip ci]

Author: VietND96

Makefile

@@ -1004,7 +1004,7 @@ chart_test_autoscaling_disabled:
 chart_test_autoscaling_deployment_https:
 	PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=postgresql TEST_MULTIPLE_VERSIONS=true AUTOSCALING_COOLDOWN_PERIOD=30 SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
 	SECURE_INGRESS_ONLY_DEFAULT=true INGRESS_DISABLE_USE_HTTP2=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_PORT=443 \
-	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=0 MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=1 MAX_SESSIONS_CHROME=1 TEST_NAME_OVERRIDE=true \
+	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=0 MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=1 MAX_SESSIONS_CHROME=1 TEST_NODE_DRAIN_AFTER_SESSION_COUNT=3 TEST_NAME_OVERRIDE=true \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_fullDistributed_basicAuth_secureIngress_defaultCerts_ingressHostName_disableHttp2_autoScaling_patchKEDA_scaledObject_subPath.yaml" \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling
@@ -1013,7 +1013,7 @@ chart_test_autoscaling_deployment:
 	PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_TRACING=true TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) AUTOSCALING_COOLDOWN_PERIOD=30 \
 	TRACING_EXPORTER_ENDPOINT="http://\$$KUBERNETES_NODE_HOST_IP:4317" TEST_CUSTOM_SPECIFIC_NAME=true \
 	SECURE_CONNECTION_SERVER=true SECURE_USE_EXTERNAL_CERT=true SERVICE_TYPE_NODEPORT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=31444 \
-	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 SET_MAX_REPLICAS=3 TEST_DELAY_AFTER_TEST=2 SELENIUM_GRID_MONITORING=false  \
+	SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 SET_MAX_REPLICAS=3 TEST_DELAY_AFTER_TEST=2 TEST_NODE_DRAIN_AFTER_SESSION_COUNT=3 SELENIUM_GRID_MONITORING=false  \
 	VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
 	TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_enableTracing_secureServer_externalCerts_nodePort_autoScaling_scaledObject_existingKEDA_subPath.yaml" \
 	./tests/charts/make/chart_test.sh DeploymentAutoscaling

charts/selenium-grid/CONFIGURATION.md

@@ -50,6 +50,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | global.seleniumGrid.affinity | object | `{}` | Specify affinity for all components, can be overridden individually |
 | global.seleniumGrid.topologySpreadConstraints | list | `[]` | Specify topologySpreadConstraints for all components, can be overridden individually |
 | global.seleniumGrid.nodeMaxSessions | int | `1` | Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled) |
+| global.seleniumGrid.nodeDrainAfterSessionCount | int | `0` | Set number of sessions will be executed in a Node before detaching it from Hub and shutting it down |
 | global.seleniumGrid.nodeEnableManagedDownloads | bool | `true` | This causes the Node to auto manage files downloaded for a given session on the Node (https://www.selenium.dev/documentation/webdriver/drivers/remote_webdriver/#enable-downloads-in-the-grid) |
 | global.seleniumGrid.nodeCustomCapabilities | string | `""` | Setting custom capabilities for matching specific Nodes (https://www.selenium.dev/documentation/grid/configuration/toml_options/#setting-custom-capabilities-for-matching-specific-nodes) |
 | global.seleniumGrid.nodeRegisterPeriod | int | `120` | How long, in seconds, will the Node try to register to the Distributor for the first time. After this period is completed, the Node will not attempt to register again. |
@@ -468,6 +469,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | chromeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for chrome-node container |
 | chromeNode.extraVolumes | list | `[]` | Extra volumes for chrome-node pod |
 | chromeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
+| chromeNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node |
 | chromeNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node |
 | chromeNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level |
 | chromeNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level |
@@ -526,6 +528,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | firefoxNode.extraVolumeMounts | list | `[]` | Extra volume mounts for firefox-node container |
 | firefoxNode.extraVolumes | list | `[]` | Extra volumes for firefox-node pod |
 | firefoxNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
+| firefoxNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node |
 | firefoxNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node |
 | firefoxNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level |
 | firefoxNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level |
@@ -584,6 +587,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | edgeNode.extraVolumeMounts | list | `[]` | Extra volume mounts for edge-node container |
 | edgeNode.extraVolumes | list | `[]` | Extra volumes for edge-node pod |
 | edgeNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
+| edgeNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node |
 | edgeNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node |
 | edgeNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level |
 | edgeNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level |
@@ -643,6 +647,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | relayNode.extraVolumeMounts | list | `[]` | Extra volume mounts for relay-node container |
 | relayNode.extraVolumes | list | `[]` | Extra volumes for relay-node pod |
 | relayNode.nodeMaxSessions | string | `nil` | Override the number of max sessions per node |
+| relayNode.nodeDrainAfterSessionCount | string | `nil` | Override the number of sessions to run before draining the node |
 | relayNode.nodeEnableManagedDownloads | string | `nil` | Override the managed downloads in node |
 | relayNode.nodeCustomCapabilities | string | `""` | Override the same config at the global level |
 | relayNode.nodeRegisterPeriod | string | `nil` | Override the same config at the global level |

charts/selenium-grid/templates/_helpers.tpl

@@ -306,6 +306,7 @@ Common pod template
 {{- $videoImageRegistry := default $.Values.global.seleniumGrid.imageRegistry .recorder.imageRegistry -}}
 {{- $videoImageTag := default $.Values.global.seleniumGrid.videoImageTag .recorder.imageTag -}}
 {{- $nodeMaxSessions := default $.Values.global.seleniumGrid.nodeMaxSessions .node.nodeMaxSessions | int64 -}}
+{{- $nodeDrainAfterSessionCount := default $.Values.global.seleniumGrid.nodeDrainAfterSessionCount .node.nodeDrainAfterSessionCount | int64 -}}
 {{- $nodeEnableManagedDownloads := default $.Values.global.seleniumGrid.nodeEnableManagedDownloads .node.nodeEnableManagedDownloads -}}
 {{- $nodeCustomCapabilities := default $.Values.global.seleniumGrid.nodeCustomCapabilities .node.nodeCustomCapabilities -}}
 {{- $nodeRegisterPeriod := default $.Values.global.seleniumGrid.nodeRegisterPeriod .node.nodeRegisterPeriod | int64 -}}
@@ -379,7 +380,7 @@ template:
           - name: SE_NODE_STEREOTYPE_EXTRA
             value: {{ $nodeCustomCapabilities | quote }}
           - name: SE_DRAIN_AFTER_SESSION_COUNT
-            value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }}
+            value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary (max $nodeMaxSessions $nodeDrainAfterSessionCount) $nodeDrainAfterSessionCount | quote }}
         {{- with .node.relayUrl }}
           - name: SE_NODE_RELAY_URL
             value: {{ . | quote }}

charts/selenium-grid/values.yaml

@@ -52,6 +52,10 @@ global:
     #    Note: If not define labelSelector, it will be added automatically based on "app" label in each component
     # -- Specify number of max sessions per node. Can be overridden by individual component (this is also set to scaler trigger parameter `nodeMaxSessions` if `autoscaling` is enabled)
     nodeMaxSessions: 1
+    # Noted:In case of autoscaling enabled, with scaling type `job`, Node will be drained following `nodeMaxSessions` by default
+    # If changing `nodeDrainAfterSessionCount` > `nodeMaxSessions` it will take precedence over `nodeMaxSessions` in scaling type `job`
+    # -- Set number of sessions will be executed in a Node before detaching it from Hub and shutting it down
+    nodeDrainAfterSessionCount: 0
     # -- This causes the Node to auto manage files downloaded for a given session on the Node (https://www.selenium.dev/documentation/webdriver/drivers/remote_webdriver/#enable-downloads-in-the-grid)
     nodeEnableManagedDownloads: true
     # -- Setting custom capabilities for matching specific Nodes (https://www.selenium.dev/documentation/grid/configuration/toml_options/#setting-custom-capabilities-for-matching-specific-nodes)
@@ -1290,6 +1294,8 @@ chromeNode:
 
   # -- Override the number of max sessions per node
   nodeMaxSessions:
+  # -- Override the number of sessions to run before draining the node
+  nodeDrainAfterSessionCount:
   # -- Override the managed downloads in node
   nodeEnableManagedDownloads:
   # -- Override the same config at the global level
@@ -1486,6 +1492,8 @@ firefoxNode:
 
   # -- Override the number of max sessions per node
   nodeMaxSessions:
+  # -- Override the number of sessions to run before draining the node
+  nodeDrainAfterSessionCount:
   # -- Override the managed downloads in node
   nodeEnableManagedDownloads:
   # -- Override the same config at the global level
@@ -1682,6 +1690,8 @@ edgeNode:
 
   # -- Override the number of max sessions per node
   nodeMaxSessions:
+  # -- Override the number of sessions to run before draining the node
+  nodeDrainAfterSessionCount:
   # -- Override the managed downloads in node
   nodeEnableManagedDownloads:
   # -- Override the same config at the global level
@@ -1879,6 +1889,8 @@ relayNode:
 
   # -- Override the number of max sessions per node
   nodeMaxSessions:
+  # -- Override the number of sessions to run before draining the node
+  nodeDrainAfterSessionCount:
   # -- Override the managed downloads in node
   nodeEnableManagedDownloads:
   # -- Override the same config at the global level

tests/charts/make/chart_test.sh

@@ -54,6 +54,7 @@ else
 fi
 EXTERNAL_TLS_SECRET_NAME=${EXTERNAL_TLS_SECRET_NAME:-"external-tls-secret-${RESOURCE_ID}"}
 SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS:-"true"}
+TEST_NODE_DRAIN_AFTER_SESSION_COUNT=${TEST_NODE_DRAIN_AFTER_SESSION_COUNT:-"0"}
 TEST_NODE_MAX_SESSIONS=${TEST_NODE_MAX_SESSIONS:-"1"}
 MAX_SESSIONS_CHROME=${MAX_SESSIONS_CHROME:-${TEST_NODE_MAX_SESSIONS}}
 MAX_SESSIONS_FIREFOX=${MAX_SESSIONS_FIREFOX:-${TEST_NODE_MAX_SESSIONS}}
@@ -184,6 +185,7 @@ HELM_COMMAND_SET_IMAGES=" \
 --set global.seleniumGrid.httpLogs=${CHART_ENABLE_TRACING} \
 --set isolateComponents=${CHART_FULL_DISTRIBUTED_MODE} \
 --set global.seleniumGrid.logLevel=${LOG_LEVEL} \
+--set global.seleniumGrid.nodeDrainAfterSessionCount=${TEST_NODE_DRAIN_AFTER_SESSION_COUNT} \
 --set chromeNode.nodeMaxSessions=${MAX_SESSIONS_CHROME} \
 --set firefoxNode.nodeMaxSessions=${MAX_SESSIONS_FIREFOX} \
 --set edgeNode.nodeMaxSessions=${MAX_SESSIONS_EDGE} \