diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b8cc30..4ff93f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- BREAKING: Adds listener support for HBase ([#639]). - Adds new telemetry CLI arguments and environment variables ([#652]). - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. @@ -39,6 +40,7 @@ - test: HBase 2.4.18 removed ([#659]): - Remove operator support for HBase 2.4 including the JMX exporter ([#672]). +[#639]: https://github.com/stackabletech/hbase-operator/pull/639 [#640]: https://github.com/stackabletech/hbase-operator/pull/640 [#645]: https://github.com/stackabletech/hbase-operator/pull/645 [#647]: https://github.com/stackabletech/hbase-operator/pull/647 diff --git a/deploy/helm/hbase-operator/crds/crds.yaml b/deploy/helm/hbase-operator/crds/crds.yaml index 6294fe40..eee702ec 100644 --- a/deploy/helm/hbase-operator/crds/crds.yaml +++ b/deploy/helm/hbase-operator/crds/crds.yaml @@ -73,20 +73,6 @@ spec: hdfsConfigMapName: description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for an HDFS cluster. type: string - listenerClass: - default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this HbaseCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - enum: - - cluster-internal - - external-unstable - type: string vectorAggregatorConfigMapName: description: Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. nullable: true @@ -210,6 +196,10 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + nullable: true + type: string logging: default: containers: {} @@ -460,6 +450,10 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + nullable: true + type: string logging: default: containers: {} @@ -691,6 +685,10 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + nullable: true + type: string logging: default: containers: {} @@ -969,6 +967,10 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + nullable: true + type: string logging: default: containers: {} @@ -1228,6 +1230,10 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + nullable: true + type: string logging: default: containers: {} @@ -1478,6 +1484,10 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + nullable: true + type: string logging: default: containers: {} diff --git a/deploy/helm/hbase-operator/templates/roles.yaml b/deploy/helm/hbase-operator/templates/roles.yaml index f13d450b..7bc43bdb 100644 --- a/deploy/helm/hbase-operator/templates/roles.yaml +++ b/deploy/helm/hbase-operator/templates/roles.yaml @@ -77,6 +77,12 @@ rules: verbs: - create - patch + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/docs/modules/hbase/pages/usage-guide/listenerclass.adoc b/docs/modules/hbase/pages/usage-guide/listenerclass.adoc index 1f6d48b8..8300aac9 100644 --- a/docs/modules/hbase/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/hbase/pages/usage-guide/listenerclass.adoc @@ -1,18 +1,31 @@ = Service exposition with ListenerClasses +:description: Configure HBase service exposure using ListenerClasses to control internal and external access for all roles. -Apache HBase offers an API. -The operator deploys a service called `` (where `` is the name of the HbaseCluster) through which HBase can be reached. - -This service can have either the `cluster-internal` or `external-unstable` type. -`external-stable` is not supported for HBase at the moment. -Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. - -This is how the listener class is configured: +The operator deploys a xref:listener-operator:listener.adoc[Listener] for each Master, Regionserver and Restserver pod. +They all default to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.{masters,regionServers,restServers}.config.listenerClass`: [source,yaml] ---- spec: - clusterConfig: - listenerClass: cluster-internal # <1> + masters: + config: + listenerClass: external-unstable # <1> + regionServers: + config: + listenerClass: external-unstable + restServers: + config: + listenerClass: external-unstable +---- +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). +This can be set separately for all roles. + +Listener endpoints are written to `hbase-site.xml` like this: + +[source,xml] +---- + + hbase.listener.endpoint + 172.19.0.3:32445 + ---- -<1> The default `cluster-internal` setting. diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b3483f3c..a7e22190 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -59,8 +59,6 @@ pub const SSL_CLIENT_XML: &str = "ssl-client.xml"; pub const HBASE_CLUSTER_DISTRIBUTED: &str = "hbase.cluster.distributed"; pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; -pub const HBASE_UNSAFE_REGIONSERVER_HOSTNAME_DISABLE_MASTER_REVERSEDNS: &str = - "hbase.unsafe.regionserver.hostname.disable.master.reversedns"; pub const HBASE_UI_PORT_NAME_HTTP: &str = "ui-http"; pub const HBASE_UI_PORT_NAME_HTTPS: &str = "ui-https"; @@ -76,10 +74,14 @@ pub const HBASE_REGIONSERVER_PORT: u16 = 16020; pub const HBASE_REGIONSERVER_UI_PORT: u16 = 16030; pub const HBASE_REST_PORT: u16 = 8080; pub const HBASE_REST_UI_PORT: u16 = 8085; +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; const DEFAULT_REGION_MOVER_TIMEOUT: Duration = Duration::from_minutes_unchecked(59); const DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN: Duration = Duration::from_minutes_unchecked(1); +const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("the role [{role}] is invalid and does not exist in HBase"))] @@ -102,6 +104,12 @@ pub enum Error { #[snafu(display("incompatible merge types"))] IncompatibleMergeTypes, + + #[snafu(display("role-group is not valid"))] + NoRoleGroup, + + #[snafu(display("role-group not found by name"))] + RoleGroupNotFound, } #[versioned(version(name = "v1alpha1"))] @@ -171,18 +179,6 @@ pub mod versioned { /// for a ZooKeeper cluster. pub zookeeper_config_map_name: String, - /// This field controls which type of Service the Operator creates for this HbaseCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, - /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/usage-guide/security). pub authentication: Option, @@ -212,6 +208,11 @@ impl v1alpha1::HbaseCluster { let defaults = AnyConfigFragment::default_for(role, &self.name_any(), hdfs_discovery_cm_name); + // Trivial values for role-groups are not allowed + if role_group.is_empty() { + return Err(Error::NoRoleGroup); + } + let (mut role_config, mut role_group_config) = match role { HbaseRole::RegionServer => { let role = self @@ -227,7 +228,7 @@ impl v1alpha1::HbaseCluster { .role_groups .get(role_group) .map(|rg| rg.config.config.clone()) - .unwrap_or_default(); + .context(RoleGroupNotFoundSnafu)?; ( AnyConfigFragment::RegionServer(role_config), @@ -249,7 +250,7 @@ impl v1alpha1::HbaseCluster { .role_groups .get(role_group) .map(|rg| rg.config.config.clone()) - .unwrap_or_default(); + .context(RoleGroupNotFoundSnafu)?; // Retrieve role resource config ( @@ -269,7 +270,7 @@ impl v1alpha1::HbaseCluster { .role_groups .get(role_group) .map(|rg| rg.config.config.clone()) - .unwrap_or_default(); + .context(RoleGroupNotFoundSnafu)?; // Retrieve role resource config ( @@ -526,7 +527,7 @@ impl v1alpha1::HbaseCluster { } /// Name of the port used by the Web UI, which depends on HTTPS usage - fn ui_port_name(&self) -> String { + pub fn ui_port_name(&self) -> String { if self.has_https_enabled() { HBASE_UI_PORT_NAME_HTTPS } else { @@ -552,27 +553,6 @@ pub fn merged_env(rolegroup_config: Option<&BTreeMap>) -> Vec String { - match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - } - } -} - #[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KerberosConfig { @@ -696,6 +676,7 @@ impl HbaseRole { affinity: get_affinity(cluster_name, self, hdfs_discovery_cm_name), graceful_shutdown_timeout: Some(graceful_shutdown_timeout), requested_secret_lifetime: Some(requested_secret_lifetime), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), } } @@ -796,6 +777,7 @@ impl AnyConfigFragment { cli_opts: None, }, requested_secret_lifetime: Some(HbaseRole::DEFAULT_REGION_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), }) } HbaseRole::RestServer => AnyConfigFragment::RestServer(HbaseConfigFragment { @@ -807,6 +789,7 @@ impl AnyConfigFragment { HbaseRole::DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, ), requested_secret_lifetime: Some(HbaseRole::DEFAULT_REST_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), }), HbaseRole::Master => AnyConfigFragment::Master(HbaseConfigFragment { hbase_rootdir: None, @@ -817,6 +800,7 @@ impl AnyConfigFragment { HbaseRole::DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, ), requested_secret_lifetime: Some(HbaseRole::DEFAULT_MASTER_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), }), } } @@ -894,6 +878,9 @@ pub struct HbaseConfig { /// Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. #[fragment_attrs(serde(default))] pub requested_secret_lifetime: Option, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose this rolegroup. + pub listener_class: String, } impl Configuration for HbaseConfigFragment { @@ -952,10 +939,6 @@ impl Configuration for HbaseConfigFragment { HBASE_CLUSTER_DISTRIBUTED.to_string(), Some("true".to_string()), ); - result.insert( - HBASE_UNSAFE_REGIONSERVER_HOSTNAME_DISABLE_MASTER_REVERSEDNS.to_string(), - Some("true".to_string()), - ); result.insert(HBASE_ROOTDIR.to_string(), self.hbase_rootdir.clone()); } _ => {} @@ -1047,6 +1030,9 @@ pub struct RegionServerConfig { /// The operator will compute a timeout period for the region move that will not exceed the graceful shutdown timeout. #[fragment_attrs(serde(default))] pub region_mover: RegionMover, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose this rolegroup. + pub listener_class: String, } impl Configuration for RegionServerConfigFragment { @@ -1103,10 +1089,6 @@ impl Configuration for RegionServerConfigFragment { HBASE_CLUSTER_DISTRIBUTED.to_string(), Some("true".to_string()), ); - result.insert( - HBASE_UNSAFE_REGIONSERVER_HOSTNAME_DISABLE_MASTER_REVERSEDNS.to_string(), - Some("true".to_string()), - ); result.insert(HBASE_ROOTDIR.to_string(), self.hbase_rootdir.clone()); } _ => {} @@ -1172,6 +1154,14 @@ impl AnyServiceConfig { } } + pub fn listener_class(&self) -> String { + match self { + AnyServiceConfig::Master(config) => config.listener_class.clone(), + AnyServiceConfig::RegionServer(config) => config.listener_class.clone(), + AnyServiceConfig::RestServer(config) => config.listener_class.clone(), + } + } + /// Returns command line arguments to pass on to the region mover tool. /// The following arguments are excluded because they are already part of the /// hbase-entrypoint.sh script. diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 33758dd1..772b9ef2 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -8,6 +8,7 @@ use std::{ }; use const_format::concatcp; +use indoc::formatdoc; use product_config::{ ProductConfigManager, types::PropertyNameKind, @@ -20,8 +21,14 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, + }, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -70,9 +77,11 @@ use crate::{ construct_role_specific_non_heap_jvm_args, }, crd::{ - APP_NAME, AnyServiceConfig, Container, HBASE_ENV_SH, HBASE_REST_PORT_NAME_HTTP, - HBASE_REST_PORT_NAME_HTTPS, HBASE_SITE_XML, HbaseClusterStatus, HbaseRole, - JVM_SECURITY_PROPERTIES_FILE, SSL_CLIENT_XML, SSL_SERVER_XML, merged_env, v1alpha1, + APP_NAME, AnyServiceConfig, Container, HBASE_ENV_SH, HBASE_MASTER_PORT, + HBASE_MASTER_UI_PORT, HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, + HBASE_REST_PORT_NAME_HTTP, HBASE_REST_PORT_NAME_HTTPS, HBASE_SITE_XML, HbaseClusterStatus, + HbaseRole, JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, + SSL_CLIENT_XML, SSL_SERVER_XML, merged_env, v1alpha1, }, discovery::build_discovery_configmap, kerberos::{ @@ -102,6 +111,9 @@ const HBASE_LOG_CONFIG_TMP_DIR: &str = "/stackable/tmp/log_config"; const DOCKER_IMAGE_BASE_NAME: &str = "hbase"; +const HBASE_MASTER_PORT_NAME: &str = "master"; +const HBASE_REGIONSERVER_PORT_NAME: &str = "regionserver"; + pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, @@ -131,9 +143,6 @@ pub enum Error { #[snafu(display("object defines no regionserver role"))] NoRegionServerRole, - #[snafu(display("failed to calculate global service name"))] - GlobalServiceNameNotFound, - #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -144,11 +153,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply global Service"))] - ApplyRoleService { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to apply Service for {}", rolegroup))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, @@ -312,6 +316,16 @@ pub enum Error { #[snafu(display("failed to construct JVM arguments"))] ConstructJvmArgument { source: crate::config::jvm::Error }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, } type Result = std::result::Result; @@ -373,26 +387,6 @@ pub async fn reconcile_hbase( ) .context(CreateClusterResourcesSnafu)?; - let region_server_role_service = - build_region_server_role_service(hbase, &resolved_product_image)?; - cluster_resources - .add(client, region_server_role_service) - .await - .context(ApplyRoleServiceSnafu)?; - - // discovery config map - let discovery_cm = build_discovery_configmap( - hbase, - &client.kubernetes_cluster_info, - &zookeeper_connection_information, - &resolved_product_image, - ) - .context(BuildDiscoveryConfigMapSnafu)?; - cluster_resources - .add(client, discovery_cm) - .await - .context(ApplyDiscoveryConfigMapSnafu)?; - let (rbac_sa, rbac_rolebinding) = build_rbac_resources( hbase, APP_NAME, @@ -441,7 +435,6 @@ pub async fn reconcile_hbase( )?; let rg_statefulset = build_rolegroup_statefulset( hbase, - &client.kubernetes_cluster_info, &hbase_role, &rolegroup, rolegroup_config, @@ -482,6 +475,20 @@ pub async fn reconcile_hbase( } } + // Discovery CM will fail to build until the rest of the cluster has been deployed, so do it last + // so that failure won't inhibit the rest of the cluster from booting up. + let discovery_cm = build_discovery_configmap( + hbase, + &client.kubernetes_cluster_info, + &zookeeper_connection_information, + &resolved_product_image, + ) + .context(BuildDiscoveryConfigMapSnafu)?; + cluster_resources + .add(client, discovery_cm) + .await + .context(ApplyDiscoveryConfigMapSnafu)?; + let cluster_operation_cond_builder = ClusterOperationsConditionBuilder::new(&hbase.spec.cluster_operation); @@ -501,59 +508,6 @@ pub async fn reconcile_hbase( Ok(Action::await_change()) } -/// The server-role service is the primary endpoint that should be used by clients that do not perform internal load balancing, -/// including targets outside of the cluster. -pub fn build_region_server_role_service( - hbase: &v1alpha1::HbaseCluster, - resolved_product_image: &ResolvedProductImage, -) -> Result { - let role = HbaseRole::RegionServer; - let role_name = role.to_string(); - let role_svc_name = hbase - .server_role_service_name() - .context(GlobalServiceNameNotFoundSnafu)?; - let ports = hbase - .ports(&role) - .into_iter() - .map(|(name, value)| ServicePort { - name: Some(name), - port: i32::from(value), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }) - .collect(); - - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .name(&role_svc_name) - .ownerreference_from_resource(hbase, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hbase, - &resolved_product_image.app_version_label, - &role_name, - "global", - )) - .context(ObjectMetaSnafu)? - .build(); - - let service_selector_labels = - Labels::role_selector(hbase, APP_NAME, &role_name).context(BuildLabelSnafu)?; - - let service_spec = ServiceSpec { - type_: Some(hbase.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(ports), - selector: Some(service_selector_labels.into()), - ..ServiceSpec::default() - }; - - Ok(Service { - metadata, - spec: Some(service_spec), - status: None, - }) -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( @@ -588,6 +542,84 @@ fn build_rolegroup_config_map( hbase_site_config .extend(hbase_opa_config.map_or(vec![], |config| config.hbase_site_config())); + // Get more useful stack traces... + // The default Netty impl gives us netty garbage and nothing else + hbase_site_config.insert( + "hbase.rpc.client.impl".to_string(), + "org.apache.hadoop.hbase.ipc.BlockingRpcClient".to_string(), + ); + + // Set listener endpoint information + hbase_site_config.insert( + "hbase.listener.endpoint".to_string(), + "${HBASE_LISTENER_ENDPOINT}".to_string(), + ); + + // Set flag to override default behaviour, which is that the + // RPC client should bind the client address (forcing outgoing + // RPC traffic to happen from the same network interface that + // the RPC server is bound on). + hbase_site_config.insert( + "hbase.client.rpc.bind.address".to_string(), + "false".to_string(), + ); + + match hbase_role { + HbaseRole::Master => { + hbase_site_config.insert( + "hbase.master.ipc.address".to_string(), + "0.0.0.0".to_string(), + ); + hbase_site_config.insert( + "hbase.master.ipc.port".to_string(), + HBASE_MASTER_PORT.to_string(), + ); + hbase_site_config.insert( + "hbase.master.hostname".to_string(), + "${HBASE_SERVICE_HOST}".to_string(), + ); + hbase_site_config.insert( + "hbase.master.port".to_string(), + "${HBASE_SERVICE_PORT}".to_string(), + ); + hbase_site_config.insert( + "hbase.master.info.port".to_string(), + "${HBASE_INFO_PORT}".to_string(), + ); + hbase_site_config.insert( + "hbase.master.bound.info.port".to_string(), + HBASE_MASTER_UI_PORT.to_string(), + ); + } + HbaseRole::RegionServer => { + hbase_site_config.insert( + "hbase.regionserver.ipc.address".to_string(), + "0.0.0.0".to_string(), + ); + hbase_site_config.insert( + "hbase.regionserver.ipc.port".to_string(), + HBASE_REGIONSERVER_PORT.to_string(), + ); + hbase_site_config.insert( + "hbase.unsafe.regionserver.hostname".to_string(), + "${HBASE_SERVICE_HOST}".to_string(), + ); + hbase_site_config.insert( + "hbase.regionserver.port".to_string(), + "${HBASE_SERVICE_PORT}".to_string(), + ); + hbase_site_config.insert( + "hbase.regionserver.info.port".to_string(), + "${HBASE_INFO_PORT}".to_string(), + ); + hbase_site_config.insert( + "hbase.regionserver.bound.info.port".to_string(), + HBASE_REGIONSERVER_UI_PORT.to_string(), + ); + } + HbaseRole::RestServer => {} + }; + // configOverride come last hbase_site_config.extend(config.clone()); hbase_site_xml = to_hadoop_xml( @@ -721,7 +753,7 @@ fn build_rolegroup_service( let metadata = ObjectMetaBuilder::new() .name_and_namespace(hbase) - .name(rolegroup.object_name()) + .name(headless_service_name(&rolegroup.object_name())) .ownerreference_from_resource(hbase, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -761,7 +793,6 @@ fn build_rolegroup_service( #[allow(clippy::too_many_arguments)] fn build_rolegroup_statefulset( hbase: &v1alpha1::HbaseCluster, - cluster_info: &KubernetesClusterInfo, hbase_role: &HbaseRole, rolegroup_ref: &RoleGroupRef, rolegroup_config: &HashMap>, @@ -785,14 +816,14 @@ fn build_rolegroup_statefulset( let probe_template = match hbase_role { HbaseRole::Master => Probe { tcp_socket: Some(TCPSocketAction { - port: IntOrString::String("master".to_string()), + port: IntOrString::String(HBASE_MASTER_PORT_NAME.to_string()), ..TCPSocketAction::default() }), ..Probe::default() }, HbaseRole::RegionServer => Probe { tcp_socket: Some(TCPSocketAction { - port: IntOrString::String("regionserver".to_string()), + port: IntOrString::String(HBASE_REGIONSERVER_PORT_NAME.to_string()), ..TCPSocketAction::default() }), ..Probe::default() @@ -856,15 +887,30 @@ fn build_rolegroup_statefulset( }, ]); + let role_name = hbase_role.cli_role_name(); let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); + + let rest_http_port_name = if hbase.has_https_enabled() { + HBASE_REST_PORT_NAME_HTTPS + } else { + HBASE_REST_PORT_NAME_HTTP + }; + hbase_container .image_from_product_image(resolved_product_image) - .command(vec!["/stackable/hbase/bin/hbase-entrypoint.sh".to_string()]) - .args(vec![ - hbase_role.cli_role_name(), - hbase_service_domain_name(hbase, rolegroup_ref, cluster_info)?, - hbase.service_port(hbase_role).to_string(), - ]) + .command(command()) + .args(vec![formatdoc! {" + {entrypoint} {role} {port} {port_name} {ui_port_name}", + entrypoint = "/stackable/hbase/bin/hbase-entrypoint.sh".to_string(), + role = role_name, + port = hbase.service_port(hbase_role).to_string(), + port_name = match hbase_role { + HbaseRole::Master => HBASE_MASTER_PORT_NAME, + HbaseRole::RegionServer => HBASE_REGIONSERVER_PORT_NAME, + HbaseRole::RestServer => rest_http_port_name, + }, + ui_port_name = hbase.ui_port_name(), + }]) .add_env_vars(merged_env) // Needed for the `containerdebug` process to log it's tracing information to. .add_env_var( @@ -879,6 +925,8 @@ fn build_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount("log", STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? .add_container_ports(ports) .resources(merged_config.resources().clone().into()) .startup_probe(startup_probe) @@ -887,13 +935,17 @@ fn build_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); + let recommended_object_labels = build_recommended_labels( + hbase, + hbase_version, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + let recommended_labels = + Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; + let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(build_recommended_labels( - hbase, - hbase_version, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(ObjectMetaSnafu)? .build(); @@ -929,6 +981,15 @@ fn build_rolegroup_statefulset( .service_account_name(service_account.name_any()) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + // externally-reachable listener endpoints should use a pvc volume... + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(merged_config.listener_class().to_string()), + &recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { @@ -1031,8 +1092,9 @@ fn build_rolegroup_statefulset( match_labels: Some(statefulset_match_labels.into()), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.object_name()), + service_name: Some(headless_service_name(&rolegroup_ref.object_name())), template: pod_template, + volume_claim_templates: Some(vec![pvc]), ..StatefulSetSpec::default() }; @@ -1043,6 +1105,17 @@ fn build_rolegroup_statefulset( }) } +/// Returns the container command. +fn command() -> Vec { + vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ] +} + fn write_hbase_env_sh<'a, T>(properties: T) -> String where T: Iterator, @@ -1104,6 +1177,7 @@ fn build_hbase_env_sh( let role_specific_non_heap_jvm_args = construct_role_specific_non_heap_jvm_args(hbase, hbase_role, role_group) .context(ConstructJvmArgumentSnafu)?; + match hbase_role { HbaseRole::Master => { result.insert( @@ -1128,26 +1202,8 @@ fn build_hbase_env_sh( Ok(result) } -/// Build the domain name of an HBase service pod. -/// The hbase-entrypoint.sh script uses this to build the fully qualified name of a pod -/// by appending it to the `HOSTNAME` environment variable. -/// This name is required by the RegionMover to function properly. -fn hbase_service_domain_name( - hbase: &v1alpha1::HbaseCluster, - rolegroup_ref: &RoleGroupRef, - cluster_info: &KubernetesClusterInfo, -) -> Result { - let hbase_cluster_name = rolegroup_ref.object_name(); - let pod_namespace = hbase - .metadata - .namespace - .clone() - .context(ObjectHasNoNamespaceSnafu)?; - let cluster_domain = &cluster_info.cluster_domain; - - Ok(format!( - "{hbase_cluster_name}.{pod_namespace}.svc.{cluster_domain}" - )) +fn headless_service_name(role_group_name: &str) -> String { + format!("{name}-metrics", name = role_group_name) } #[cfg(test)] diff --git a/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 index 4732f740..6989c892 100644 --- a/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 @@ -29,6 +29,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -37,6 +38,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -45,6 +47,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 index ffc8c7c0..0431cf88 100644 --- a/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 @@ -32,6 +32,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -40,6 +41,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -48,6 +50,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 2 # ignored because reconciliation is paused diff --git a/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 index 0f4c5665..8bc4007f 100644 --- a/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 @@ -32,6 +32,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -40,6 +41,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -48,6 +50,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 # set to 0 by the operator because cluster is stopped diff --git a/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 index 388110b2..9a29aff5 100644 --- a/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 @@ -32,6 +32,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -40,6 +41,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -48,6 +50,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/external-access/00-assert.yaml.j2 b/tests/templates/kuttl/external-access/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/external-access/01-assert.yaml b/tests/templates/kuttl/external-access/01-assert.yaml new file mode 100644 index 00000000..e0766c49 --- /dev/null +++ b/tests/templates/kuttl/external-access/01-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-zk-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/01-install-zookeeper.yaml.j2 b/tests/templates/kuttl/external-access/01-install-zookeeper.yaml.j2 new file mode 100644 index 00000000..0a331d50 --- /dev/null +++ b/tests/templates/kuttl/external-access/01-install-zookeeper.yaml.j2 @@ -0,0 +1,29 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: test-zk +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: test-znode +spec: + clusterRef: + name: test-zk diff --git a/tests/templates/kuttl/external-access/02-assert.yaml b/tests/templates/kuttl/external-access/02-assert.yaml new file mode 100644 index 00000000..99b25f8e --- /dev/null +++ b/tests/templates/kuttl/external-access/02-assert.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hdfs-namenode-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hdfs-journalnode-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hdfs-datanode-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/02-install-hdfs.yaml.j2 b/tests/templates/kuttl/external-access/02-install-hdfs.yaml.j2 new file mode 100644 index 00000000..f9194a60 --- /dev/null +++ b/tests/templates/kuttl/external-access/02-install-hdfs.yaml.j2 @@ -0,0 +1,39 @@ +--- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: test-hdfs +spec: + image: + productVersion: "{{ test_scenario['values']['hdfs-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: + dfsReplication: 1 + zookeeperConfigMapName: test-znode +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nameNodes: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 2 + dataNodes: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + journalNodes: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/external-access/10-listener-classes.yaml b/tests/templates/kuttl/external-access/10-listener-classes.yaml new file mode 100644 index 00000000..893032c5 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-listener-classes.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/20-assert.yaml.j2 b/tests/templates/kuttl/external-access/20-assert.yaml.j2 new file mode 100644 index 00000000..223cc784 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-assert.yaml.j2 @@ -0,0 +1,96 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available hbaseclusters.hbase.stackable.tech/hbase --timeout 301s +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-hbase +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-master-external-unstable +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-regionserver-external-unstable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-regionserver-external-stable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-restserver-external-unstable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-restserver-external-stable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: listener-test-hbase-master-external-unstable-0 +spec: + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: listener-test-hbase-master-external-unstable-1 +spec: + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: listener-test-hbase-regionserver-external-stable-0 +spec: + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: listener-test-hbase-regionserver-external-unstable-0 +spec: + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: listener-test-hbase-restserver-external-stable-0 +spec: + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: listener-test-hbase-restserver-external-unstable-0 +spec: + type: NodePort diff --git a/tests/templates/kuttl/external-access/20-install-hbase.yaml b/tests/templates/kuttl/external-access/20-install-hbase.yaml new file mode 100644 index 00000000..843b488d --- /dev/null +++ b/tests/templates/kuttl/external-access/20-install-hbase.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: > + envsubst < install-hbase.yaml | + kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/install-hbase.yaml.j2 b/tests/templates/kuttl/external-access/install-hbase.yaml.j2 new file mode 100644 index 00000000..690577b1 --- /dev/null +++ b/tests/templates/kuttl/external-access/install-hbase.yaml.j2 @@ -0,0 +1,55 @@ +--- +apiVersion: hbase.stackable.tech/v1alpha1 +kind: HbaseCluster +metadata: + name: test-hbase +spec: + image: +{% if test_scenario['values']['hbase'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hbase'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hbase'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hbase'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + hdfsConfigMapName: test-hdfs-namenode-default + zookeeperConfigMapName: test-znode +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + masters: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: test-external-unstable-$NAMESPACE + roleGroups: + external-unstable: + replicas: 2 + regionServers: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: test-external-unstable-$NAMESPACE + roleGroups: + external-unstable: + replicas: 1 + external-stable: + replicas: 1 + config: + listenerClass: test-external-stable-$NAMESPACE + restServers: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: test-external-stable-$NAMESPACE + roleGroups: + external-stable: + replicas: 1 + external-unstable: + replicas: 1 + config: + listenerClass: test-external-unstable-$NAMESPACE diff --git a/tests/templates/kuttl/external-access/listener-classes.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml new file mode 100644 index 00000000..4131526a --- /dev/null +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal-$NAMESPACE +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable-$NAMESPACE +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 index 28766a48..a21046d1 100644 --- a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 @@ -28,7 +28,6 @@ commands: clusterConfig: hdfsConfigMapName: hdfs zookeeperConfigMapName: hbase-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: tlsSecretClass: tls kerberos: @@ -41,6 +40,7 @@ commands: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} resources: memory: limit: 1536Mi @@ -52,6 +52,7 @@ commands: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 2 @@ -60,6 +61,7 @@ commands: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml index 5b29af5a..73515e9c 100644 --- a/tests/templates/kuttl/kerberos/42-test-rest-server.yaml +++ b/tests/templates/kuttl/kerberos/42-test-rest-server.yaml @@ -14,7 +14,7 @@ spec: - /bin/bash - -c - | - status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default:8080") + status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default-metrics:8080") if [[ "$status_code" -eq 401 ]] ; then echo "[PASS] Successfully got 401 as we did not authenticate" diff --git a/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 b/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 index fea35545..5751bbae 100644 --- a/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 @@ -45,7 +45,6 @@ commands: clusterConfig: hdfsConfigMapName: hdfs zookeeperConfigMapName: hbase-znode - listenerClass: 'cluster-internal' authentication: tlsSecretClass: tls kerberos: diff --git a/tests/templates/kuttl/opa/42-test-rest-server.yaml b/tests/templates/kuttl/opa/42-test-rest-server.yaml index 5b29af5a..73515e9c 100644 --- a/tests/templates/kuttl/opa/42-test-rest-server.yaml +++ b/tests/templates/kuttl/opa/42-test-rest-server.yaml @@ -14,7 +14,7 @@ spec: - /bin/bash - -c - | - status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default:8080") + status_code=$(curl --write-out '%{http_code}' --silent --insecure --output /dev/null "https://hbase-restserver-default-metrics:8080") if [[ "$status_code" -eq 401 ]] ; then echo "[PASS] Successfully got 401 as we did not authenticate" diff --git a/tests/templates/kuttl/orphaned_resources/05-assert.yaml b/tests/templates/kuttl/orphaned_resources/05-assert.yaml index bfeeddc9..3b21f998 100644 --- a/tests/templates/kuttl/orphaned_resources/05-assert.yaml +++ b/tests/templates/kuttl/orphaned_resources/05-assert.yaml @@ -23,4 +23,4 @@ metadata: apiVersion: v1 kind: Service metadata: - name: test-hbase-regionserver-newrolegroup + name: test-hbase-regionserver-newrolegroup-metrics diff --git a/tests/templates/kuttl/profiling/run-profiler.py b/tests/templates/kuttl/profiling/run-profiler.py index 7afe0d71..d5edc7d5 100644 --- a/tests/templates/kuttl/profiling/run-profiler.py +++ b/tests/templates/kuttl/profiling/run-profiler.py @@ -8,8 +8,7 @@ def start_profiling_and_get_refresh_header(service_url): prof_page = requests.get( - f"{service_url}/prof" - f"?event={EVENT_TYPE}&duration={PROFILING_DURATION_IN_SEC}" + f"{service_url}/prof?event={EVENT_TYPE}&duration={PROFILING_DURATION_IN_SEC}" ) assert prof_page.ok, f"""Profiling could not be started. @@ -56,9 +55,7 @@ def fetch_flamegraph(service_url, refresh_path): def test_profiling(role, port): - service_url = ( - f"http://test-hbase-{role}-default-0.test-hbase-{role}-default" f":{port}" - ) + service_url = f"http://test-hbase-{role}-default-metrics:{port}" print(f"Test profiling on {service_url}") diff --git a/tests/templates/kuttl/shutdown/30-install-hbase.yaml.j2 b/tests/templates/kuttl/shutdown/30-install-hbase.yaml.j2 index 4ef58b80..e05aa96b 100644 --- a/tests/templates/kuttl/shutdown/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/shutdown/30-install-hbase.yaml.j2 @@ -15,7 +15,6 @@ spec: clusterConfig: hdfsConfigMapName: test-hdfs zookeeperConfigMapName: test-znode - listenerClass: "cluster-internal" {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 index 53e9a98e..7535a3e8 100644 --- a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 @@ -15,7 +15,6 @@ spec: clusterConfig: hdfsConfigMapName: test-hdfs zookeeperConfigMapName: test-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -23,6 +22,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: configOverrides: @@ -34,6 +34,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: configOverrides: @@ -45,6 +46,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} resources: memory: limit: 1Gi diff --git a/tests/templates/kuttl/smoke/31-assert.yaml b/tests/templates/kuttl/smoke/31-assert.yaml index 25f259d3..d41d277e 100644 --- a/tests/templates/kuttl/smoke/31-assert.yaml +++ b/tests/templates/kuttl/smoke/31-assert.yaml @@ -4,6 +4,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-master-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"' - - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-regionserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"' - - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-restserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"' + - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-master-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-regionserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-restserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/50-assert.yaml b/tests/templates/kuttl/smoke/50-assert.yaml index 5ae54c4c..0fab6554 100644 --- a/tests/templates/kuttl/smoke/50-assert.yaml +++ b/tests/templates/kuttl/smoke/50-assert.yaml @@ -4,6 +4,6 @@ kind: TestAssert metadata: name: test-hbase commands: - - script: kubectl exec --namespace=$NAMESPACE hbase-test-runner-0 -- python /tmp/test-hbase.py http://test-hbase-restserver-default:8080 + - script: kubectl exec --namespace=$NAMESPACE hbase-test-runner-0 -- python /tmp/test-hbase.py http://test-hbase-restserver-default-metrics:8080 - script: kubectl exec --namespace=$NAMESPACE hbase-test-runner-0 -- python /tmp/test_prometheus_metrics.py $NAMESPACE timeout: 240 diff --git a/tests/templates/kuttl/smoke/test_prometheus_metrics.py b/tests/templates/kuttl/smoke/test_prometheus_metrics.py index 7e8c36ab..401dbc74 100644 --- a/tests/templates/kuttl/smoke/test_prometheus_metrics.py +++ b/tests/templates/kuttl/smoke/test_prometheus_metrics.py @@ -10,7 +10,7 @@ def check_metrics( namespace: str, role: str, port: int, expected_metrics: list[str] ) -> None: response: requests.Response = requests.get( - f"http://test-hbase-{role}-default-0.test-hbase-{role}-default.{namespace}.svc.cluster.local:{port}/prometheus", + f"http://test-hbase-{role}-default-metrics:{port}/prometheus", timeout=10, ) assert response.ok, "Requesting metrics failed" diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 240a53f8..e4ec1e40 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -137,6 +137,12 @@ tests: - hdfs-latest - zookeeper-latest - openshift + - name: external-access + dimensions: + - hbase + - hdfs-latest + - zookeeper-latest + - openshift suites: - name: nightly patch: