From 8e817005052937a54d1a84a0225a9d9d0e98aec6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 2 May 2024 15:06:43 +0200 Subject: [PATCH 1/7] fix: move metastore db credentials into a secret --- deploy/helm/hive-operator/crds/crds.yaml | 12 ++---- rust/crd/src/affinity.rs | 1 + rust/crd/src/lib.rs | 21 ++++++---- rust/operator-binary/src/command.rs | 10 ++++- rust/operator-binary/src/controller.rs | 40 +++++++++++++++++-- .../cluster-operation/10-install-hive.yaml.j2 | 12 +++++- .../cluster-operation/20-stop-hive.yaml.j2 | 3 +- .../cluster-operation/30-pause-hive.yaml.j2 | 3 +- .../cluster-operation/40-restart-hive.yaml.j2 | 3 +- .../kerberos-hdfs/60-install-hive.yaml.j2 | 12 +++++- .../kuttl/kerberos-s3/60-install-hive.yaml.j2 | 12 +++++- .../kuttl/logging/03-install-hive.yaml.j2 | 12 +++++- .../01-install-hive.yaml.j2 | 12 +++++- .../kuttl/resources/10-install-hive.yaml.j2 | 12 +++++- .../kuttl/smoke/60-install-hive.yaml.j2 | 12 +++++- 15 files changed, 136 insertions(+), 41 deletions(-) diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index 458dc1ff..862779fc 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -50,6 +50,9 @@ spec: connString: description: 'A connection string for the database. For example: `jdbc:postgresql://hivehdfs-postgresql:5432/hivehdfs`' type: string + credentialsSecret: + description: A reference to a Secret containing the database credentials. The Secret needs to contain the keys `username` and `password`. + type: string dbType: description: 'The type of database to connect to. Supported are: `postgres`, `mysql`, `oracle`, `mssql` and `derby`. This value is used to configure the jdbc driver class.' enum: @@ -59,17 +62,10 @@ spec: - oracle - mssql type: string - password: - description: The password for the database user. - type: string - user: - description: The database user. - type: string required: - connString + - credentialsSecret - dbType - - password - - user type: object hdfs: description: HDFS connection specification. diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index a61be85e..f98396fa 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -52,6 +52,7 @@ mod tests { user: APP password: mine dbType: derby + credentialsSecret: mySecret metastore: roleGroups: default: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 22dc4463..3e443daf 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -72,6 +72,12 @@ pub const HADOOP_OPTS: &str = "HADOOP_OPTS"; pub const HADOOP_HEAPSIZE: &str = "HADOOP_HEAPSIZE"; pub const JVM_HEAP_FACTOR: f32 = 0.8; +// DB credentials +pub const DB_USERNAME_PLACEHOLDER: &str = "xxx_db_username_xxx"; +pub const DB_PASSWORD_PLACEHOLDER: &str = "xxx_db_password_xxx"; +pub const DB_USERNAME_ENV: &str = "DB_USERNAME_ENV"; +pub const DB_PASSWORD_ENV: &str = "DB_PASSWORD_ENV"; + const DEFAULT_METASTORE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); #[derive(Snafu, Debug)] @@ -423,16 +429,14 @@ pub struct DatabaseConnectionSpec { /// `jdbc:postgresql://hivehdfs-postgresql:5432/hivehdfs` pub conn_string: String, - /// The database user. - pub user: String, - - /// The password for the database user. - pub password: String, - /// The type of database to connect to. Supported are: /// `postgres`, `mysql`, `oracle`, `mssql` and `derby`. /// This value is used to configure the jdbc driver class. pub db_type: DbType, + + /// A reference to a Secret containing the database credentials. + /// The Secret needs to contain the keys `username` and `password`. + pub credentials_secret: String, } impl Configuration for MetaStoreConfigFragment { @@ -493,13 +497,14 @@ impl Configuration for MetaStoreConfigFragment { MetaStoreConfig::CONNECTION_URL.to_string(), Some(hive.spec.cluster_config.database.conn_string.clone()), ); + // use a placeholder that will be replaced in the start command (also for the password) result.insert( MetaStoreConfig::CONNECTION_USER_NAME.to_string(), - Some(hive.spec.cluster_config.database.user.clone()), + Some(DB_USERNAME_PLACEHOLDER.into()), ); result.insert( MetaStoreConfig::CONNECTION_PASSWORD.to_string(), - Some(hive.spec.cluster_config.database.password.clone()), + Some(DB_PASSWORD_PLACEHOLDER.into()), ); result.insert( MetaStoreConfig::CONNECTION_DRIVER_NAME.to_string(), diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs index 9b80f120..76766503 100644 --- a/rust/operator-binary/src/command.rs +++ b/rust/operator-binary/src/command.rs @@ -1,5 +1,6 @@ use stackable_hive_crd::{ - HiveCluster, HIVE_METASTORE_LOG4J2_PROPERTIES, HIVE_SITE_XML, STACKABLE_CONFIG_DIR, + HiveCluster, DB_PASSWORD_ENV, DB_PASSWORD_PLACEHOLDER, DB_USERNAME_ENV, + DB_USERNAME_PLACEHOLDER, HIVE_METASTORE_LOG4J2_PROPERTIES, HIVE_SITE_XML, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, SYSTEM_TRUST_STORE, SYSTEM_TRUST_STORE_PASSWORD, }; @@ -59,6 +60,13 @@ pub fn build_container_command_args( } } + // db credentials + args.extend([ + format!("echo replacing {DB_USERNAME_PLACEHOLDER} and {DB_PASSWORD_PLACEHOLDER} with secret values."), + format!("sed -i \"s|{DB_USERNAME_PLACEHOLDER}|${DB_USERNAME_ENV}|g\" {STACKABLE_CONFIG_DIR}/{HIVE_SITE_XML}"), + format!("sed -i \"s|{DB_PASSWORD_PLACEHOLDER}|${DB_PASSWORD_ENV}|g\" {STACKABLE_CONFIG_DIR}/{HIVE_SITE_XML}"), + ]); + // metastore start command args.push(start_command); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d1e7da89..41d6e935 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -17,13 +17,14 @@ use product_config::{ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hive_crd::{ Container, DbType, HiveCluster, HiveClusterStatus, HiveRole, MetaStoreConfig, APP_NAME, - CERTS_DIR, CORE_SITE_XML, HADOOP_HEAPSIZE, HIVE_ENV_SH, HIVE_PORT, HIVE_PORT_NAME, - HIVE_SITE_XML, JVM_HEAP_FACTOR, JVM_SECURITY_PROPERTIES_FILE, METRICS_PORT, METRICS_PORT_NAME, - STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, - STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, + CERTS_DIR, CORE_SITE_XML, DB_PASSWORD_ENV, DB_USERNAME_ENV, HADOOP_HEAPSIZE, HIVE_ENV_SH, + HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, JVM_HEAP_FACTOR, JVM_SECURITY_PROPERTIES_FILE, + METRICS_PORT, METRICS_PORT_NAME, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_DIR_NAME, + STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, STACKABLE_LOG_DIR_NAME, }; +use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, SecretKeySelector}; use stackable_operator::{ builder::{ resources::ResourceRequirementsBuilder, ConfigMapBuilder, ContainerBuilder, @@ -830,6 +831,22 @@ fn build_metastore_rolegroup_statefulset( } } + // load database credentials to environment variables: these will be used to replace + // the placeholders in hive-site.xml so that the operator does not "touch" the secret. + let credentials_secret_name = hive.spec.cluster_config.database.credentials_secret.clone(); + + let mut env: Vec = vec![env_var_from_secret( + DB_USERNAME_ENV, + &credentials_secret_name, + "username", + )]; + env.push(env_var_from_secret( + DB_PASSWORD_ENV, + &credentials_secret_name, + "password", + )); + container_builder.add_env_vars(env); + let mut pod_builder = PodBuilder::new(); if let Some(hdfs) = &hive.spec.cluster_config.hdfs { @@ -1089,6 +1106,21 @@ fn build_metastore_rolegroup_statefulset( }) } +fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> EnvVar { + EnvVar { + name: String::from(var_name), + value_from: Some(EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: Some(String::from(secret)), + key: String::from(secret_key), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } +} + pub fn error_policy(_obj: Arc, _error: &Error, _ctx: Arc) -> Action { Action::requeue(*Duration::from_secs(5)) } diff --git a/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 index 09469f54..1636328e 100644 --- a/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 @@ -15,9 +15,8 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine dbType: derby + credentialsSecret: hive-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -28,3 +27,12 @@ spec: roleGroups: default: replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine diff --git a/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 index 29a0e890..9bb428d7 100644 --- a/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-stop-hive.yaml.j2 @@ -15,8 +15,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery diff --git a/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 index 2af06bec..17001faf 100644 --- a/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-pause-hive.yaml.j2 @@ -15,8 +15,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery diff --git a/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 index 9187ec14..ac5669a9 100644 --- a/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/40-restart-hive.yaml.j2 @@ -10,8 +10,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery diff --git a/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 b/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 index 4b4bffaf..1a47cb0c 100644 --- a/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-hdfs/60-install-hive.yaml.j2 @@ -21,8 +21,7 @@ commands: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres hdfs: configMap: hdfs @@ -40,3 +39,12 @@ commands: default: replicas: 1 EOF +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 b/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 index 2a2bc65d..5dc77ede 100644 --- a/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/kerberos-s3/60-install-hive.yaml.j2 @@ -21,8 +21,7 @@ commands: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres s3: reference: minio @@ -78,3 +77,12 @@ metadata: stringData: accessKey: hive secretKey: hivehive +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/tests/templates/kuttl/logging/03-install-hive.yaml.j2 b/tests/templates/kuttl/logging/03-install-hive.yaml.j2 index fb58e715..396e340c 100644 --- a/tests/templates/kuttl/logging/03-install-hive.yaml.j2 +++ b/tests/templates/kuttl/logging/03-install-hive.yaml.j2 @@ -49,8 +49,7 @@ spec: clusterConfig: database: connString: jdbc:postgresql://hive-postgresql:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres vectorAggregatorConfigMapName: hive-vector-aggregator-discovery metastore: @@ -88,3 +87,12 @@ spec: hive: custom: configMap: hive-log-config +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 b/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 index 315ece5a..cd8b1df0 100644 --- a/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 @@ -15,8 +15,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery @@ -30,3 +29,12 @@ spec: replicas: 1 remove: replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine diff --git a/tests/templates/kuttl/resources/10-install-hive.yaml.j2 b/tests/templates/kuttl/resources/10-install-hive.yaml.j2 index 2aebe0c8..a985772d 100644 --- a/tests/templates/kuttl/resources/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/resources/10-install-hive.yaml.j2 @@ -15,8 +15,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery @@ -53,3 +52,12 @@ spec: cpu: 500m limits: cpu: 3100m +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine diff --git a/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 b/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 index 7629fcfc..e6c8c077 100644 --- a/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 @@ -15,9 +15,8 @@ spec: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive dbType: postgres + credentialsSecret: hive-credentials s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %} @@ -68,3 +67,12 @@ metadata: stringData: accessKey: hive secretKey: hivehive +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive From 18efeb2cd5513238e5ea47378d25eda618eaee41 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 2 May 2024 15:12:22 +0200 Subject: [PATCH 2/7] changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7de71faa..7fcaf4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ All notable changes to this project will be documented in this file. - Added documentation/tutorial on using external database drivers ([#449]). +### Fixed + +- Move the metastore DB credentials out fo the CRD into a secret ([#452]). + ### Changed - BREAKING: Switch to new image that only contains HMS. @@ -17,6 +21,7 @@ All notable changes to this project will be documented in this file. [#447]: https://github.com/stackabletech/hive-operator/pull/447 [#449]: https://github.com/stackabletech/hive-operator/pull/449 +[#452]: https://github.com/stackabletech/hive-operator/pull/452 ## [24.3.0] - 2024-03-20 From 92f0226146884fb918a61c9d9a390190143a2a1e Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 2 May 2024 15:34:23 +0200 Subject: [PATCH 3/7] changed references in documentation/examples --- .../getting_started/hive-postgres-s3.yaml | 12 +++++-- .../getting_started/hive-postgres-s3.yaml.j2 | 12 +++++-- .../hive/pages/reference/discovery.adoc | 12 +++++-- .../pages/usage-guide/database-driver.adoc | 14 ++++++-- .../hive/pages/usage-guide/derby-example.adoc | 36 +++++++++++++++---- examples/simple-hive-cluster-postgres-s3.yaml | 12 +++++-- examples/simple-hive-cluster.yaml | 12 +++++-- rust/crd/src/affinity.rs | 2 -- .../03-remove-role-group.yaml.j2 | 3 +- .../04-change-rolegroup.yaml | 3 +- 10 files changed, 93 insertions(+), 25 deletions(-) diff --git a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml index 7c27c9c2..b578b2a5 100644 --- a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml +++ b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml @@ -9,8 +9,7 @@ spec: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres s3: reference: minio @@ -18,3 +17,12 @@ spec: roleGroups: default: replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 index 7c27c9c2..b578b2a5 100644 --- a/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 +++ b/docs/modules/hive/examples/getting_started/hive-postgres-s3.yaml.j2 @@ -9,8 +9,7 @@ spec: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres s3: reference: minio @@ -18,3 +17,12 @@ spec: roleGroups: default: replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/docs/modules/hive/pages/reference/discovery.adoc b/docs/modules/hive/pages/reference/discovery.adoc index 88441cb3..64da5d6f 100644 --- a/docs/modules/hive/pages/reference/discovery.adoc +++ b/docs/modules/hive/pages/reference/discovery.adoc @@ -25,13 +25,21 @@ spec: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres metastore: roleGroups: default: # <3> replicas: 2 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive ---- <1> The name of the Hive cluster, which is also the name of the created discovery ConfigMap. <2> The namespace of the discovery ConfigMap. diff --git a/docs/modules/hive/pages/usage-guide/database-driver.adoc b/docs/modules/hive/pages/usage-guide/database-driver.adoc index cc1f0d6e..336a108c 100644 --- a/docs/modules/hive/pages/usage-guide/database-driver.adoc +++ b/docs/modules/hive/pages/usage-guide/database-driver.adoc @@ -145,8 +145,7 @@ spec: clusterConfig: database: connString: jdbc:mysql://mysql:3306/hive # <1> - user: hive # <2> - password: hive + credentialsSecret: hive-credentials # <2> dbType: mysql s3: reference: minio # <3> @@ -167,10 +166,19 @@ spec: persistentVolumeClaim: claimName: pvc-hive-drivers replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials # <2> +type: Opaque +stringData: + username: hive + password: hive ---- <1> The database connection details matching those given when deploying the MySQL Helm chart -<2> Plain-text Hive credentials will be replaced in an upcoming release! +<2> Hive credentials are retrieved from a Secret <3> A reference to the file store using S3 (this has been omitted from this article for the sake of brevity, but is described in e.g. the xref:getting_started/first_steps.adoc[] guide) <4> Use `envOverrides` to set the driver path <5> Use `podOverrides` to mount the driver diff --git a/docs/modules/hive/pages/usage-guide/derby-example.adoc b/docs/modules/hive/pages/usage-guide/derby-example.adoc index 2d27296c..5e795afb 100644 --- a/docs/modules/hive/pages/usage-guide/derby-example.adoc +++ b/docs/modules/hive/pages/usage-guide/derby-example.adoc @@ -20,13 +20,21 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/metastore_db;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby metastore: roleGroups: default: replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine ---- WARNING: You should not use the `Derby` database in production. Derby stores data locally which does not work in high availability setups (multiple replicas) and all data is lost after Pod restarts. @@ -62,8 +70,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/stackable/metastore_db;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby s3: inline: @@ -96,6 +103,15 @@ metadata: stringData: accessKey: minio-access-key secretKey: minio-secret-key +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine ---- @@ -131,12 +147,20 @@ spec: clusterConfig: database: connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive - user: hive - password: hive + credentialsSecret: hive-credentials dbType: postgres metastore: roleGroups: default: replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: hive + password: hive ---- diff --git a/examples/simple-hive-cluster-postgres-s3.yaml b/examples/simple-hive-cluster-postgres-s3.yaml index d68facf0..60e7039e 100644 --- a/examples/simple-hive-cluster-postgres-s3.yaml +++ b/examples/simple-hive-cluster-postgres-s3.yaml @@ -22,8 +22,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby s3: inline: @@ -56,3 +55,12 @@ metadata: stringData: accessKey: minio-access-key secretKey: minio-secret-key +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine diff --git a/examples/simple-hive-cluster.yaml b/examples/simple-hive-cluster.yaml index f0f9dc2e..9c9655c1 100644 --- a/examples/simple-hive-cluster.yaml +++ b/examples/simple-hive-cluster.yaml @@ -10,8 +10,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby metastore: roleGroups: @@ -24,3 +23,12 @@ spec: max: "2" memory: limit: 5Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: hive-credentials +type: Opaque +stringData: + username: APP + password: mine diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs index f98396fa..cbe50c4a 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/crd/src/affinity.rs @@ -49,8 +49,6 @@ mod tests { clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine dbType: derby credentialsSecret: mySecret metastore: diff --git a/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 b/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 index e9bd1df6..19ae7250 100644 --- a/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/03-remove-role-group.yaml.j2 @@ -15,8 +15,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby metastore: roleGroups: diff --git a/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml b/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml index 8e70c973..669d5592 100644 --- a/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml +++ b/tests/templates/kuttl/orphaned-resources/04-change-rolegroup.yaml @@ -7,8 +7,7 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - user: APP - password: mine + credentialsSecret: hive-credentials dbType: derby metastore: roleGroups: From 4fb5729aa8f78cdb6b97f2dcc820e4027b99b90d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 7 May 2024 12:11:38 +0200 Subject: [PATCH 4/7] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fcaf4d9..08380ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file. ### Fixed -- Move the metastore DB credentials out fo the CRD into a secret ([#452]). +- [BREAKING] Move the metastore DB credentials out fo the CRD into a secret ([#452]). ### Changed From ad9dbe9394a88e01dab8f3b9629a61a4385bcf1a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 13 May 2024 09:54:32 +0200 Subject: [PATCH 5/7] Update rust/operator-binary/src/controller.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/controller.rs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 41d6e935..ead412dd 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -835,17 +835,10 @@ fn build_metastore_rolegroup_statefulset( // the placeholders in hive-site.xml so that the operator does not "touch" the secret. let credentials_secret_name = hive.spec.cluster_config.database.credentials_secret.clone(); - let mut env: Vec = vec![env_var_from_secret( - DB_USERNAME_ENV, - &credentials_secret_name, - "username", - )]; - env.push(env_var_from_secret( - DB_PASSWORD_ENV, - &credentials_secret_name, - "password", - )); - container_builder.add_env_vars(env); + container_builder.add_env_vars(vec![ + env_var_from_secret(DB_USERNAME_ENV, &credentials_secret_name, "username"), + env_var_from_secret(DB_PASSWORD_ENV, &credentials_secret_name, "password"), + ]); let mut pod_builder = PodBuilder::new(); From 50be4d4d5d239a4844c6282738ce388d0d378027 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 13 May 2024 09:54:51 +0200 Subject: [PATCH 6/7] Update CHANGELOG.md Co-authored-by: Malte Sander --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08380ac4..40d82b09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to this project will be documented in this file. ### Fixed -- [BREAKING] Move the metastore DB credentials out fo the CRD into a secret ([#452]). +- [BREAKING] Move the metastore `user` and `password` DB credentials out of the CRD into a Secret containing the keys `username` and `password` ([#452]). ### Changed From 67ed2e0f1d7704ffd68e4efebc148f0e76de6551 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 13 May 2024 11:38:29 +0200 Subject: [PATCH 7/7] consistent ordering --- tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 | 2 +- tests/templates/kuttl/smoke/60-install-hive.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 index 1636328e..e7f4f782 100644 --- a/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-install-hive.yaml.j2 @@ -15,8 +15,8 @@ spec: clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true - dbType: derby credentialsSecret: hive-credentials + dbType: derby {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} diff --git a/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 b/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 index e6c8c077..fecab705 100644 --- a/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke/60-install-hive.yaml.j2 @@ -15,8 +15,8 @@ spec: clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive - dbType: postgres credentialsSecret: hive-credentials + dbType: postgres s3: reference: minio {% if lookup('env', 'VECTOR_AGGREGATOR') %}