|
| 1 | +#!/bin/sh |
| 2 | + |
| 3 | + |
| 4 | +MODE=$1 |
| 5 | + |
| 6 | +if [ "$MODE" = "hms" ]; then |
| 7 | + echo "Will run metastore" |
| 8 | +elif [ "$MODE" = "init" ]; then |
| 9 | + echo "Will initialize DB if not yet done" |
| 10 | +else |
| 11 | + echo "First parameter must be 'hms' or 'init'" |
| 12 | + exit 20 |
| 13 | +fi |
| 14 | + |
| 15 | +if [ -z "${HIVEMS_DB}" ]; then echo "HIVEMS_DB env variable must be defined!"; exit 1; fi |
| 16 | +if [ -z "${HIVEMS_USER}" ]; then echo "HIVEMS_USER env variable must be defined!"; exit 1; fi |
| 17 | +if [ -z "${HIVEMS_PASSWORD}" ]; then echo "HIVEMS_PASSWORD env variable must be defined!"; exit 1; fi |
| 18 | +if [ -z "${DB_HOST}" ]; then echo "DB_HOST env variable must be defined!"; exit 1; fi |
| 19 | +if [ -z "${DB_PORT}" ]; then echo "DB_PORT env variable must be defined!"; exit 1; fi |
| 20 | +if [ -z "${METASTORE_VERSION}" ]; then echo "METASTORE_VERSION env variable must be defined!"; exit 1; fi |
| 21 | +if [ -z "${HADOOP_VERSION}" ]; then echo "HADOOP_VERSION env variable must be defined!"; exit 1; fi |
| 22 | + |
| 23 | +# May be null in case of usage of AWS instance roles |
| 24 | +#if [ -z "${S3_ENDPOINT}" ]; then echo "S3_ENDPOINT env variable must be defined!"; exit 1; fi |
| 25 | +#if [ -z "${S3_ACCESS_KEY}" ]; then echo "S3_ACCESS_KEY env variable must be defined!"; exit 1; fi |
| 26 | +#if [ -z "${S3_SECRET_KEY}" ]; then echo "S3_SECRET_KEY env variable must be defined!"; exit 1; fi |
| 27 | + |
| 28 | +if [ -z "${JAVA_HOME}" ]; then export JAVA_HOME=/usr/local/openjdk-8; fi |
| 29 | +if [ -z "${BASEDIR}" ]; then export BASEDIR=/opt; fi |
| 30 | +if [ -z "${LOG_LEVEL}" ]; then export LOG_LEVEL=INFO; fi |
| 31 | +if [ -z "${THRIFT_LISTENING_PORT}" ]; then export THRIFT_LISTENING_PORT=9083; fi |
| 32 | +if [ -z "${S3_REQUEST_TIMEOUT}" ]; then export S3_REQUEST_TIMEOUT=0; fi |
| 33 | + |
| 34 | +export HADOOP_HOME=${BASEDIR}/hadoop-${HADOOP_VERSION} |
| 35 | +export HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/aws-java-sdk-bundle-*.jar:${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-aws-${HADOOP_VERSION}.jar |
| 36 | + |
| 37 | +echo "" |
| 38 | +echo "METASTORE_VERSION=$METASTORE_VERSION" |
| 39 | +echo "HADOOP_VERSION=$HADOOP_VERSION" |
| 40 | +echo "" |
| 41 | + |
| 42 | +cat >${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/conf/metastore-log4j2.properties <<-EOF |
| 43 | +status = INFO |
| 44 | +name = MetastoreLog4j2 |
| 45 | +packages = org.apache.hadoop.hive.metastore |
| 46 | +# list of all appenders |
| 47 | +appenders = console |
| 48 | +# console appender |
| 49 | +appender.console.type = Console |
| 50 | +appender.console.name = console |
| 51 | +appender.console.target = SYSTEM_ERR |
| 52 | +appender.console.layout.type = PatternLayout |
| 53 | +appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n |
| 54 | +# list of all loggers |
| 55 | +loggers = DataNucleus, Datastore, JPOX, PerfLogger |
| 56 | +logger.DataNucleus.name = DataNucleus |
| 57 | +logger.DataNucleus.level = ERROR |
| 58 | +logger.Datastore.name = Datastore |
| 59 | +logger.Datastore.level = ERROR |
| 60 | +logger.JPOX.name = JPOX |
| 61 | +logger.JPOX.level = ERROR |
| 62 | +logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger |
| 63 | +logger.PerfLogger.level = INFO |
| 64 | +# root logger |
| 65 | +rootLogger.level = ${LOG_LEVEL} |
| 66 | +rootLogger.appenderRefs = root |
| 67 | +rootLogger.appenderRef.root.ref = console |
| 68 | +EOF |
| 69 | + |
| 70 | +cat >${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/conf/metastore-site.xml <<-EOF |
| 71 | +<?xml version="1.0" encoding="UTF-8" standalone="no"?> |
| 72 | +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> |
| 73 | +<configuration> |
| 74 | + <property> |
| 75 | + <name>metastore.thrift.uris</name> |
| 76 | + <value>thrift://localhost:9083</value> |
| 77 | + <description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description> |
| 78 | + </property> |
| 79 | + <property> |
| 80 | + <name>metastore.task.threads.always</name> |
| 81 | + <value>org.apache.hadoop.hive.metastore.events.EventCleanerTask,org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask</value> |
| 82 | + </property> |
| 83 | + <property> |
| 84 | + <name>metastore.expression.proxy</name> |
| 85 | + <value>org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy</value> |
| 86 | + </property> |
| 87 | + <property> |
| 88 | + <name>metastore.server.min.threads</name> |
| 89 | + <value>5</value> |
| 90 | + </property> |
| 91 | + <property> |
| 92 | + <name>metastore.server.max.threads</name> |
| 93 | + <value>20</value> |
| 94 | + </property> |
| 95 | + <property> |
| 96 | + <name>javax.jdo.option.Multithreaded</name> |
| 97 | + <value>true</value> |
| 98 | + <description>Set this to true if multiple threads access metastore through JDO concurrently.</description> |
| 99 | + </property> |
| 100 | + <property> |
| 101 | + <name>javax.jdo.PersistenceManagerFactoryClass</name> |
| 102 | + <value>org.datanucleus.api.jdo.JDOPersistenceManagerFactory</value> |
| 103 | + <description>class implementing the jdo persistence</description> |
| 104 | + </property> |
| 105 | + <property> |
| 106 | + <name>javax.jdo.option.ConnectionDriverName</name> |
| 107 | + <value>org.postgresql.Driver</value> |
| 108 | + </property> |
| 109 | + <property> |
| 110 | + <name>javax.jdo.option.ConnectionURL</name> |
| 111 | + <value>jdbc:postgresql://${DB_HOST}:${DB_PORT}/${HIVEMS_DB}</value> |
| 112 | + </property> |
| 113 | + <property> |
| 114 | + <name>javax.jdo.option.ConnectionUserName</name> |
| 115 | + <value>${HIVEMS_USER}</value> |
| 116 | + </property> |
| 117 | + <property> |
| 118 | + <name>javax.jdo.option.ConnectionPassword</name> |
| 119 | + <value>${HIVEMS_PASSWORD}</value> |
| 120 | + </property> |
| 121 | + <property> |
| 122 | + <name>fs.s3a.path.style.access</name> |
| 123 | + <value>true</value> |
| 124 | + </property> |
| 125 | + <property> |
| 126 | + <name>fs.s3a.connection.request.timeout</name> |
| 127 | + <value>${S3_REQUEST_TIMEOUT}</value> |
| 128 | + </property> |
| 129 | +EOF |
| 130 | + |
| 131 | +if [ ! -z "${S3_ENDPOINT}" ] |
| 132 | +then |
| 133 | +cat >>${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/conf/metastore-site.xml <<-EOF |
| 134 | + <property> |
| 135 | + <name>fs.s3a.endpoint</name> |
| 136 | + <value>${S3_ENDPOINT}</value> |
| 137 | + </property> |
| 138 | +EOF |
| 139 | +fi |
| 140 | + |
| 141 | +if [ ! -z "${S3_ACCESS_KEY}" ] |
| 142 | +then |
| 143 | +cat >>${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/conf/metastore-site.xml <<-EOF |
| 144 | + <property> |
| 145 | + <name>fs.s3a.access.key</name> |
| 146 | + <value>${S3_ACCESS_KEY}</value> |
| 147 | + </property> |
| 148 | + <property> |
| 149 | + <name>fs.s3a.secret.key</name> |
| 150 | + <value>${S3_SECRET_KEY}</value> |
| 151 | + </property> |
| 152 | +EOF |
| 153 | +fi |
| 154 | + |
| 155 | +if [ ! -z "${ASSUME_ROLE_ARN}" ] |
| 156 | +then |
| 157 | +cat >>${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/conf/metastore-site.xml <<-EOF |
| 158 | + <property> |
| 159 | + <name>fs.s3a.aws.credentials.provider</name> |
| 160 | + <value>org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider</value> |
| 161 | + </property> |
| 162 | + <property> |
| 163 | + <name>fs.s3a.assumed.role.credentials.provider</name> |
| 164 | + <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value> |
| 165 | + </property> |
| 166 | + <property> |
| 167 | + <name>fs.s3a.assumed.role.arn</name> |
| 168 | + <value>${ASSUME_ROLE_ARN}</value> |
| 169 | + </property> |
| 170 | +EOF |
| 171 | +fi |
| 172 | + |
| 173 | +cat >>${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/conf/metastore-site.xml <<-EOF |
| 174 | +</configuration> |
| 175 | +EOF |
| 176 | + |
| 177 | +# set +x |
| 178 | + |
| 179 | +export PGPASSWORD=${HIVEMS_PASSWORD} |
| 180 | + |
| 181 | +echo "Will wait for postgresql server to be ready" |
| 182 | +while ! pg_isready --host ${DB_HOST} --port ${DB_PORT}; do echo "Waiting for postgresql to be ready..."; sleep 2; done; |
| 183 | + |
| 184 | +echo "Will wait for \"${HIVEMS_DB}\" to exists" |
| 185 | +while ! psql --host ${DB_HOST} --port ${DB_PORT} -U ${HIVEMS_USER} -d ${HIVEMS_DB} -c "\c ${HIVEMS_DB}" >/dev/null 2>&1; do echo "Waiting for ${HIVEMS_DB} database to be ready..."; sleep 2; done; |
| 186 | + |
| 187 | + |
| 188 | +if [ "$MODE" = "init" ]; then |
| 189 | + echo "Initialize schema if DBS table does not exists" |
| 190 | + psql --host ${DB_HOST} --port ${DB_PORT} -U ${HIVEMS_USER} -d ${HIVEMS_DB} -c 'SELECT "DB_ID" FROM "DBS"' >/dev/null 2>&1; |
| 191 | + if [ $? -ne 0 ] |
| 192 | + then |
| 193 | + echo "Will initialize the DB" |
| 194 | + ${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/bin/schematool -initSchema -dbType postgres |
| 195 | + fi |
| 196 | + echo "DATABASE SCHEMA SHOULD BE OK NOW!!" |
| 197 | + exit 0 |
| 198 | +fi |
| 199 | + |
| 200 | +# MODE = "hms" here |
| 201 | + |
| 202 | +echo "Will wait for database schema to be ready...." |
| 203 | +while ! psql --host ${DB_HOST} --port ${DB_PORT} -U ${HIVEMS_USER} -d ${HIVEMS_DB} -c 'SELECT "SCHEMA_VERSION" FROM "VERSION"' >/dev/null 2>&1; do echo "Waiting for ${HIVEMS_DB} schema to be ready..."; sleep 2; done; |
| 204 | +echo "DATABASE SCHEMA IS OK. CAN LAUNCH!!" |
| 205 | +echo "" |
| 206 | + |
| 207 | +unset PGPASSWORD |
| 208 | + |
| 209 | +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dcom.amazonaws.sdk.disableCertChecking=true" |
| 210 | + |
| 211 | +# WARNING: This variable is set by Kubernetes in a form: tcp://XX.XX.XX.XX:9083. |
| 212 | +# For the metastore, this is an entry variable hosting only the listening port, as a single number. So failure. |
| 213 | +unset METASTORE_PORT |
| 214 | + |
| 215 | +${BASEDIR}/apache-hive-metastore-${METASTORE_VERSION}-bin/bin/start-metastore -p $THRIFT_LISTENING_PORT |
| 216 | +err=$? |
| 217 | + |
| 218 | +if [ -n "$WAIT_ON_ERROR" ]; then |
| 219 | + if [ $err -ne 0 ]; then |
| 220 | + echo "ERROR: rc=$err. Will wait $WAIT_ON_ERROR sec...." |
| 221 | + sleep $WAIT_ON_ERROR |
| 222 | + fi |
| 223 | +fi |
| 224 | + |
| 225 | +return $err |
| 226 | + |
| 227 | + |
0 commit comments