Skip to content

Commit 3aa3382

Browse files
KernelMakerKernelMaker
authored
Description: (#655)
Extract the necessary files from the rondb-docker repository to integrate the RonDB Docker image building components into the RonDB repository.
1 parent 76fa8fd commit 3aa3382

31 files changed

+1786
-0
lines changed

‎build_scripts/release_scripts/build-docker-image/Dockerfile‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# syntax=docker/dockerfile:1
2+
3+
# Explaining order of ARGS in Dockerfiles: https://stackoverflow.com/a/53683625/9068781
4+
5+
ARG RONDB_TARBALL_LOCAL_REMOTE=remote
6+
7+
# Download all required Ubuntu dependencies
8+
FROM ubuntu:22.04 AS rondb_runtime_dependencies
9+
10+
ARG BUILDPLATFORM
11+
ARG TARGETPLATFORM
12+
13+
ARG TARGETARCH
14+
ARG TARGETVARIANT
15+
16+
RUN echo "Running on $BUILDPLATFORM, building for $TARGETPLATFORM"
17+
RUN echo "TARGETARCH: $TARGETARCH; TARGETVARIANT: $TARGETVARIANT"
18+
19+
# Fail the build if TARGETARCH is not arm64 or amd64
20+
RUN if [ "$TARGETARCH" != "arm64" ] && [ "$TARGETARCH" != "amd64" ]; then \
21+
echo "Unsupported architecture: $TARGETARCH" && exit 1; \
22+
fi
23+
24+
RUN --mount=type=cache,target=/var/cache/apt,id=ubuntu22-apt-$TARGETPLATFORM \
25+
--mount=type=cache,target=/var/lib/apt/lists,id=ubuntu22-apt-lists-$TARGETPLATFORM \
26+
apt-get update -y \
27+
&& apt-get install -y wget tar gzip vim \
28+
libaio1 libaio-dev \
29+
libncurses5 libnuma-dev \
30+
bc default-jdk maven \
31+
sudo iproute2 dnsutils
32+
# Java & Maven are required by YCSB
33+
# bc is required by dbt2
34+
# libaio is a dynamic library used by RonDB
35+
# libncurses5 & libnuma-dev are required for x86 only
36+
# sudo is required in the entrypoint
37+
# iproute2 is for using the command `ss`
38+
# dnsutils is for using the command `nslookup` in K8s
39+
40+
# Let PATH survive through sudo
41+
RUN sed -ri '/secure_path/d' /etc/sudoers
42+
43+
# Creating a cache dir for downloads to avoid redownloading
44+
ENV DOWNLOADS_CACHE_DIR=/tmp/downloads
45+
RUN mkdir $DOWNLOADS_CACHE_DIR
46+
47+
# Copying bare minimum of Hopsworks cloud environment for now
48+
FROM rondb_runtime_dependencies AS cloud_preparation
49+
ARG RONDB_VERSION=21.04.16
50+
ENV HOPSWORK_DIR=/srv/hops
51+
ENV RONDB_BIN_DIR=$HOPSWORK_DIR/mysql-$RONDB_VERSION
52+
RUN mkdir -p $RONDB_BIN_DIR
53+
54+
# Get RonDB tarball from local path & unpack it
55+
FROM cloud_preparation AS local_tarball
56+
ARG RONDB_X86_TARBALL_URI
57+
ARG RONDB_ARM_TARBALL_URI
58+
59+
RUN case "$TARGETARCH" in \
60+
amd64) echo "export TARBALL_PATH=$RONDB_X86_TARBALL_URI" >> /env.sh;; \
61+
arm64) echo "export TARBALL_PATH=$RONDB_ARM_TARBALL_URI" >> /env.sh;; \
62+
esac
63+
64+
RUN --mount=type=bind,source=.,target=/context \
65+
. /env.sh \
66+
&& tar xfz /context/${TARBALL_PATH} -C $RONDB_BIN_DIR --strip-components=1
67+
68+
# Get RonDB tarball from remote url & unpack it
69+
FROM cloud_preparation AS remote_tarball
70+
ARG RONDB_X86_TARBALL_URI
71+
ARG RONDB_ARM_TARBALL_URI
72+
73+
RUN case "$TARGETARCH" in \
74+
amd64) TARBALL_URL=$RONDB_X86_TARBALL_URI;; \
75+
arm64) TARBALL_URL=$RONDB_ARM_TARBALL_URI;; \
76+
esac \
77+
&& wget $TARBALL_URL -O ./temp_tarball.tar.gz \
78+
&& tar xfz ./temp_tarball.tar.gz -C $RONDB_BIN_DIR --strip-components=1 \
79+
&& rm ./temp_tarball.tar.gz
80+
81+
FROM ${RONDB_TARBALL_LOCAL_REMOTE}_tarball
82+
83+
WORKDIR $HOPSWORK_DIR
84+
85+
# We use symlinks in case we want to exchange binaries
86+
ENV RONDB_BIN_DIR_SYMLINK=$HOPSWORK_DIR/mysql
87+
RUN ln -s $RONDB_BIN_DIR $RONDB_BIN_DIR_SYMLINK
88+
89+
ENV PATH=$RONDB_BIN_DIR_SYMLINK/bin:$PATH
90+
91+
# Add RonDB libs to system path (cannot use env variables here)
92+
COPY <<-"EOF" /etc/ld.so.conf.d/rondb.conf
93+
/srv/hops/mysql/lib
94+
/srv/hops/mysql/lib/private
95+
EOF
96+
RUN ldconfig --verbose
97+
98+
ENV RONDB_DATA_DIR=$HOPSWORK_DIR/mysql-cluster
99+
ENV MGMD_DATA_DIR=$RONDB_DATA_DIR/mgmd
100+
ENV MYSQLD_DATA_DIR=$RONDB_DATA_DIR/mysql
101+
ENV NDBD_DATA_DIR=$RONDB_DATA_DIR/ndb_data
102+
103+
RUN mkdir -p $MGMD_DATA_DIR $NDBD_DATA_DIR $MYSQLD_DATA_DIR
104+
105+
ENV MYSQL_FILES_DIR=$RONDB_DATA_DIR/mysql-files
106+
RUN mkdir -p $MYSQL_FILES_DIR
107+
108+
ENV LOG_DIR=$RONDB_DATA_DIR/log
109+
ENV RONDB_SCRIPTS_DIR=$RONDB_DATA_DIR/ndb/scripts
110+
ENV BACKUP_DATA_DIR=$RONDB_DATA_DIR/ndb/backups
111+
ENV DISK_COLUMNS_DIR=$RONDB_DATA_DIR/ndb_disk_columns
112+
ENV MYSQL_UNIX_PORT=$RONDB_DATA_DIR/mysql.sock
113+
114+
RUN mkdir -p $LOG_DIR $RONDB_SCRIPTS_DIR $BACKUP_DATA_DIR $DISK_COLUMNS_DIR
115+
116+
COPY ./resources/rondb_scripts $RONDB_SCRIPTS_DIR
117+
ENV PATH=$RONDB_SCRIPTS_DIR:$PATH
118+
119+
RUN touch $MYSQL_UNIX_PORT
120+
121+
# We expect this image to be used as base image to other
122+
# images with additional files specific to Docker
123+
COPY ./resources/entrypoints ./docker/rondb_standalone/entrypoints
124+
COPY ./resources/healthcheck.sh ./docker/rondb_standalone/healthcheck.sh
125+
126+
# Can be used to mount SQL init scripts
127+
ENV SQL_INIT_SCRIPTS_DIR=$HOPSWORK_DIR/docker/rondb_standalone/sql_init_scripts
128+
RUN mkdir $SQL_INIT_SCRIPTS_DIR
129+
130+
# Creating benchmarking files/directories
131+
# When using load balancers, "sysbench" can be used for both _single and _multi
132+
ENV BENCHMARKS_DIR=$HOPSWORK_DIR/benchmarks
133+
RUN mkdir $BENCHMARKS_DIR && cd $BENCHMARKS_DIR \
134+
&& mkdir -p sysbench sysbench_single sysbench_multi dbt2_single dbt2_multi dbt2_data
135+
136+
# These directories have to have 777 permissions if we want to
137+
# run RonDB containers with arbitrary users
138+
RUN chmod 777 -R $RONDB_DATA_DIR \
139+
&& chmod 777 -R $HOPSWORK_DIR/docker \
140+
&& chmod 777 -R $BENCHMARKS_DIR
141+
142+
# Remove awkward message when using arbitrary user
143+
RUN echo "PS1='${debian_chroot:+(\$debian_chroot)}\h:\w\$ '" >> /etc/bash.bashrc
144+
145+
ENTRYPOINT ["./docker/rondb_standalone/entrypoints/main.sh"]
146+
EXPOSE 3306 33060 11860 1186 4406 5406
147+
CMD ["mysqld"]

‎build_scripts/release_scripts/build-docker-image/README‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
The files in this directory are extracted from the rondb-docker repository
2+
to integrate the Docker image building process into the RonDB repository.

‎build_scripts/release_scripts/build-docker-image/resources/entrypoints/main.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
# Copyright (c) 2017, 2021, Oracle and/or its affiliates.
3+
# Copyright (c) 2021, 2021, Hopsworks AB and/or its affiliates.
4+
#
5+
# This program is free software; you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation; version 2 of the License.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU General Public License
15+
# along with this program; if not, write to the Free Software
16+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17+
set -e
18+
19+
# https://stackoverflow.com/a/246128/9068781
20+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
21+
22+
if [ "$1" = 'mysqld' ]; then
23+
"$SCRIPT_DIR/mysqld.sh" "$@"
24+
else
25+
# "set" lets us set the arguments to the current script.
26+
# the command also has its own commands (see set --help).
27+
# to avoid accidentally using one of the set-commands,
28+
# we use "set --" to make clear that everything following
29+
# this is an argument to the script itself and not the set
30+
# command.
31+
32+
# The default for mgmds & ndbmtds is to run as daemon processes
33+
if [ "$1" != "rdrs" ]; then
34+
set -- "$@" --nodaemon
35+
fi
36+
37+
if [ "$1" == "rdrs" ]; then
38+
echo "[entrypoints/main.sh] Starting REST API server: $@"
39+
40+
elif [ "$1" == "ndb_mgmd" ]; then
41+
echo "[entrypoints/main.sh] Starting ndb_mgmd"
42+
set -- "$@" -f "$RONDB_DATA_DIR/config.ini" --configdir="$RONDB_DATA_DIR/log"
43+
elif [ "$1" == "ndbmtd" ]; then
44+
echo "[entrypoints/main.sh] Starting ndbmtd"
45+
# Command for more verbosity with ndbmtds: `set -- "$@" --verbose=TRUE`
46+
47+
# We have to run ndbmtd as a child process, since trap and exec
48+
# do not play nice.
49+
"$@"
50+
exit $?
51+
elif [ "$1" == "ndb_mgm" ]; then
52+
echo "[entrypoints/main.sh] Starting ndb_mgm"
53+
fi
54+
55+
echo "[entrypoints/main.sh] Running: $*"
56+
exec "$@"
57+
fi

‎build_scripts/release_scripts/build-docker-image/resources/entrypoints/mysqld.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
# Copyright (c) 2017, 2021, Oracle and/or its affiliates.
3+
# Copyright (c) 2021, 2022, Hopsworks AB and/or its affiliates.
4+
#
5+
# This program is free software; you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation; version 2 of the License.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU General Public License
15+
# along with this program; if not, write to the Free Software
16+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17+
set -e
18+
19+
# https://stackoverflow.com/a/246128/9068781
20+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
21+
22+
source $SCRIPT_DIR/mysqld_configure.sh "$@"
23+
24+
export MYSQLD_PARENT_PID=$$
25+
26+
if [ ! -z "$MYSQL_INITIALIZE_DB" ]; then
27+
source $SCRIPT_DIR/mysqld_init_db.sh "$@"
28+
else
29+
echo "[entrypoints/mysqld.sh] Not initializing MySQL databases"
30+
fi
31+
32+
# This is not being used anymore
33+
if [ -n "$MYSQL_INITIALIZE_ONLY" ]; then
34+
echo "[entrypoints/mysqld.sh] MYSQL_INITIALIZE_ONLY is set, so we're exiting without starting the MySQLd"
35+
exit 0
36+
fi
37+
38+
echo '[entrypoints/mysqld.sh] Ready for starting up MySQLd'
39+
echo "[entrypoints/mysqld.sh] Running: $*"
40+
exec "$@"

‎build_scripts/release_scripts/build-docker-image/resources/entrypoints/mysqld_configure.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
# Copyright (c) 2017, 2021, Oracle and/or its affiliates.
3+
# Copyright (c) 2021, 2022, 2023 Hopsworks AB and/or its affiliates.
4+
#
5+
# This program is free software; you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation; version 2 of the License.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU General Public License
15+
# along with this program; if not, write to the Free Software
16+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17+
set -e
18+
19+
# Fetch value from server config
20+
# We use mysqld --verbose --help instead of my_print_defaults because the
21+
# latter only show values present in config files, and not server defaults
22+
_get_config() {
23+
local conf="$1"
24+
shift
25+
"$@" --verbose --help 2>/dev/null | grep "^$conf" | awk '$1 == "'"$conf"'" { print $2; exit }'
26+
}
27+
28+
# Make sure that "--defaults-file" is always run as second argument
29+
# Otherwise there is a risk that it might not be read
30+
shift
31+
set -- mysqld --defaults-file="$RONDB_DATA_DIR/my.cnf" "$@"
32+
echo "[entrypoints/mysqld_configure.sh] \$@: $*"
33+
34+
# Test that the server can start. We redirect stdout to /dev/null so
35+
# only the error messages are left.
36+
result=0
37+
output=$("$@" --validate-config) || result=$?
38+
if [ ! "$result" = "0" ]; then
39+
echo >&2 '[entrypoints/mysqld_configure.sh] ERROR: Unable to start MySQL. Please check your configuration.'
40+
echo >&2 "[entrypoints/mysqld_configure.sh] $output"
41+
exit 1
42+
fi
43+
echo "[entrypoints/mysqld_configure.sh] The MySQL configuration has been validated"
44+
45+
echo '[entrypoints/mysqld_configure.sh] Initializing MySQL...'
46+
47+
# Technically, specifying the user here is unnecessary since that is
48+
# the default user according to the Dockerfile
49+
"$@" \
50+
--log-error-verbosity=3 \
51+
--initialize-insecure \
52+
--explicit_defaults_for_timestamp
53+
54+
echo '[entrypoints/mysqld_configure.sh] MySQL initialized'

0 commit comments

Comments
 (0)