From 962e8632d6afb0a6ae820987f02a69980016ddd0 Mon Sep 17 00:00:00 2001 From: matthias Date: Wed, 15 Feb 2023 09:16:28 +0100 Subject: [PATCH 01/11] commit stuff --- Makefile | 56 ++-- docker/base/Dockerfile | 2 + docker/exporter/Dockerfile | 39 +++ docker/newpg/Dockerfile | 134 +++++++++ docker/pgbackrest/Dockerfile | 6 +- docker/postgres-oracle/Dockerfile | 274 ++++++++++++++++++ .../Dockerfile | 67 +++-- docker/postgres/Dockerfile | 44 +-- launcher/exporter/launch.sh | 20 ++ launcher/pgbackrest/launch.sh | 38 ++- scripts/configure_spilo.py | 3 +- scripts/exporter/queries/queries.yaml | 253 ++++++++++++++++ scripts/exporter/queries/queries_nodemx.yaml | 181 ++++++++++++ .../exporter/queries/queries_pgbackrest.yaml | 164 +++++++++++ .../exporter/queries/queries_statistics.yaml | 93 ++++++ scripts/exporter/storage.sh | 12 + scripts/exporter/structure.sql | 109 +++++++ .../pg14/pg14_statement_queries.yaml | 34 +++ scripts/pgbackrest/backup/start.sh | 12 +- 19 files changed, 1451 insertions(+), 90 deletions(-) create mode 100644 docker/exporter/Dockerfile create mode 100644 docker/newpg/Dockerfile create mode 100644 docker/postgres-oracle/Dockerfile rename docker/{postgres_stage => postgres-stage}/Dockerfile (79%) create mode 100755 launcher/exporter/launch.sh create mode 100644 scripts/exporter/queries/queries.yaml create mode 100644 scripts/exporter/queries/queries_nodemx.yaml create mode 100644 scripts/exporter/queries/queries_pgbackrest.yaml create mode 100644 scripts/exporter/queries/queries_statistics.yaml create mode 100644 scripts/exporter/storage.sh create mode 100644 scripts/exporter/structure.sql create mode 100644 scripts/exporter/version_specific/pg14/pg14_statement_queries.yaml diff --git a/Makefile b/Makefile index 044efa5..1716565 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,25 @@ # Define Default if Values not exist -BASE_IMAGE ?= rockylinux:9-minimal +BASE_IMAGE ?= rockylinux:9.1-minimal BASEOS ?= rocky9 IMAGE_REPOSITORY ?= docker.io -IMAGE_PATH ?= cybertec-os-container -PG_MAJOR ?= 15 -PG_VERSION ?= 15.0 +IMAGE_PATH ?= cybertec-pg-container +PGVERSION ?= 15 +PGVERSION_FULL ?= 15.2 OLD_PG_VERSIONS ?= 10 11 12 13 14 PATRONI_VERSION ?= 2.1.4 PGBACKREST_VERSION ?= 2.41 POSTGIS_VERSION ?= 3.2 PACKAGER ?= dnf BUILD ?= 0 -IMAGE_TAG ?= $(BASEOS)-$(PG_VERSION)-$(BUILD) -POSTGIS_IMAGE_TAG ?= $(BASEOS)-$(PG_VERSION)-$(POSTGIS_VERSION)-$(BUILD) +IMAGE_TAG ?= $(BASEOS)-$(PGVERSION_FULL)-$(BUILD) +POSTGIS_IMAGE_TAG ?= $(BASEOS)-$(PGVERSION_FULL)-$(POSTGIS_VERSION)-$(BUILD) # Settings for the Build-Process BUILDWITH ?= docker -ROOTPATH = $(GOPATH)/src/github.com/cybertec/cybertec-os-container +ROOTPATH = $(GOPATH)/src/github.com/cybertec/cybertec-pg-container ifndef ROOTPATH - export ROOTPATH=$(GOPATH)/src/github.com/cybertec/cybertec-os-container + export ROOTPATH=$(GOPATH)/src/github.com/cybertec/cybertec-pg-container endif # Build Images @@ -28,12 +28,13 @@ all: base pgbackrest postgres base: base pgbackrest: pgbackrest postgres: base postgres -postgres-stage: postgres-stage +postgres-stage: base postgres-stage +exporter: exporter base-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/base/Dockerfile \ - --tag cybertec-os-container/base:0.0.$(BUILD) \ + --tag cybertec-pg-container/base:0.0.$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -44,7 +45,7 @@ base: base-build; pgbackrest-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/pgbackrest/Dockerfile \ - --tag cybertec-os-container/pgbackrest:0.0.$(BUILD) \ + --tag cybertec-pg-container/pgbackrest:0.0.$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -52,14 +53,14 @@ pgbackrest-build: --build-arg CONTAINERSUITE \ --build-arg BUILD \ --build-arg PGBACKREST_VERSION \ - --build-arg PG_MAJOR + --build-arg PGVERSION pgbackrest: pgbackrest-build; postgres-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/postgres/Dockerfile \ - --tag cybertec-os-container/postgres:$(PG_MAJOR).0.$(BUILD) \ + --tag cybertec-pg-container/postgres:$(PGVERSION_FULL)-$(BETA)$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -68,16 +69,32 @@ postgres-build: --build-arg BUILD \ --build-arg PATRONI_VERSION \ --build-arg PGBACKREST_VERSION \ - --build-arg PG_VERSION \ --build-arg OLD_PG_VERSIONS \ - --build-arg PG_MAJOR + --build-arg PGVERSION postgres: postgres-build postgres-stage-build: docker build $(ROOTPATH) \ - --file $(ROOTPATH)/docker/postgres_stage/Dockerfile \ - --tag cybertec-os-container/postgres-stage:$(PG_MAJOR).0.$(BUILD) \ + --file $(ROOTPATH)/docker/postgres-stage/Dockerfile \ + --tag cybertec-pg-container/postgres-stage:$(PGVERSION_FULL)-$(BETA)$(BUILD) \ + --build-arg BASE_IMAGE \ + --build-arg IMAGE_REPOSITORY \ + --build-arg BASEOS \ + --build-arg PACKAGER \ + --build-arg CONTAINERSUITE \ + --build-arg BUILD \ + --build-arg PATRONI_VERSION \ + --build-arg PGBACKREST_VERSION \ + --build-arg OLD_PG_VERSIONS \ + --build-arg PGVERSION + +postgres-stage: postgres-stage-build + +exporter-build: + docker build $(ROOTPATH) \ + --file $(ROOTPATH)/docker/exporter/Dockerfile \ + --tag cybertec-pg-container/exporter:0.1.$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -86,8 +103,7 @@ postgres-stage-build: --build-arg BUILD \ --build-arg PATRONI_VERSION \ --build-arg PGBACKREST_VERSION \ - --build-arg PG_VERSION \ --build-arg OLD_PG_VERSION \ - --build-arg PG_MAJOR + --build-arg PGVERSION -postgres-stage: postgres-stage-build \ No newline at end of file +exporter: exporter-build \ No newline at end of file diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 5930194..81743a4 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -28,6 +28,8 @@ RUN ${PACKAGER} -y install --nodocs \ glibc-langpack-en \ && ${PACKAGER} -y clean all ; +#Enable CRB-Repo (Powertools) +RUN ${PACKAGER} -y config-manager --set-enabled crb #Add PostgreSQL-Repo to Base RUN ${PACKAGER} install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm && ${PACKAGER} -y update && ${PACKAGER} -y clean all #RUN dnf -qy module disable postgresql diff --git a/docker/exporter/Dockerfile b/docker/exporter/Dockerfile new file mode 100644 index 0000000..0703254 --- /dev/null +++ b/docker/exporter/Dockerfile @@ -0,0 +1,39 @@ +ARG CONTAINERSUITE +ARG BUILD + +FROM ${CONTAINERSUITE}/base:0.0.${BUILD} + +# Dockerfile specific informations +ARG PACKAGER +ARG BASEOS +ARG PGBACKREST_VERSION +ARG PG_MAJOR + +RUN if [ "$BASEOS" = "ubi8" ] ; then \ + ${PACKAGER} -y install --nodocs \ + shadow-utils \ + tar \ + bzip2 \ + lz4 \ + #crunchy-backrest-${BACKREST_VER} \ + && ${PACKAGER} -y clean all ; \ +else \ + ${PACKAGER} -y install --nodocs \ + --setopt=skip_missing_names_on_install=False \ + bzip2 \ + lz4 \ + nano \ + git \ + go \ + dumb-init \ + && ${PACKAGER} -y clean all ; \ +fi + +RUN git clone https://github.com/prometheus-community/postgres_exporter.git && cd postgres_exporter && make build +COPY launcher/exporter/launch.sh / +COPY scripts/exporter/queries/ /postgres_exporter/queries + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] + +CMD ["/bin/sh", "/launch.sh", "init"] +#CMD ["/bin/sh", "/scripts/postgres/promote.sh"] \ No newline at end of file diff --git a/docker/newpg/Dockerfile b/docker/newpg/Dockerfile new file mode 100644 index 0000000..3a1d7b3 --- /dev/null +++ b/docker/newpg/Dockerfile @@ -0,0 +1,134 @@ +ARG BASE_IMAGE=ubuntu:22.04 +ARG PGVERSION=15 +ARG TIMESCALEDB="1.7.5 2.3.1 2.9.2" +ARG DEMO=false +ARG COMPRESS=false +ARG ADDITIONAL_LOCALES= +ARG CONTAINERSUITE +ARG BUILD + + +FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as base + +ARG ADDITIONAL_LOCALES + +COPY build_scripts/locales.sh /builddeps/ + +RUN bash /builddeps/locales.sh + + +FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as dependencies-builder + +ARG DEMO + +ENV WALG_VERSION=v2.0.1 + +COPY build_scripts/dependencies.sh /builddeps/ + +COPY dependencies/debs /builddeps/ + +RUN bash /builddeps/dependencies.sh + + +FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as builder-false + +ARG DEMO +ARG ADDITIONAL_LOCALES + +COPY build_scripts/prepare.sh build_scripts/locales.sh /builddeps/ + +RUN bash /builddeps/prepare.sh + +COPY --from=base /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.18 + +COPY cron_unprivileged.c dependencies/src /builddeps/ +COPY build_scripts/base.sh /builddeps/ +COPY --from=dependencies-builder /builddeps/*.deb /builddeps/ + +ARG PGVERSION +ARG TIMESCALEDB +ARG TIMESCALEDB_APACHE_ONLY=true +ARG TIMESCALEDB_TOOLKIT=true +ARG COMPRESS +ARG PGOLDVERSIONS="10 11 12 13 14" +ARG WITH_PERL=false + +ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" + +# Install PostgreSQL, extensions and contribs +ENV POSTGIS_VERSION=3.3 \ + POSTGIS_LEGACY=3.2 \ + BG_MON_COMMIT=241d8134e4dda4ffe6f21d40abf8d544d78bc9d8 \ + PG_AUTH_MON_COMMIT=439697fe2980cf48f1760f45e04c2d69b2748e73 \ + PG_MON_COMMIT=34c35a86e6dea26930bf1f770048cea2c695b80b \ + SET_USER=REL3_0_0 \ + PLPROFILER=REL4_2_1 \ + PG_PROFILE=4.1 \ + PAM_OAUTH2=v1.0.1 \ + PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \ + PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \ + PG_TM_AUX_COMMIT=51cf42f6043608919802fcc48458471d0ea3a127 + +WORKDIR /builddeps +RUN bash base.sh + +# Install wal-g +COPY --from=dependencies-builder /builddeps/wal-g /usr/local/bin/ + +COPY build_scripts/patroni_wale.sh build_scripts/compress_build.sh /builddeps/ + +# Install patroni and wal-e +ENV PATRONIVERSION=3.0.0 +ENV WALE_VERSION=1.1.1 + +WORKDIR / + +RUN bash /builddeps/patroni_wale.sh + +RUN if [ "$COMPRESS" = "true" ]; then bash /builddeps/compress_build.sh; fi + + +FROM scratch as builder-true +COPY --from=builder-false / / + + +FROM builder-${COMPRESS} + +LABEL maintainer="Polina Bungina " + +ARG PGVERSION +ARG TIMESCALEDB +ARG DEMO +ARG COMPRESS + +EXPOSE 5432 8008 8080 + +ENV LC_ALL=en_US.utf-8 \ + PATH=$PATH:/usr/lib/postgresql/$PGVERSION/bin \ + PGHOME=/home/postgres \ + RW_DIR=/run \ + TIMESCALEDB=$TIMESCALEDB \ + DEMO=$DEMO + +ENV WALE_ENV_DIR=$RW_DIR/etc/wal-e.d/env \ + LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ + PGROOT=$PGHOME/pgdata/pgroot + +ENV PGDATA=$PGROOT/data \ + PGLOG=$PGROOT/pg_log + +ENV USE_OLD_LOCALES=false + +WORKDIR $PGHOME + +COPY motd /etc/ +COPY runit /etc/service/ +COPY pgq_ticker.ini $PGHOME/ +COPY build_scripts/post_build.sh /builddeps/ + +RUN sh /builddeps/post_build.sh && rm -rf /builddeps/ + +COPY scripts bootstrap major_upgrade /scripts/ +COPY launch.sh / + +CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file diff --git a/docker/pgbackrest/Dockerfile b/docker/pgbackrest/Dockerfile index e29a774..a61410a 100644 --- a/docker/pgbackrest/Dockerfile +++ b/docker/pgbackrest/Dockerfile @@ -7,7 +7,7 @@ FROM ${CONTAINERSUITE}/base:0.0.${BUILD} ARG PACKAGER ARG BASEOS ARG PGBACKREST_VERSION -ARG PG_MAJOR +ARG PGVERSION RUN if [ "$BASEOS" = "ubi8" ] ; then \ ${PACKAGER} -y install --nodocs \ @@ -26,7 +26,7 @@ else \ openssh-server \ bzip2 \ lz4 \ - postgresql${PG_MAJOR}-server \ + postgresql${PGVERSION}-server \ pgbackrest-${PGBACKREST_VERSION} \ nano \ && ${PACKAGER} -y clean all ; \ @@ -67,7 +67,7 @@ RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod RUN chown -R postgres:postgres /opt/pgbackrest \ /backrestrepo /home/postgres/pgdata -ENV PATH=$PATH:/usr/pgsql-$PG_MAJOR/bin +ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin COPY launcher/pgbackrest/launch.sh / VOLUME ["sshd", "/home/postgres/pgdata", "/backrestrepo"] diff --git a/docker/postgres-oracle/Dockerfile b/docker/postgres-oracle/Dockerfile new file mode 100644 index 0000000..eea85bc --- /dev/null +++ b/docker/postgres-oracle/Dockerfile @@ -0,0 +1,274 @@ +ARG CONTAINERSUITE +ARG BUILD + +FROM ${CONTAINERSUITE}/base:0.0.${BUILD} + +# Dockerfile specific informations +ARG PACKAGER +ARG BASEOS +ARG PGBACKREST_VERSION +ARG PATRONI_VERSION +ARG PGVERSION +ARG OLD_PG_VERSIONS +ARG PG_SUPPORTED_VERSIONS="$PGVERSION" +ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION" +ARG TIMESCALEDB="1.7.5 2.3.1 2.8.1" + +# Spilo-specific +ENV SET_USER=REL3_0_0 \ + #BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ + #PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ + #PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ + PLPROFILER=REL4_1 \ + PG_PROFILE=0.3.6 \ + PAM_OAUTH2=v1.0.1 \ + PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \ + PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \ + PG_TM_AUX_COMMIT=6c012d38a4c1b0ba4a36952d60b0ce3a22ac9c3d + +# Get some Standard-Stuff +RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ + openssh-clients \ + openssh-server \ + shadow-utils \ + tar \ + bzip2 \ + lz4 \ + python3 \ + python3-pip \ + python3-psycopg2 \ + python3-psutil \ + python3-requests \ + python3-etcd \ + #python3-pyyaml \ + git \ + patchutils \ + binutils \ + make \ + cmake \ + gcc \ + #curl \ + libcurl-devel \ + pam-devel \ + wget \ + mlocate \ + git-clang-format \ + openssl-devel \ + ccache \ + redhat-rpm-config \ + krb5-devel \ + busybox \ + dpkg \ + jq \ + rsync \ + && ${PACKAGER} -y clean all; + + +RUN if [ "$BASEOS" = "ubi8" ] ; then \ + ${PACKAGER} -y install --nodocs \ + pgbackrest-${PGBACKREST_VERSION} \ + && ${PACKAGER} -y clean all ; \ +else \ + ${PACKAGER} -y install --nodocs \ + --setopt=skip_missing_names_on_install=False \ + pgbackrest-${PGBACKREST_VERSION} \ + && ${PACKAGER} -y clean all ; \ +fi + +# install etcdctl +RUN ETCDVERSION=3.3.27 \ + && curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \ + | tar xz -C /bin --strip=1 --wildcards --no-anchored --no-same-owner etcdctl etcd; + +# Install Patroni +RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto +#swiftclient +RUN ${PACKAGER} -y install python3-etcd python3-consul \ + python3-gevent python3-greenlet python3-cachetools \ + python3-rsa python3-pyasn1-modules python3-cffi \ + && ${PACKAGER} -y clean all; + + # python3-kazoo python3-meld3 python3-boto python3-swiftclient + +RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION + +RUN mkdir /usr/lib/postgresql +ENV PATHBACKUP = $PATH +#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin +#RUN echo $PATH +# Install pam_oauth2.so +RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \ + && make -C pam-oauth2 install \ + #&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ + && curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \ + && curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \ + && curl -sL https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz | tar xz \ + && git clone -b $SET_USER https://github.com/pgaudit/set_user.git \ + && git clone https://github.com/timescale/timescaledb.git \ + && git clone https://github.com/dimitri/pgextwlist.git \ + && git clone https://github.com/powa-team/pg_stat_kcache.git \ + && git clone https://github.com/crunchydata/pgnodemx \ + # Oracle-Client for oracle_fdw + && git clone https://github.com/laurenz/oracle_fdw.git \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-basic-21.9.0.0.0-1.el8.x86_64.rpm \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-sqlplus-21.9.0.0.0-1.el8.x86_64.rpm \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-devel-21.9.0.0.0-1.el8.x86_64.rpm \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-jdbc-21.9.0.0.0-1.el8.x86_64.rpm \ + && ${PACKAGER} install -y oracle-*.rpm \ + #&& git clone https://github.com/pgq/pgqd.git \ +\ + && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ + && ${PACKAGER} -y clean all\ +\ + # forbid creation of a main cluster when package is installed + #&& sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ +\ +# Install PostgreSQL + && for version in $PG_SUPPORTED_VERSIONS; do \ + ${PACKAGER} -y update && \ + ${PACKAGER} -y install postgresql${version}-pltcl \ +# postgresql${version}-dirtyread \ +# postgresql${version}-extra-window-functions \ +# postgresql${version}-first-last-agg \ +# postgresql${version}-hll \ +# postgresql${version}-hypopg \ + pgaudit*${version} \ +# postgresql${version}-pg-checksums \ +# postgresql${version}-pgl-ddl-deploy \ +# postgresql${version}-pglogical \ +# postgresql${version}-pglogical-ticker \ +# postgresql${version}-pgq-node \ +# postgresql${version}-pldebugger \ +# postgresql${version}-pllua \ +# postgresql${version}-plpgsql-check \ +# postgresql${version}-plproxy \ +# postgresql${version}-repack \ +# postgresql${version}-wal2json \ +# postgresql${version}-pgextwlist \ + #&& ${PACKAGER} -y clean all; \ + # Install PostgreSQL binaries, contrib, plproxy and multiple pl's + && ${PACKAGER} -y install -y postgresql${version}-contrib \ + postgresql${version}-plpython3 postgresql${version}-devel \ + pg_cron_${version} \ + #postgresql-${version}-pgq3 \ + #postgresql-${version}-pg-stat-kcache $EXTRAS \ + # Modify for using origial-spilo scripts + && ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \ + && export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \ +\ + # Install TimescaleDB + && cd /timescaledb \ + && for v in $TIMESCALEDB; do \ + git checkout $v \ + && sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt \ + && if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ + -DTAP_CHECKS=OFF -DPG_CONFIG=/usr/pgsql-$version/bin/pg_config \ + -DAPACHE_ONLY=$TIMESCALEDB_APACHE_ONLY -DSEND_TELEMETRY_DEFAULT=NO; then \ + make -C build install \ + && strip /usr/pgsql-$version/lib/timescaledb*.so; \ + fi \ + && git reset --hard \ + && git clean -f -d; \ + done \ + && cd .. \ + && for n in bg_mon-${BG_MON_COMMIT} pg_auth_mon-${PG_AUTH_MON_COMMIT} set_user pg_permissions-${PG_PERMISSIONS_COMMIT} pg_tm_aux-${PG_TM_AUX_COMMIT} pg_profile-${PG_PROFILE} $EXTRA_EXTENSIONS; do \ + make -C $n USE_PGXS=1 clean install-strip; \ + done \ + && cd /pgextwlist && make clean && make && make install \ + && cd /pg_stat_kcache && make clean && make && make install \ + && cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ + #oracle_fdw + && cd /oracle_fdw && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ + #&& cd /pgqd && ./configure --prefix=/opt && make && make install \ + && ${PACKAGER} -y clean all; \ + done + +# Clean Up git-repos +RUN rm -rf /pgextwlist \ + && rm -rf /pg_stat_kcache \ + && rm -rf /pgnodemx ; + +EXPOSE 5432 8008 8080 +ENV LC_ALL=en_US.utf-8 \ + PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ + PGHOME=/home/postgres \ + RW_DIR=/run \ + TIMESCALEDB=$TIMESCALEDB \ + DEMO=$DEMO + + + +ENV LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ + PGROOT=$PGHOME/pgdata/pgroot + +ENV PGDATA=$PGROOT/data \ + PGLOG=$PGROOT/pg_log + +WORKDIR $PGHOME + +COPY motd /etc/ +#COPY runit /etc/service/ + +COPY runit /etc/runit/runsvdir/default/ +COPY pgq_ticker.ini $PGHOME/ +RUN rm -rf /etc/service + +RUN sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \ + && chown -R postgres:postgres $PGHOME $RW_DIR \ + && rm -fr /var/spool/cron /var/tmp \ + && mkdir -p /var/spool \ + && ln -s $RW_DIR/cron /var/spool/cron \ + && ln -s $RW_DIR/tmp /var/tmp \ + && for d in /etc/runit/runsvdir/default/*; do \ + chmod 755 $d/* \ + && ln -s /run/supervise/$(basename $d) $d/supervise; \ + done \ + && ln -snf $RW_DIR/service /etc/service \ + #&& ln -s $RW_DIR/pam.d-postgresql /etc/pam.d/postgresql \ + && ln -s $RW_DIR/postgres.yml $PGHOME/postgres.yml \ + && ln -s $RW_DIR/.bash_history /root/.bash_history \ + && ln -s $RW_DIR/postgresql/.bash_history $PGHOME/.bash_history \ + && ln -s $RW_DIR/postgresql/.psql_history $PGHOME/.psql_history \ + && ln -s $RW_DIR/etc $PGHOME/etc \ + && for d in $PGHOME /root; do \ + d=$d/.config/patroni \ + && mkdir -p $d \ + && ln -s $PGHOME/postgres.yml $d/patronictl.yaml; \ + done \ + && sed -i 's/set compatible/set nocompatible/' /etc/vimrc \ + && echo "PATH=\"$PATH\"" > /etc/environment \ + && for e in TERM=linux LC_ALL=C.UTF-8 LANG=C.UTF-8 EDITOR=editor; \ + do echo "export $e" >> /etc/bash.bashrc; \ + done \ + && ln -s /etc/skel/.bashrc $PGHOME/.bashrc \ + && echo "source /etc/motd" >> /root/.bashrc \ + # Allow users in the root group to access the following files and dirs + && if [ "$COMPRESS" != "true" ]; then \ + chmod 664 /etc/passwd \ + && chmod o+r /etc/shadow \ + && chgrp -R 0 $PGHOME $RW_DIR \ + && chmod -R g=u $PGHOME $RW_DIR \ + && usermod -a -G root postgres; \ + fi + +# Remove default pgbackrest-config and spool-path +RUN rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; + +COPY scripts bootstrap major_upgrade /scripts/ +COPY launcher/postgres/launch.sh / + +#RUN dnf -y install 'dnf-command(config-manager)' +#RUN dnf -y config-manager --set-enabled crb +RUN ${PACKAGER} -y install glibc-devel glibc-static + +RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install + +# Modify for using origial-spilo scripts +RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir +RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ + +USER 26 + +#RUN chown -R postgres:root /var/lib/pgsql +CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file diff --git a/docker/postgres_stage/Dockerfile b/docker/postgres-stage/Dockerfile similarity index 79% rename from docker/postgres_stage/Dockerfile rename to docker/postgres-stage/Dockerfile index 99ddb0e..d2cd660 100644 --- a/docker/postgres_stage/Dockerfile +++ b/docker/postgres-stage/Dockerfile @@ -1,5 +1,7 @@ ARG CONTAINERSUITE +ARG BASE_IMAGE ARG BUILD +ARG PGVERSION FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as builder-false @@ -8,18 +10,17 @@ ARG PACKAGER ARG BASEOS ARG PGBACKREST_VERSION ARG PATRONI_VERSION -ARG PG_VERSION -ARG PG_MAJOR +ARG PGVERSION ARG OLD_PG_VERSIONS -#ARG PG_SUPPORTED_VERSIONS="$PG_MAJOR" -ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PG_MAJOR" +ARG PG_SUPPORTED_VERSIONS="$PGVERSION" +ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION" ARG TIMESCALEDB="1.7.5 2.3.1 2.8.1" # Spilo-specific -ENV BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ - PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ - PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ - SET_USER=REL3_0_0 \ +ENV SET_USER=REL3_0_0 \ + #BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ + #PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ + #PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ PLPROFILER=REL4_1 \ PG_PROFILE=0.3.6 \ PAM_OAUTH2=v1.0.1 \ @@ -48,7 +49,7 @@ RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ make \ cmake \ gcc \ - curl \ + #curl \ libcurl-devel \ pam-devel \ wget \ @@ -60,6 +61,7 @@ RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ krb5-devel \ busybox \ dpkg \ + jq \ && ${PACKAGER} -y clean all; @@ -93,13 +95,13 @@ RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION RUN mkdir /usr/lib/postgresql ENV PATHBACKUP = $PATH -#ENV PATH=$PATH:/usr/pgsql-$PG_MAJOR/bin +#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin #RUN echo $PATH # Install pam_oauth2.so RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \ && make -C pam-oauth2 install \ - && curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz | tar xz \ @@ -107,9 +109,16 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ && git clone https://github.com/timescale/timescaledb.git \ && git clone https://github.com/dimitri/pgextwlist.git \ && git clone https://github.com/powa-team/pg_stat_kcache.git \ + && git clone https://github.com/crunchydata/pgnodemx \ + && git clone https://github.com/laurenz/oracle_fdw.git \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-basic-21.9.0.0.0-1.el8.x86_64.rpm \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-sqlplus-21.9.0.0.0-1.el8.x86_64.rpm \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-devel-21.9.0.0.0-1.el8.x86_64.rpm \ + && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-jdbc-21.9.0.0.0-1.el8.x86_64.rpm \ + && ${PACKAGER} install -y oracle-*.rpm \ #&& git clone https://github.com/pgq/pgqd.git \ \ - && ${PACKAGER} -y install postgresql${PG_MAJOR} libevent-devel brotli-devel libbrotli \ + && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ && ${PACKAGER} -y clean all\ \ # forbid creation of a main cluster when package is installed @@ -124,7 +133,7 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ # postgresql${version}-first-last-agg \ # postgresql${version}-hll \ # postgresql${version}-hypopg \ - pgaudit1*_${version} \ + pgaudit*${version} \ # postgresql${version}-pg-checksums \ # postgresql${version}-pgl-ddl-deploy \ # postgresql${version}-pglogical \ @@ -145,8 +154,8 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ #postgresql-${version}-pgq3 \ #postgresql-${version}-pg-stat-kcache $EXTRAS \ # Modify for using origial-spilo scripts - && ln -s /usr/pgsql-$version /usr/lib/postgresql/$version \ - && export PATH=$PATHBACKUP:/usr/pgsql-$version/bin \ + && ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \ + && export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \ \ # Install TimescaleDB && cd /timescaledb \ @@ -168,6 +177,8 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ done \ && cd /pgextwlist && make && make install \ && cd /pg_stat_kcache && make clean && make && make install \ + && cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ + && cd /oracle_fdw && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ #&& cd /pgqd && ./configure --prefix=/opt && make && make install \ && ${PACKAGER} -y clean all; \ done \ @@ -198,8 +209,6 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ && find /var/log -type f -exec truncate --size 0 {} \; # Install runit -RUN dnf -y install 'dnf-command(config-manager)' -RUN dnf -y config-manager --set-enabled crb RUN dnf -y install glibc-devel glibc-static RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install @@ -208,7 +217,7 @@ RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2. RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir # Clean Up git-repos -RUN rm -rf /pgextwlist /pg_stat_kcache /package \ +RUN rm -rf /pgextwlist /pg_stat_kcache /package /oracle_fdw /pgnodemx \ && rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; # Clean Up Packages @@ -220,13 +229,12 @@ RUN ${PACKAGER} -y remove python3-pip python3-wheel python3-dev git patchutils b FROM builder-false ARG PG_VERSION -ARG PG_MAJOR +ARG PGVERSION ARG TIMESCALEDB EXPOSE 5432 8008 8080 - ENV LC_ALL=en_US.utf-8 \ - PATH=$PATH:/usr/pgsql-$PG_MAJOR/bin \ + PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ PGHOME=/home/postgres \ RW_DIR=/run \ TIMESCALEDB=$TIMESCALEDB \ @@ -287,11 +295,24 @@ RUN sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \ && usermod -a -G root postgres; \ fi - +# Remove default pgbackrest-config and spool-path +RUN rm -rf /var/spool/pgbackrest; COPY scripts bootstrap major_upgrade /scripts/ COPY launcher/postgres/launch.sh / +#1RUN dnf -y install 'dnf-command(config-manager)' +#RUN dnf -y config-manager --set-enabled crb +#RUN dnf -y install glibc-devel glibc-static + +#RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install + +# Modify for using origial-spilo scripts +#RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir + +#RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ + USER 26 +#RUN chown -R postgres:root /var/lib/pgsql CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile index 9caf4b1..2da38ed 100644 --- a/docker/postgres/Dockerfile +++ b/docker/postgres/Dockerfile @@ -8,18 +8,17 @@ ARG PACKAGER ARG BASEOS ARG PGBACKREST_VERSION ARG PATRONI_VERSION -ARG PG_VERSION -ARG PG_MAJOR +ARG PGVERSION ARG OLD_PG_VERSIONS -#ARG PG_SUPPORTED_VERSIONS="$PG_MAJOR" -ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PG_MAJOR" +ARG PG_SUPPORTED_VERSIONS="$PGVERSION" +ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION" ARG TIMESCALEDB="1.7.5 2.3.1 2.8.1" # Spilo-specific -ENV BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ - PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ - PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ - SET_USER=REL3_0_0 \ +ENV SET_USER=REL3_0_0 \ + #BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ + #PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ + #PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ PLPROFILER=REL4_1 \ PG_PROFILE=0.3.6 \ PAM_OAUTH2=v1.0.1 \ @@ -61,6 +60,7 @@ RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ busybox \ dpkg \ jq \ + rsync \ && ${PACKAGER} -y clean all; @@ -94,13 +94,13 @@ RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION RUN mkdir /usr/lib/postgresql ENV PATHBACKUP = $PATH -#ENV PATH=$PATH:/usr/pgsql-$PG_MAJOR/bin +#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin #RUN echo $PATH # Install pam_oauth2.so RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \ && make -C pam-oauth2 install \ - && curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz | tar xz \ @@ -108,9 +108,10 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ && git clone https://github.com/timescale/timescaledb.git \ && git clone https://github.com/dimitri/pgextwlist.git \ && git clone https://github.com/powa-team/pg_stat_kcache.git \ + && git clone https://github.com/crunchydata/pgnodemx \ #&& git clone https://github.com/pgq/pgqd.git \ \ - && ${PACKAGER} -y install postgresql${PG_MAJOR} libevent-devel brotli-devel libbrotli \ + && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ && ${PACKAGER} -y clean all\ \ # forbid creation of a main cluster when package is installed @@ -125,7 +126,7 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ # postgresql${version}-first-last-agg \ # postgresql${version}-hll \ # postgresql${version}-hypopg \ - pgaudit1*_${version} \ + pgaudit*${version} \ # postgresql${version}-pg-checksums \ # postgresql${version}-pgl-ddl-deploy \ # postgresql${version}-pglogical \ @@ -146,8 +147,8 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ #postgresql-${version}-pgq3 \ #postgresql-${version}-pg-stat-kcache $EXTRAS \ # Modify for using origial-spilo scripts - && ln -s /usr/pgsql-$version /usr/lib/postgresql/$version \ - && export PATH=$PATHBACKUP:/usr/pgsql-$version/bin \ + && ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \ + && export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \ \ # Install TimescaleDB && cd /timescaledb \ @@ -169,17 +170,19 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ done \ && cd /pgextwlist && make clean && make && make install \ && cd /pg_stat_kcache && make clean && make && make install \ + && cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ #&& cd /pgqd && ./configure --prefix=/opt && make && make install \ && ${PACKAGER} -y clean all; \ done # Clean Up git-repos RUN rm -rf /pgextwlist \ - && rm -rf /pg_stat_kcache; + && rm -rf /pg_stat_kcache \ + && rm -rf /pgnodemx ; EXPOSE 5432 8008 8080 ENV LC_ALL=en_US.utf-8 \ - PATH=$PATH:/usr/pgsql-$PG_MAJOR/bin \ + PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ PGHOME=/home/postgres \ RW_DIR=/run \ TIMESCALEDB=$TIMESCALEDB \ @@ -246,15 +249,14 @@ RUN rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; COPY scripts bootstrap major_upgrade /scripts/ COPY launcher/postgres/launch.sh / -RUN dnf -y install 'dnf-command(config-manager)' -RUN dnf -y config-manager --set-enabled crb -RUN dnf -y install glibc-devel glibc-static +RUN ${PACKAGER} -y install glibc-devel glibc-static RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install # Modify for using origial-spilo scripts RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir -RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ + +#RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ USER 26 diff --git a/launcher/exporter/launch.sh b/launcher/exporter/launch.sh new file mode 100755 index 0000000..5245e48 --- /dev/null +++ b/launcher/exporter/launch.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Start the Exporter + +EXPORTER_DIR='/postgres_exporter' +QUERIE_FOLDER="$EXPORTER_DIR/queries" +FILES="${QUERIE_FOLDER}/*" +touch /tmp/cpo_queries.yaml +for file in $FILES +do + if [[ -f ${file?} ]] + then + cat ${file?} >> /tmp/cpo_queries.yaml + else + echo ${FILES} + echo "Query file ${file?} does not exist (it should).." + exit 1 + fi +done + +cd postgres_exporter && ./postgres_exporter --extend.query-path=/tmp/cpo_queries.yaml #--auto-discover-databases \ No newline at end of file diff --git a/launcher/pgbackrest/launch.sh b/launcher/pgbackrest/launch.sh index 9ad3210..d7dfdce 100755 --- a/launcher/pgbackrest/launch.sh +++ b/launcher/pgbackrest/launch.sh @@ -6,19 +6,31 @@ PGBACKREST_PATH=${PGBACKREST_PATH:-'/opt/pgbackrest'} source "${PGBACKREST_PATH}/bin/shell_lib.sh" output_info "Start pgBackRest-PreCondition-Check" - -#For Restore with pgBackrest -if [ "$RESTORE_ENABLE" == "true" ]; then - output_info "pgBackRest: Restore-Job found" - source "${PGBACKREST_PATH}/bin/restore/start.sh" - output_success "Restore-Job completed" +if [ "$MODE" == "pgbackrest" ] && [ "$COMMAND" == "backup" ]; then + output_info "pgBackRest: Backup-Job found" + source "${PGBACKREST_PATH}/bin/backup/start.sh" + output_success "pgBackRest: Backup-Job completed" else -output_info "Restore not defined - Skip Restore-Step" - if [ "$RESTORE_BASEBACKUP" == "false" ]; then - output_info "pgBackRest: Backup-Job found" - source "${PGBACKREST_PATH}/bin/backup/start.sh" - output_success "Backup-Job completed" + #For Restore with pgBackrest + if [ "$RESTORE_ENABLE" == "true" ]; then + output_info "pgBackRest: Restore-Job found" + source "${PGBACKREST_PATH}/bin/restore/start.sh" + output_success "Restore-Job completed" else - output_info "Basebackup not defined - Skip create basebackup" + output_info "Restore not defined - Skip Restore-Step" + if [ "$RESTORE_BASEBACKUP" == "false" ]; then + output_info "pgBackRest: Backup-Job found" + export SELECTOR="cluster-name=${SCOPE},spilo-role=master" + export COMMAND_OPTS="--type=full --stanza=db --repo=1" + source "${PGBACKREST_PATH}/bin/backup/start.sh" + output_info "pgBackRest: Update Restore-Configmap" + configmap="${SCOPE}-pgbackrest-restore" + kubectl get cm $configmap -o yaml | \ + sed -e 's|restore_basebackup: "false"|restore_basebackup: "true"|' | \ + kubectl apply -f - + output_success "pgBackRest: Backup-Job completed" + else + output_info "Basebackup not defined - Skip create basebackup" + fi fi -fi \ No newline at end of file +fi diff --git a/scripts/configure_spilo.py b/scripts/configure_spilo.py index 90bbdc6..0a3f3ba 100755 --- a/scripts/configure_spilo.py +++ b/scripts/configure_spilo.py @@ -305,7 +305,8 @@ def deep_update(a, b): {{/SSL_CRL_FILE}} ssl_cert_file: {{SSL_CERTIFICATE_FILE}} ssl_key_file: {{SSL_PRIVATE_KEY_FILE}} - shared_preload_libraries: 'bg_mon,pg_stat_statements,pgextwlist,pg_auth_mon,set_user' + #shared_preload_libraries: 'bg_mon,pg_stat_statements,pgextwlist,pg_auth_mon,set_user' + shared_preload_libraries: 'pg_stat_statements,pgextwlist,set_user' bg_mon.listen_address: '{{BGMON_LISTEN_IP}}' bg_mon.history_buckets: 120 pg_stat_statements.track_utility: 'off' diff --git a/scripts/exporter/queries/queries.yaml b/scripts/exporter/queries/queries.yaml new file mode 100644 index 0000000..ac7abd0 --- /dev/null +++ b/scripts/exporter/queries/queries.yaml @@ -0,0 +1,253 @@ +pg_replication: + query: "SELECT CASE WHEN NOT pg_is_in_recovery() THEN 0 ELSE GREATEST (0, EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))) END AS lag, pg_is_in_recovery()::int as recovery" + master: true + metrics: + - lag: + usage: "GAUGE" + description: "Replication lag behind master in seconds" + - recovery: + usage: "GAUGE" + description: "Instance is in recovery-mode" + +pg_postmaster: + query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" + master: true + metrics: + - start_time_seconds: + usage: "GAUGE" + description: "Time at which postmaster started" + +pg_stat_user_tables: + query: | + SELECT + current_database() datname, + schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_tup_ins, + n_tup_upd, + n_tup_del, + n_tup_hot_upd, + n_live_tup, + n_dead_tup, + n_mod_since_analyze, + COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, + COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, + COALESCE(last_analyze, '1970-01-01Z') as last_analyze, + COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count + FROM + pg_stat_user_tables + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - seq_scan: + usage: "COUNTER" + description: "Number of sequential scans initiated on this table" + - seq_tup_read: + usage: "COUNTER" + description: "Number of live rows fetched by sequential scans" + - idx_scan: + usage: "COUNTER" + description: "Number of index scans initiated on this table" + - idx_tup_fetch: + usage: "COUNTER" + description: "Number of live rows fetched by index scans" + - n_tup_ins: + usage: "COUNTER" + description: "Number of rows inserted" + - n_tup_upd: + usage: "COUNTER" + description: "Number of rows updated" + - n_tup_del: + usage: "COUNTER" + description: "Number of rows deleted" + - n_tup_hot_upd: + usage: "COUNTER" + description: "Number of rows HOT updated (i.e., with no separate index update required)" + - n_live_tup: + usage: "GAUGE" + description: "Estimated number of live rows" + - n_dead_tup: + usage: "GAUGE" + description: "Estimated number of dead rows" + - n_mod_since_analyze: + usage: "GAUGE" + description: "Estimated number of rows changed since last analyze" + - last_vacuum: + usage: "GAUGE" + description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)" + - last_autovacuum: + usage: "GAUGE" + description: "Last time at which this table was vacuumed by the autovacuum daemon" + - last_analyze: + usage: "GAUGE" + description: "Last time at which this table was manually analyzed" + - last_autoanalyze: + usage: "GAUGE" + description: "Last time at which this table was analyzed by the autovacuum daemon" + - vacuum_count: + usage: "COUNTER" + description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)" + - autovacuum_count: + usage: "COUNTER" + description: "Number of times this table has been vacuumed by the autovacuum daemon" + - analyze_count: + usage: "COUNTER" + description: "Number of times this table has been manually analyzed" + - autoanalyze_count: + usage: "COUNTER" + description: "Number of times this table has been analyzed by the autovacuum daemon" + +pg_statio_user_tables: + query: "SELECT current_database() datname, schemaname, relname, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables" + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - heap_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table" + - heap_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table" + - idx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from all indexes on this table" + - idx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in all indexes on this table" + - toast_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table (if any)" + - toast_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table (if any)" + - tidx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table indexes (if any)" + - tidx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table indexes (if any)" + + +pg_process_idle: + query: | + WITH + metrics AS ( + SELECT + application_name, + SUM(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change))::bigint)::float AS process_idle_seconds_sum, + COUNT(*) AS process_idle_seconds_count + FROM pg_stat_activity + WHERE state = 'idle' + GROUP BY application_name + ), + buckets AS ( + SELECT + application_name, + le, + SUM( + CASE WHEN EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change)) <= le + THEN 1 + ELSE 0 + END + )::bigint AS bucket + FROM + pg_stat_activity, + UNNEST(ARRAY[1, 2, 5, 15, 30, 60, 90, 120, 300]) AS le + GROUP BY application_name, le + ORDER BY application_name, le + ) + SELECT + application_name, + process_idle_seconds_sum as seconds_sum, + process_idle_seconds_count as seconds_count, + ARRAY_AGG(le) AS seconds, + ARRAY_AGG(bucket) AS seconds_bucket + FROM metrics JOIN buckets USING (application_name) + GROUP BY 1, 2, 3 + metrics: + - application_name: + usage: "LABEL" + description: "Application Name" + - seconds: + usage: "HISTOGRAM" + description: "Idle time of server processes" + +#Get some Custom-Data + +cpo_pg_replication: + query: "select r.application_name as pod, r.client_addr as podip, rs.active::int , pg_wal_lsn_diff(sent_lsn, replay_lsn) as lag from pg_catalog.pg_stat_replication r Join pg_catalog.pg_replication_slots rs on r.pid = rs.active_pid where rs.slot_type='physical'" + master: true + metrics: + - pod: + usage: "LABEL" + description: Pod-Name of the replica + - podip: + usage: "LABEL" + description: Pod-Ip of the replica + - active: + usage: "GAUGE" + description: If replication-slot is active = 1 else 0 + - lag: + usage: "GAUGE" + description: "Replication lag behind master in bytes" + +cpo_pg_connections: + query: "SELECT pid, usename, state, backend_start, (Select max(extract(epoch from (clock_timestamp() -state_change))) from pg_stat_activity where state = 'idle in transaction') max_idle_in_transaction FROM pg_stat_activity;" + metrics: + - pid: + usage: "LABEL" + description: "Process-id" + - usename: + usage: "LABEL" + description: "usename" + - state: + usage: "LABEL" + description: "Connection-state" + - starttime: + usage: "GAUGE" + description: "starttime backend without timezone" + - max_idle_in_transaction: + usage: "GAUGE" + description: "time since last state change" + +cpo_stat_connections: + query: "SELECT count(pid) total_connections, + (SELECT count(pid) from pg_stat_activity where state = 'idle in transaction') as total_idle_in_transaction_connections, + (SELECT count(pid) from pg_stat_activity where state = 'idle') as total_idle_connections, + (SELECT max(extract(epoch from (clock_timestamp() -state_change))) from pg_stat_activity where state = 'idle in transaction') max_idle_in_transaction_time + from pg_stat_activity;" + metrics: + - total_connections: + usage: "GAUGE" + description: "total_connections" + - total_idle_in_transaction_connections: + usage: "GAUGE" + description: "total_idle_in_transaction_connections" + - total_idle_connections: + usage: "GAUGE" + description: "total_connections" + - max_idle_in_transaction_time: + usage: "GAUGE" + description: "max_idle_in_transaction" \ No newline at end of file diff --git a/scripts/exporter/queries/queries_nodemx.yaml b/scripts/exporter/queries/queries_nodemx.yaml new file mode 100644 index 0000000..fdd8c53 --- /dev/null +++ b/scripts/exporter/queries/queries_nodemx.yaml @@ -0,0 +1,181 @@ +# This file adds Queries using pgnodemx + +cpo_pgnodemx_network: + query: "SELECT interface,tx_bytes,tx_packets, rx_bytes,rx_packets from exporter.proc_network_stats()" + metrics: + - interface: + usage: "LABEL" + - tx_bytes: + usage: "GAUGE" + description: "Number of bytes transmitted" + - tx_packets: + usage: "GAUGE" + description: "Number of packets transmitted" + - rx_bytes: + usage: "GAUGE" + description: "Number of bytes received" + - rx_packets: + usage: "GAUGE" + description: "Number of packets received" + +cpo_pgnodemx_process: + query: "SELECT exporter.cgroup_process_count() as count" + metrics: + - count: + usage: "GAUGE" + description: "Total number of database processes" + + +cpo_pgnodemx_mem: + query: " +with d(key, val) as +(select key, val from exporter.cgroup_setof_kv('memory.stat')) +SELECT + exporter.kdapi_scalar_bigint('mem_request') as request, +case when exporter.cgroup_scalar_bigint('memory.limit_in_bytes') = 9223372036854771712 then 0 else exporter.cgroup_scalar_bigint('memory.limit_in_bytes') end as limit, + (select val from d where key='cache') as cache, + (select val from d where key='rss') as rss, + (select val from d where key='shmem') as shmem, + (select val from d where key='mapped_file') as mapped_file, + (select val from d where key='dirty') as dirty, + (select val from d where key='active_anon') as active_anon, + (select val from d where key='inactive_anon') as inactive_anon, + (select val from d where key='active_file') as active_file, + (select val from d where key='inactive_file') as inactive_file, + exporter.cgroup_scalar_bigint('memory.usage_in_bytes') as usage_in_bytes, + exporter.cgroup_scalar_bigint('memory.kmem.usage_in_bytes') as kmem_usage_in_bytes +" + metrics: + - request: + usage: "GAUGE" + description: "Memory request value in bytes" + - limit: + usage: "GAUGE" + description: "Memory limit value in bytes" + - cache: + usage: "GAUGE" + description: "Total bytes of page cache memory" + - rss: + usage: "GAUGE" + description: "Total bytes of anonymous and swap cache memory" + - shmem: + usage: "GAUGE" + description: "Total bytes of shared memory" + - mapped_file: + usage: "GAUGE" + description: "Total bytes of mapped file (includes tmpfs/shmem)" + - dirty: + usage: "GAUGE" + description: "Total bytes that are waiting to get written back to the disk" + - active_anon: + usage: "GAUGE" + description: "Total bytes of anonymous and swap cache memory on active LRU list" + - inactive_anon: + usage: "GAUGE" + description: "Total bytes of anonymous and swap cache memory on inactive LRU list" + - active_file: + usage: "GAUGE" + description: "Total bytes of file-backed memory on active LRU list" + - inactive_file: + usage: "GAUGE" + description: "Total bytes of file-backed memory on inactive LRU list" + - usage_in_bytes: + usage: "GAUGE" + description: "Total usage in bytes" + - kmem_usage_in_bytes: + usage: "GAUGE" + description: "Total kernel memory usage in bytes" + + +cpo_pgnodemx_cpu: + query: "SELECT exporter.kdapi_scalar_bigint('cpu_request') as request, exporter.kdapi_scalar_bigint('cpu_limit') as limit" + metrics: + - request: + usage: "GAUGE" + description: "CPU request value in milli cores" + - limit: + usage: "GAUGE" + description: "CPU limit value in milli cores" + +cpo_pgnodemx_cpucfs: + query: "SELECT exporter.cgroup_scalar_bigint('cpu.cfs_period_us') as period_us, + case when exporter.cgroup_scalar_bigint('cpu.cfs_quota_us') < 0 then 0 else exporter.cgroup_scalar_bigint('cpu.cfs_quota_us') end as quota_us" + metrics: + - period_us: + usage: "GAUGE" + description: "the total available run-time within a period (in microseconds)" + - quota_us: + usage: "GAUGE" + description: "the length of a period (in microseconds)" + +cpo_pgnodemx_cpuacct: + query: "SELECT exporter.cgroup_scalar_bigint('cpuacct.usage') as usage, clock_timestamp() as usage_ts" + metrics: + - usage: + usage: "GAUGE" + description: "CPU usage in nanoseconds" + - usage_ts: + usage: "GAUGE" + description: "CPU usage snapshot timestamp" + +cpo_pgnodemx_cpustat: + query: "WITH d(key, val) AS +(select key, val from exporter.cgroup_setof_kv('cpu.stat')) +SELECT + (SELECT val FROM d WHERE key='nr_periods') AS nr_periods, + (SELECT val FROM d WHERE key='nr_throttled') AS nr_throttled, + (SELECT val FROM d WHERE key='throttled_time') AS throttled_time, clock_timestamp() as snap_ts" + metrics: + - nr_periods: + usage: "GAUGE" + description: "number of periods that any thread was runnable" + - nr_throttled: + usage: "GAUGE" + description: "number of runnable periods in which the application used its entire quota and was throttled" + - throttled_time: + usage: "GAUGE" + description: "sum total amount of time individual threads within the exporter.cgroup were throttled" + - snap_ts: + usage: "GAUGE" + description: "CPU stat snapshot timestamp" + +cpo_pgnodemx_data_disk: + query: "SELECT mount_point,fs_type,total_bytes,available_bytes,total_file_nodes,free_file_nodes + FROM exporter.proc_mountinfo() m + JOIN exporter.fsinfo(m.mount_point) f USING (major_number, minor_number) + WHERE m.mount_point = '/home/postgres/pgdata'" + metrics: + - mount_point: + usage: "LABEL" + description: "mount point" + - fs_type: + usage: "LABEL" + description: "File system type" + - total_bytes: + usage: "GAUGE" + description: "Size in bytes" + - available_bytes: + usage: "GAUGE" + description: "Available size in bytes" + - total_file_nodes: + usage: "GAUGE" + description: "Total file nodes" + - free_file_nodes: + usage: "GAUGE" + description: "Available file nodes" + +cpo_pgnodemx_disk_activities: + query: "SELECT mount_point,sectors_read,sectors_written + FROM exporter.proc_mountinfo() m + JOIN exporter.proc_diskstats() d USING (major_number, minor_number) + WHERE m.mount_point IN ('/home/postgres/pgdata')" + metrics: + - mount_point: + usage: "LABEL" + description: "mount point" + - sectors_read: + usage: "GAUGE" + description: "Total sectors read" + - sectors_written: + usage: "GAUGE" + description: "Total sectors writen" \ No newline at end of file diff --git a/scripts/exporter/queries/queries_pgbackrest.yaml b/scripts/exporter/queries/queries_pgbackrest.yaml new file mode 100644 index 0000000..e412010 --- /dev/null +++ b/scripts/exporter/queries/queries_pgbackrest.yaml @@ -0,0 +1,164 @@ +# This file adds Queries to monitor pgbackrest + +cpo_pgbackrest_time_since_backup: + query: "WITH backups AS ( + SELECT jsonb_array_elements(data) AS stanza_data + FROM exporter.pgbackrestbackupinfo + ), + per_stanza AS ( + SELECT stanza_data->>'name' AS stanza, + jsonb_array_elements(stanza_data->'backup') AS backup_data, + CASE + WHEN stanza_data->'status'->>'message'='ok' THEN 1 + ELSE 0 + END + AS message + FROM backups + ), + full_bck AS ( + SELECT stanza, min(message) as message, extract(epoch from (CURRENT_TIMESTAMP - max(to_timestamp((backup_data->'timestamp'->>'stop')::bigint)))) AS time_since_full_backup_completion + FROM per_stanza + WHERE backup_data->>'type' = 'full' + GROUP BY stanza + ), + diff_bck as ( + SELECT stanza, min(message) as message, extract(epoch from (CURRENT_TIMESTAMP - max(to_timestamp((backup_data->'timestamp'->>'stop')::bigint)))) AS time_since_diff_backup_completion + FROM per_stanza + WHERE backup_data->>'type' = 'diff' + GROUP BY stanza + ), + incr_bck as ( + SELECT stanza, min(message) as message, extract(epoch from (CURRENT_TIMESTAMP - max(to_timestamp((backup_data->'timestamp'->>'stop')::bigint)))) AS time_since_incr_backup_completion + FROM per_stanza + WHERE backup_data->>'type' = 'incr' + GROUP BY stanza + ) + SELECT now() as last_check, a.stanza as stanza, coalesce(a.time_since_full_backup_completion, 0) time_since_full_backup_completion, coalesce(b.time_since_diff_backup_completion,0) time_since_diff_backup_completion, coalesce(c.time_since_incr_backup_completion,0) time_since_incr_backup_completion, LEAST(a.message, b.message, c.message) as status FROM full_bck a LEFT JOIN diff_bck b on a.stanza = b.stanza LEFT JOIN incr_bck c on a.stanza = c.stanza;" + metrics: + - last_check: + usage: "GAUGE" + description: "last_check" + - stanza: + usage: "LABEL" + description: "Stanza" + - time_since_full_backup_completion: + usage: "LABEL" + description: "Seconds since last full-backup" + - time_since_diff_backup_completion: + usage: "LABEL" + description: "Seconds since last diff-backup" + - time_since_incr_backup_completion: + usage: "LABEL" + description: "Seconds since last incr-backup" + - status: + usage: "LABEL" + description: "Min-Status per Backup based on stanza" + +cpo_pgbackrest_duration_backup: + query: "WITH backups AS ( + SELECT jsonb_array_elements(data) AS stanza_data + FROM exporter.pgbackrestbackupinfo + ) + , per_stanza AS ( + SELECT stanza_data->>'name' AS stanza, + jsonb_array_elements(stanza_data->'backup') AS backup_data + FROM backups + ) + SELECT stanza, + extract(epoch from (max(to_timestamp((backup_data->'timestamp'->>'stop')::bigint)))) AS stop, + extract(epoch from (max(to_timestamp((backup_data->'timestamp'->>'start')::bigint)))) AS start, + extract(epoch from (max(to_timestamp((backup_data->'timestamp'->>'stop')::bigint)) - max(to_timestamp((backup_data->'timestamp'->>'start')::bigint)))) AS duration, + backup_data->>'type' as type + FROM per_stanza + WHERE backup_data->>'type' IN ('full', 'diff', 'incr') + GROUP BY stanza, type;" + metrics: + - stanza: + usage: "LABEL" + description: "Stanza" + - stop: + usage: "LABEL" + description: "Stoptime as Timestamp" + - start: + usage: "LABEL" + description: "Starttime as Timestamp" + - duration: + usage: "GAUGE" + description: "Duration of the Backup" + - type: + usage: "LABEL" + description: "Backup-Type" + +cpo_pgbackrest_list_backup: + query: "WITH backups AS ( + SELECT jsonb_array_elements(data) AS stanza_data + FROM exporter.pgbackrestbackupinfo + ) + , per_stanza AS ( + SELECT stanza_data->>'name' AS stanza, + jsonb_array_elements(stanza_data->'backup') AS backup_data + FROM backups + ) + SELECT stanza, + backup_data->'database'->>'repo-key' as repo, + backup_data->>'label' as name, + backup_data->'info'->>'size' AS size, + backup_data->'info'->'repository'->>'size' AS repo_size, + backup_data->>'type' as type, + CASE + WHEN backup_data->>'error'='false' THEN -1 + ELSE -2 + END + AS bck_status + FROM per_stanza + WHERE backup_data->>'type' IN ('full', 'diff', 'incr') + GROUP BY name,backup_data,stanza;" + metrics: + - stanza: + usage: "LABEL" + description: "Stanza" + - repo: + usage: "LABEL" + description: "Reponame" + - name: + usage: "LABEL" + description: "Starttime as Timestamp" + - size: + usage: "GAUGE" + description: "Backup-Size" + - repo_size: + usage: "GAUGE" + description: "Backup-Size on Repo" + - type: + usage: "LABEL" + description: "Backup-Type" + - bck_status: + usage: "LABEL" + description: "Backup-Status" + +cpo_pgbackrest_time_since_last_backup: + query: "WITH backups AS ( + SELECT jsonb_array_elements(data) AS stanza_data + FROM exporter.pgbackrestbackupinfo + ), + per_stanza AS ( + SELECT stanza_data->>'name' AS stanza, + jsonb_array_elements(stanza_data->'backup') AS backup_data, + CASE + WHEN stanza_data->'status'->>'message'='ok' THEN 1 + ELSE 0 + END + AS message + FROM backups + ) + SELECT stanza, extract(epoch from (CURRENT_TIMESTAMP - max(to_timestamp((backup_data->'timestamp'->>'stop')::bigint)))) AS time_since_last_backup_completion + FROM per_stanza + WHERE backup_data->>'type' IN ('full','diff','incr') + GROUP BY stanza;" + metrics: + - stanza: + usage: "LABEL" + description: "Stanza" + - time_since_last_backup_completion: + usage: "GAUGE" + description: "Seconds since last backup" \ No newline at end of file diff --git a/scripts/exporter/queries/queries_statistics.yaml b/scripts/exporter/queries/queries_statistics.yaml new file mode 100644 index 0000000..2c8b4ea --- /dev/null +++ b/scripts/exporter/queries/queries_statistics.yaml @@ -0,0 +1,93 @@ +# Queries for Checking your Tables and Queries + +# PG 14 specific Queries +# Top by mean_exec_time +cpo_pg_statistics_statements_mean: + query: "SELECT queryid,pg_database.datname,rolname,max(calls),max(mean_exec_time) mean_exec_time,query,max(ROWS) rows FROM public.pg_stat_statements JOIN pg_database ON dbid = pg_database.oid JOIN pg_authid ON userid = pg_authid.oid GROUP BY 1,2,3,6 ORDER BY mean_exec_time DESC NULLS LAST LIMIT 10;" + metrics: + - queryid: + usage: "LABEL" + description: "Process-id" + - datname: + usage: "LABEL" + description: "usename" + - rolname: + usage: "LABEL" + description: "Connection-state" + - calls: + usage: "LABEL" + description: "Connection-state" + - mean_exec_time: + usage: "GAUGE" + description: "Connection-state" + - rows: + usage: "LABEL" + description: "starttime backend without timezone" + - query: + usage: "LABEL" + description: "starttime backend without timezone" + +# Top by total_exec_time +cpo_pg_statistics_statements_total: + query: "SELECT queryid,pg_database.datname,rolname,max(calls),max(total_exec_time) total_exec_time,query,max(ROWS) rows FROM public.pg_stat_statements JOIN pg_database ON dbid = pg_database.oid JOIN pg_authid ON userid = pg_authid.oid GROUP BY 1,2,3,6 ORDER BY total_exec_time DESC NULLS LAST LIMIT 10;" + metrics: + - queryid: + usage: "LABEL" + description: "Process-id" + - datname: + usage: "LABEL" + description: "usename" + - rolname: + usage: "LABEL" + description: "Connection-state" + - calls: + usage: "LABEL" + description: "Connection-state" + - total_exec_time: + usage: "GAUGE" + description: "Connection-state" + - rows: + usage: "LABEL" + description: "starttime backend without timezone" + - query: + usage: "LABEL" + description: "starttime backend without timezone" + +# top statements order by +cpo_pg_statistics_statements_write: + query: "SELECT queryid,pg_database.datname,rolname,max(calls),max(shared_blks_read)blks_read,query,max(ROWS) rows FROM public.pg_stat_statements JOIN pg_database ON dbid = pg_database.oid JOIN pg_authid ON userid = pg_authid.oid GROUP BY 1,2,3,6 ORDER BY blks_read DESC NULLS LAST LIMIT 10;" + metrics: + - datname: + usage: "LABEL" + description: "tbd" + - rolname: + usage: "LABEL" + description: "tbd" + - calls: + usage: "LABEL" + description: "tbd" + - blks_read: + usage: "GAUGE" + description: "tbd" + - rows: + usage: "LABEL" + description: "starttime backend without timezone" + - query: + usage: "LABEL" + description: "tbd" + +cpo_pg_stat_database_info: + query: "select datname, blks_read, checksum_failures, checksum_last_failure from pg_stat_database;" + metrics: + - datname: + usage: "LABEL" + description: "tbd" + - blks_read: + usage: "GAUGE" + description: "tbd" + - checksum_failures: + usage: "LABEL" + description: "tbd" + - checksum_last_failure: + usage: "LABEL" + description: "tbd" \ No newline at end of file diff --git a/scripts/exporter/storage.sh b/scripts/exporter/storage.sh new file mode 100644 index 0000000..ba58fce --- /dev/null +++ b/scripts/exporter/storage.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#get Disk-Usage as json +df -Ph | jq -R -s ' + [ + split("\n") | + .[] | + if test("^/") then + gsub(" +"; " ") | split(" ") | {mount: .[0], spacetotal: .[1], used: .[2], spaceavail: .[3], usedPercentage: .[4]} + else + empty + end + ]' \ No newline at end of file diff --git a/scripts/exporter/structure.sql b/scripts/exporter/structure.sql new file mode 100644 index 0000000..ec1be9c --- /dev/null +++ b/scripts/exporter/structure.sql @@ -0,0 +1,109 @@ + +# User + +GRANT pg_monitor TO postgres_exporter; +GRANT SELECT ON TABLE pg_authid TO postgres_exporter; + +# Structure + +CREATE SCHEMA exporter; + +ALTER SCHEMA exporter OWNER TO postgres_exporter; + +CREATE TABLE exporter.pgbackrestbackupinfo ( + name text NOT NULL, + data jsonb NOT NULL, + data_time timestamp with time zone DEFAULT now() NOT NULL +) +WITH (autovacuum_analyze_scale_factor='0', autovacuum_vacuum_scale_factor='0', autovacuum_vacuum_threshold='8', autovacuum_analyze_threshold='8'); + +ALTER TABLE exporter.pgbackrestbackupinfo OWNER TO postgres_exporter; + +CREATE TABLE exporter.storageUsage ( + name text NOT NULL, + data jsonb NOT NULL, + data_time timestamp with time zone DEFAULT now() NOT NULL +) +WITH (autovacuum_analyze_scale_factor='0', autovacuum_vacuum_scale_factor='0', autovacuum_vacuum_threshold='8', autovacuum_analyze_threshold='8'); + +ALTER TABLE exporter.storageUsage OWNER TO postgres_exporter; + +EXECUTE COPY exporter.storageUsage (config_file, data) FROM program '/home/postgres/pgbackrest/test.sh' ; + + + +CREATE or replace FUNCTION exporter.test() RETURNS boolean AS $$ + my $filename = '/home/postgres/pgbackrest/test.sh'; + if (-e $filename) { + system '/home/postgres/pgbackrest/test.sh' and return 1; + } + return; +$$ LANGUAGE plperlu; + + + + +CREATE OR REPLACE FUNCTION exporter.test() RETURNS SETOF exporter.storageUsage + LANGUAGE plpgsql + SET search_path TO 'pg_catalog', 'pg_temp' + AS $_$ +DECLARE + +BEGIN +-- Get pgBackRest info in JSON format + +-- Ensure table is empty +DELETE FROM exporter.storageUsage; + +-- Copy data into the table directory from the pgBackRest into command +COPY exporter.storageUsage (data) FROM '/home/postgres/pgdata/pgbackrest/test.sh' ; + + +RETURN QUERY SELECT * FROM exporter.storageUsage; + +IF NOT FOUND THEN + RAISE EXCEPTION 'No Data found'; +END IF; + +END +$_$; + +CREATE FUNCTION exporter.get_pgbackrest_backupdata(p_throttle_minutes integer DEFAULT 10) RETURNS SETOF exporter.pgbackrestbackupinfo + LANGUAGE plpgsql + SET search_path TO 'pg_catalog', 'pg_temp' + AS $_$ +DECLARE + +v_gather_timestamp timestamptz; +v_throttle interval; +v_system_identifier bigint; + +BEGIN +-- Get pgBackRest info in JSON format + +v_throttle := make_interval(mins := p_throttle_minutes); + +SELECT COALESCE(max(gather_timestamp), '1970-01-01'::timestamptz) INTO v_gather_timestamp FROM monitor.pgbackrest_info; + +IF pg_catalog.pg_is_in_recovery() = 'f' THEN + IF ((CURRENT_TIMESTAMP - v_gather_timestamp) > v_throttle) THEN + + -- Ensure table is empty + DELETE FROM monitor.pgbackrest_info; + + SELECT system_identifier into v_system_identifier FROM pg_control_system(); + + -- Copy data into the table directory from the pgBackRest into command + EXECUTE format( $cmd$ COPY exporter.pgbackrestbackupinfo (config_file, data) FROM program '/opt/crunchy/bin/postgres/pgbackrest_info.sh %s' WITH (format text,DELIMITER '|') $cmd$, v_system_identifier::text ); + + END IF; +END IF; + +RETURN QUERY SELECT * FROM monitor.pgbackrest_info; + +IF NOT FOUND THEN + RAISE EXCEPTION 'No backups being returned from pgbackrest info command'; +END IF; + +END +$_$; \ No newline at end of file diff --git a/scripts/exporter/version_specific/pg14/pg14_statement_queries.yaml b/scripts/exporter/version_specific/pg14/pg14_statement_queries.yaml new file mode 100644 index 0000000..7f3ab49 --- /dev/null +++ b/scripts/exporter/version_specific/pg14/pg14_statement_queries.yaml @@ -0,0 +1,34 @@ +# PG 14 specific Queries +cpo_pg_connections: + query: "SELECT queryid,pg_database.datname,rolname,calls,total_exec_time,mean_exec_time,stddev_plan_time,ROWS as rows,ROWS / calls AS rows_avg,100.0 * shared_blks_hit / nullif (shared_blks_hit + shared_blks_read, 0) AS hit_percent,query FROM public.pg_stat_statements JOIN pg_database ON dbid = pg_database.oid JOIN pg_authid ON userid = pg_authid.oid ORDER BY mean_exec_time DESC NULLS LAST LIMIT 10;" + metrics: + - queryid: + usage: "LABEL" + description: "Process-id" + - datname: + usage: "LABEL" + description: "usename" + - rolname: + usage: "LABEL" + description: "Connection-state" + - calls: + usage: "LABEL" + description: "Connection-state" + - total_exec_time: + usage: "LABEL" + description: "Connection-state" + - mean_exec_time: + usage: "GAUGE" + description: "Connection-state" + - stddev_plan_time: + usage: "LABEL" + description: "Connection-state" + - rows: + usage: "LABEL" + description: "starttime backend without timezone" + - hit_percent: + usage: "LABEL" + description: "starttime backend without timezone" + - query: + usage: "LABEL" + description: "starttime backend without timezone" diff --git a/scripts/pgbackrest/backup/start.sh b/scripts/pgbackrest/backup/start.sh index 3429d30..3473d4e 100755 --- a/scripts/pgbackrest/backup/start.sh +++ b/scripts/pgbackrest/backup/start.sh @@ -4,7 +4,7 @@ # call Primary Pod and start basebackup count=0 while true ; do - hostName=$(kubectl get pods -l cluster-name=${SCOPE},spilo-role=master -o go-template --template '{{range .items}}{{.metadata.name}} {{ break }}{{end}}') + hostName=$(kubectl get pods -l ${SELECTOR} -o go-template --template '{{range .items}}{{.metadata.name}} {{ break }}{{end}}') if [ -z "$hostName" ]; then sleep 5 output_info "pgBackRest: waiting for primary-Pod" @@ -25,11 +25,11 @@ else output_info "pgBackRest: Detect Primary-Pod $hostName" while true ; do unset error - kubectl exec $hostName -c postgres -- /bin/bash -c 'pgbackrest backup --type=full --stanza=db --repo=1' || { output_error "pgBackRest: Create basebackup failed"; error=true; } + kubectl exec $hostName -c postgres -- /bin/bash -c "pgbackrest backup ${COMMAND_OPTS}" || { output_error "pgBackRest: Create basebackup failed"; error=true; } if [ "$error" = true ]; then if [ "$count" == 3 ]; then output_error "pgBackRest: Basebackup could not be created. Abort init-script"; - exit 0 + exit 1 else ((count++)) sleep 5 @@ -39,12 +39,6 @@ else fi done - output_info "pgBackRest: Update Restore-Configmap" - configmap="${SCOPE}-pgbackrest-restore" - kubectl get cm $configmap -o yaml | \ - sed -e 's|restore_basebackup: "false"|restore_basebackup: "true"|' | \ - kubectl apply -f - - output_info "pgBackRest: Backup complete" fi From a0dac1cf763cba6bb14b42e4a897181024786757 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 2 Mar 2023 10:26:08 +0100 Subject: [PATCH 02/11] some modifications --- docker/postgres-stage/Dockerfile | 5 +++-- scripts/exporter/structure.sql | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/postgres-stage/Dockerfile b/docker/postgres-stage/Dockerfile index d2cd660..0ba0115 100644 --- a/docker/postgres-stage/Dockerfile +++ b/docker/postgres-stage/Dockerfile @@ -62,6 +62,7 @@ RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ busybox \ dpkg \ jq \ + rsync \ && ${PACKAGER} -y clean all; @@ -217,7 +218,7 @@ RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2. RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir # Clean Up git-repos -RUN rm -rf /pgextwlist /pg_stat_kcache /package /oracle_fdw /pgnodemx \ +RUN rm -rf /pgextwlist /pg_stat_kcache /oracle_fdw /pgnodemx \ && rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; # Clean Up Packages @@ -234,7 +235,7 @@ ARG TIMESCALEDB EXPOSE 5432 8008 8080 ENV LC_ALL=en_US.utf-8 \ - PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ + PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ PGHOME=/home/postgres \ RW_DIR=/run \ TIMESCALEDB=$TIMESCALEDB \ diff --git a/scripts/exporter/structure.sql b/scripts/exporter/structure.sql index ec1be9c..c47fd95 100644 --- a/scripts/exporter/structure.sql +++ b/scripts/exporter/structure.sql @@ -1,10 +1,12 @@ # User +CREATE ROLE postgres_exporter with password 'password'; +ALTER ROLE postgres_exporter login; GRANT pg_monitor TO postgres_exporter; GRANT SELECT ON TABLE pg_authid TO postgres_exporter; -# Structure +// Structure CREATE SCHEMA exporter; From f0fd98efb0cd6adf5c6299a70a63ce22916515cb Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 07:40:49 +0200 Subject: [PATCH 03/11] init --- Makefile | 53 ++++++- docker/base/Dockerfile | 11 +- docker/pgbackrest/Dockerfile | 4 +- docker/postgres-gis/Dockerfile | 241 +++++++++++++++++++++++++----- docker/postgres-oracle/Dockerfile | 3 +- docker/postgres/Dockerfile | 25 ++-- 6 files changed, 269 insertions(+), 68 deletions(-) diff --git a/Makefile b/Makefile index 1716565..f783b6d 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,12 @@ IMAGE_REPOSITORY ?= docker.io IMAGE_PATH ?= cybertec-pg-container PGVERSION ?= 15 PGVERSION_FULL ?= 15.2 -OLD_PG_VERSIONS ?= 10 11 12 13 14 -PATRONI_VERSION ?= 2.1.4 -PGBACKREST_VERSION ?= 2.41 -POSTGIS_VERSION ?= 3.2 +OLD_PG_VERSIONS ?= 11 12 13 14 +PATRONI_VERSION ?= 3.0.1 +PGBACKREST_VERSION ?= 2.44 +POSTGIS_VERSION ?= 33 PACKAGER ?= dnf -BUILD ?= 0 +BUILD ?= 1 IMAGE_TAG ?= $(BASEOS)-$(PGVERSION_FULL)-$(BUILD) POSTGIS_IMAGE_TAG ?= $(BASEOS)-$(PGVERSION_FULL)-$(POSTGIS_VERSION)-$(BUILD) @@ -29,12 +29,14 @@ base: base pgbackrest: pgbackrest postgres: base postgres postgres-stage: base postgres-stage +postgres-gis: base postgres-gis +postgres-oracle: base postgres-oracle exporter: exporter base-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/base/Dockerfile \ - --tag cybertec-pg-container/base:0.0.$(BUILD) \ + --tag cybertec-pg-container/base:$(BASEOS)-$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -45,7 +47,7 @@ base: base-build; pgbackrest-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/pgbackrest/Dockerfile \ - --tag cybertec-pg-container/pgbackrest:0.0.$(BUILD) \ + --tag cybertec-pg-container/pgbackrest:$(IMAGE_TAG)-$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -60,7 +62,7 @@ pgbackrest: pgbackrest-build; postgres-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/postgres/Dockerfile \ - --tag cybertec-pg-container/postgres:$(PGVERSION_FULL)-$(BETA)$(BUILD) \ + --tag cybertec-pg-container/postgres:$(IMAGE_TAG)-$(BETA)$(BUILD) \ --build-arg BASE_IMAGE \ --build-arg IMAGE_REPOSITORY \ --build-arg BASEOS \ @@ -91,6 +93,41 @@ postgres-stage-build: postgres-stage: postgres-stage-build +postgres-gis-build: + docker build $(ROOTPATH) \ + --file $(ROOTPATH)/docker/postgres-gis/Dockerfile \ + --tag cybertec-pg-container/postgres-gis:$(IMAGE_TAG)-$(BETA)$(BUILD) \ + --build-arg BASE_IMAGE \ + --build-arg IMAGE_REPOSITORY \ + --build-arg BASEOS \ + --build-arg PACKAGER \ + --build-arg CONTAINERSUITE \ + --build-arg BUILD \ + --build-arg PATRONI_VERSION \ + --build-arg PGBACKREST_VERSION \ + --build-arg OLD_PG_VERSIONS \ + --build-arg PGVERSION \ + --build-arg POSTGIS_VERSION + +postgres-gis: postgres-gis-build + +postgres-oracle-build: + docker build $(ROOTPATH) \ + --file $(ROOTPATH)/docker/postgres-oracle/Dockerfile \ + --tag cybertec-pg-container/postgres-oracle:$(IMAGE_TAG)-$(BETA)$(BUILD) \ + --build-arg BASE_IMAGE \ + --build-arg IMAGE_REPOSITORY \ + --build-arg BASEOS \ + --build-arg PACKAGER \ + --build-arg CONTAINERSUITE \ + --build-arg BUILD \ + --build-arg PATRONI_VERSION \ + --build-arg PGBACKREST_VERSION \ + --build-arg OLD_PG_VERSIONS \ + --build-arg PGVERSION + +postgres-oracle: postgres-oracle-build + exporter-build: docker build $(ROOTPATH) \ --file $(ROOTPATH)/docker/exporter/Dockerfile \ diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 81743a4..47eae47 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -12,11 +12,11 @@ MAINTAINER CYBERTEC PostgreSQL International GmbH LABEL vendor="CYBERTEC PostgreSQL International GmbH" url="https://www.cybertec-postgresql.com" LABEL maintainer="Matthias Groemmer " -#RUN microdnf -y install ${PACKAGER} +#RUN microdnf -y --nodocs --noplugins --setopt=install_weak_deps=0 install ${PACKAGER} -RUN ${PACKAGER} -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm vim && ${PACKAGER} -y upgrade +RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm vim && ${PACKAGER} -y upgrade -RUN ${PACKAGER} -y install --nodocs \ +RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \ bind-utils \ gettext \ hostname \ @@ -26,12 +26,13 @@ RUN ${PACKAGER} -y install --nodocs \ openssl \ vim \ glibc-langpack-en \ + #config-manager \ && ${PACKAGER} -y clean all ; #Enable CRB-Repo (Powertools) -RUN ${PACKAGER} -y config-manager --set-enabled crb +#RUN ${PACKAGER} -y config-manager --set-enabled crb #Add PostgreSQL-Repo to Base -RUN ${PACKAGER} install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm && ${PACKAGER} -y update && ${PACKAGER} -y clean all +RUN ${PACKAGER} --nodocs --noplugins --setopt=install_weak_deps=0 install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm && ${PACKAGER} -y update && ${PACKAGER} -y clean all #RUN dnf -qy module disable postgresql diff --git a/docker/pgbackrest/Dockerfile b/docker/pgbackrest/Dockerfile index a61410a..9958bc9 100644 --- a/docker/pgbackrest/Dockerfile +++ b/docker/pgbackrest/Dockerfile @@ -1,11 +1,11 @@ ARG CONTAINERSUITE ARG BUILD +ARG BASEOS -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} +FROM ${CONTAINERSUITE}/base:${BASEOS}-${BUILD} # Dockerfile specific informations ARG PACKAGER -ARG BASEOS ARG PGBACKREST_VERSION ARG PGVERSION diff --git a/docker/postgres-gis/Dockerfile b/docker/postgres-gis/Dockerfile index b03b69a..c17b28e 100644 --- a/docker/postgres-gis/Dockerfile +++ b/docker/postgres-gis/Dockerfile @@ -1,21 +1,25 @@ ARG CONTAINERSUITE +ARG BUILD +ARG BASEOS -FROM ${CONTAINERSUITE}/base:0.0.1 +FROM ${CONTAINERSUITE}/base:${BASEOS}-${BUILD} # Dockerfile specific informations ARG PACKAGER -ARG BASEOS ARG PGBACKREST_VERSION ARG PATRONI_VERSION -ARG PG_VERSION -ARG OLD_PG_VERSION -ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSION $PG_VERSION" +ARG PGVERSION +ARG OLD_PG_VERSIONS +ARG PG_SUPPORTED_VERSIONS="$PGVERSION" +ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION" +ARG TIMESCALEDB="1.7.5 2.3.1 2.8.1" +ARG POSTGIS_VERSION # Spilo-specific -ENV BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ - PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ - PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ - SET_USER=REL3_0_0 \ +ENV SET_USER=REL3_0_0 \ + #BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ + #PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ + #PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ PLPROFILER=REL4_1 \ PG_PROFILE=0.3.6 \ PAM_OAUTH2=v1.0.1 \ @@ -35,14 +39,29 @@ RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ python3-pip \ python3-psycopg2 \ python3-psutil \ + python3-requests \ + python3-etcd \ + #python3-pyyaml \ git \ patchutils \ binutils \ make \ + cmake \ gcc \ - curl \ + #curl \ libcurl-devel \ pam-devel \ + wget \ + mlocate \ + git-clang-format \ + openssl-devel \ + ccache \ + redhat-rpm-config \ + krb5-devel \ + busybox \ + dpkg \ + jq \ + rsync \ && ${PACKAGER} -y clean all; @@ -57,47 +76,191 @@ else \ && ${PACKAGER} -y clean all ; \ fi -# Remove default pgbackrest-config -RUN rm /etc/pgbackrest.conf +# install etcdctl +RUN ETCDVERSION=3.3.27 \ + && curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \ + | tar xz -C /bin --strip=1 --wildcards --no-anchored --no-same-owner etcdctl etcd; # Install Patroni -RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION +RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto +#swiftclient +RUN ${PACKAGER} -y install python3-etcd python3-consul \ + python3-gevent python3-greenlet python3-cachetools \ + python3-rsa python3-pyasn1-modules python3-cffi \ + && ${PACKAGER} -y clean all; + # python3-kazoo python3-meld3 python3-boto python3-swiftclient + +RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION + +RUN mkdir /usr/lib/postgresql +ENV PATHBACKUP = $PATH +#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin +#RUN echo $PATH # Install pam_oauth2.so RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \ && make -C pam-oauth2 install \ - && curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ + #&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \ && curl -sL https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz | tar xz \ && git clone -b $SET_USER https://github.com/pgaudit/set_user.git \ && git clone https://github.com/timescale/timescaledb.git \ - && ${PACKAGER} -y install postgresql libevent brotli libbrotli \ - + && git clone https://github.com/dimitri/pgextwlist.git \ + && git clone https://github.com/powa-team/pg_stat_kcache.git \ + && git clone https://github.com/crunchydata/pgnodemx \ + #&& git clone https://github.com/pgq/pgqd.git \ +\ + && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ + && ${PACKAGER} -y clean all\ +\ + # forbid creation of a main cluster when package is installed + #&& sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ +\ # Install PostgreSQL -RUN for version in $PG_SUPPORTED_VERSIONS; do \ + && for version in $PG_SUPPORTED_VERSIONS; do \ ${PACKAGER} -y update && \ - ${PACKAGER} -y install postgresql-pltcl-${version} \ + ${PACKAGER} -y install postgresql${version}-pltcl \ # postgresql${version}-dirtyread \ - postgresql${version}-extra-window-functions \ - postgresql${version}-first-last-agg \ - postgresql${version}-hll \ - postgresql${version}-hypopg \ - postgresql${version}-pgaudit \ - postgresql${version}-pg-checksums \ - postgresql${version}-pgl-ddl-deploy \ - postgresql${version}-pglogical \ - postgresql${version}-pglogical-ticker \ - postgresql${version}-pgq-node \ - postgresql${version}-pldebugger \ - postgresql${version}-pllua \ - postgresql${version}-plpgsql-check \ - postgresql${version}-plproxy \ - postgis${POSTGIS_VERSION%.*}_${version}*\ -# postgis-${POSTGIS_VERSION%.*}_${version}-scripts \ - postgresql${version}-repack \ - postgresql${version}-wal2json \ - postgresql${version}-pgextwlist \ +# postgresql${version}-extra-window-functions \ +# postgresql${version}-first-last-agg \ +# postgresql${version}-hll \ +# postgresql${version}-hypopg \ + postgis${POSTGIS_VERSION}_${version} \ + pgaudit*${version} \ +# postgresql${version}-pg-checksums \ +# postgresql${version}-pgl-ddl-deploy \ +# postgresql${version}-pglogical \ +# postgresql${version}-pglogical-ticker \ +# postgresql${version}-pgq-node \ +# postgresql${version}-pldebugger \ +# postgresql${version}-pllua \ +# postgresql${version}-plpgsql-check \ +# postgresql${version}-plproxy \ +# postgresql${version}-repack \ +# postgresql${version}-wal2json \ +# postgresql${version}-pgextwlist \ + #&& ${PACKAGER} -y clean all; \ + # Install PostgreSQL binaries, contrib, plproxy and multiple pl's + && ${PACKAGER} -y install -y postgresql${version}-contrib \ + postgresql${version}-plpython3 postgresql${version}-devel \ + pg_cron_${version} \ + #postgresql-${version}-pgq3 \ + #postgresql-${version}-pg-stat-kcache $EXTRAS \ + # Modify for using origial-spilo scripts + && ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \ + && export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \ +\ + # Install TimescaleDB + && cd /timescaledb \ + && for v in $TIMESCALEDB; do \ + git checkout $v \ + && sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt \ + && if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ + -DTAP_CHECKS=OFF -DPG_CONFIG=/usr/pgsql-$version/bin/pg_config \ + -DAPACHE_ONLY=$TIMESCALEDB_APACHE_ONLY -DSEND_TELEMETRY_DEFAULT=NO; then \ + make -C build install \ + && strip /usr/pgsql-$version/lib/timescaledb*.so; \ + fi \ + && git reset --hard \ + && git clean -f -d; \ + done \ + && cd .. \ + && for n in bg_mon-${BG_MON_COMMIT} pg_auth_mon-${PG_AUTH_MON_COMMIT} set_user pg_permissions-${PG_PERMISSIONS_COMMIT} pg_tm_aux-${PG_TM_AUX_COMMIT} pg_profile-${PG_PROFILE} $EXTRA_EXTENSIONS; do \ + make -C $n USE_PGXS=1 clean install-strip; \ + done \ + && cd /pgextwlist && make clean && make && make install \ + && cd /pg_stat_kcache && make clean && make && make install \ + && cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ + #&& cd /pgqd && ./configure --prefix=/opt && make && make install \ && ${PACKAGER} -y clean all; \ - done \ No newline at end of file + done + +# Clean Up git-repos +RUN rm -rf /pgextwlist \ + && rm -rf /pg_stat_kcache \ + && rm -rf /pgnodemx ; + +EXPOSE 5432 8008 8080 +ENV LC_ALL=en_US.utf-8 \ + PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ + PGHOME=/home/postgres \ + RW_DIR=/run \ + TIMESCALEDB=$TIMESCALEDB \ + DEMO=$DEMO + + + +ENV LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ + PGROOT=$PGHOME/pgdata/pgroot + +ENV PGDATA=$PGROOT/data \ + PGLOG=$PGROOT/pg_log + +WORKDIR $PGHOME + +COPY motd /etc/ +#COPY runit /etc/service/ + +COPY runit /etc/runit/runsvdir/default/ +COPY pgq_ticker.ini $PGHOME/ +RUN rm -rf /etc/service + +RUN sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \ + && chown -R postgres:postgres $PGHOME $RW_DIR \ + && rm -fr /var/spool/cron /var/tmp \ + && mkdir -p /var/spool \ + && ln -s $RW_DIR/cron /var/spool/cron \ + && ln -s $RW_DIR/tmp /var/tmp \ + && for d in /etc/runit/runsvdir/default/*; do \ + chmod 755 $d/* \ + && ln -s /run/supervise/$(basename $d) $d/supervise; \ + done \ + && ln -snf $RW_DIR/service /etc/service \ + #&& ln -s $RW_DIR/pam.d-postgresql /etc/pam.d/postgresql \ + && ln -s $RW_DIR/postgres.yml $PGHOME/postgres.yml \ + && ln -s $RW_DIR/.bash_history /root/.bash_history \ + && ln -s $RW_DIR/postgresql/.bash_history $PGHOME/.bash_history \ + && ln -s $RW_DIR/postgresql/.psql_history $PGHOME/.psql_history \ + && ln -s $RW_DIR/etc $PGHOME/etc \ + && for d in $PGHOME /root; do \ + d=$d/.config/patroni \ + && mkdir -p $d \ + && ln -s $PGHOME/postgres.yml $d/patronictl.yaml; \ + done \ + && sed -i 's/set compatible/set nocompatible/' /etc/vimrc \ + && echo "PATH=\"$PATH\"" > /etc/environment \ + && for e in TERM=linux LC_ALL=C.UTF-8 LANG=C.UTF-8 EDITOR=editor; \ + do echo "export $e" >> /etc/bash.bashrc; \ + done \ + && ln -s /etc/skel/.bashrc $PGHOME/.bashrc \ + && echo "source /etc/motd" >> /root/.bashrc \ + # Allow users in the root group to access the following files and dirs + && if [ "$COMPRESS" != "true" ]; then \ + chmod 664 /etc/passwd \ + && chmod o+r /etc/shadow \ + && chgrp -R 0 $PGHOME $RW_DIR \ + && chmod -R g=u $PGHOME $RW_DIR \ + && usermod -a -G root postgres; \ + fi + +# Remove default pgbackrest-config and spool-path +RUN rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; + +COPY scripts bootstrap major_upgrade /scripts/ +COPY launcher/postgres/launch.sh / + +RUN ${PACKAGER} -y install glibc-devel glibc-static + +RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install + +# Modify for using origial-spilo scripts +RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir + +#RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ + +USER 26 + +#RUN chown -R postgres:root /var/lib/pgsql +CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file diff --git a/docker/postgres-oracle/Dockerfile b/docker/postgres-oracle/Dockerfile index eea85bc..49742de 100644 --- a/docker/postgres-oracle/Dockerfile +++ b/docker/postgres-oracle/Dockerfile @@ -1,7 +1,8 @@ ARG CONTAINERSUITE ARG BUILD +ARG BASEOS -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} +FROM ${CONTAINERSUITE}/base:${BASEOS}-${BUILD} # Dockerfile specific informations ARG PACKAGER diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile index 2da38ed..751a486 100644 --- a/docker/postgres/Dockerfile +++ b/docker/postgres/Dockerfile @@ -1,11 +1,11 @@ ARG CONTAINERSUITE ARG BUILD +ARG BASEOS -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} +FROM ${CONTAINERSUITE}/base:${BASEOS}-${BUILD} # Dockerfile specific informations ARG PACKAGER -ARG BASEOS ARG PGBACKREST_VERSION ARG PATRONI_VERSION ARG PGVERSION @@ -27,7 +27,7 @@ ENV SET_USER=REL3_0_0 \ PG_TM_AUX_COMMIT=6c012d38a4c1b0ba4a36952d60b0ce3a22ac9c3d # Get some Standard-Stuff -RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ +RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \ openssh-clients \ openssh-server \ shadow-utils \ @@ -65,11 +65,11 @@ RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ RUN if [ "$BASEOS" = "ubi8" ] ; then \ - ${PACKAGER} -y install --nodocs \ + ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \ pgbackrest-${PGBACKREST_VERSION} \ && ${PACKAGER} -y clean all ; \ else \ - ${PACKAGER} -y install --nodocs \ + ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \ --setopt=skip_missing_names_on_install=False \ pgbackrest-${PGBACKREST_VERSION} \ && ${PACKAGER} -y clean all ; \ @@ -83,9 +83,10 @@ RUN ETCDVERSION=3.3.27 \ # Install Patroni RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto #swiftclient -RUN ${PACKAGER} -y install python3-etcd python3-consul \ - python3-gevent python3-greenlet python3-cachetools \ - python3-rsa python3-pyasn1-modules python3-cffi \ +RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \ + python3-etcd python3-consul \ + python3-gevent python3-greenlet python3-cachetools \ + python3-rsa python3-pyasn1-modules python3-cffi \ && ${PACKAGER} -y clean all; # python3-kazoo python3-meld3 python3-boto python3-swiftclient @@ -111,7 +112,7 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ && git clone https://github.com/crunchydata/pgnodemx \ #&& git clone https://github.com/pgq/pgqd.git \ \ - && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ + && ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ && ${PACKAGER} -y clean all\ \ # forbid creation of a main cluster when package is installed @@ -138,7 +139,6 @@ RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/ # postgresql${version}-plproxy \ # postgresql${version}-repack \ # postgresql${version}-wal2json \ -# postgresql${version}-pgextwlist \ #&& ${PACKAGER} -y clean all; \ # Install PostgreSQL binaries, contrib, plproxy and multiple pl's && ${PACKAGER} -y install -y postgresql${version}-contrib \ @@ -188,8 +188,6 @@ ENV LC_ALL=en_US.utf-8 \ TIMESCALEDB=$TIMESCALEDB \ DEMO=$DEMO - - ENV LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ PGROOT=$PGHOME/pgdata/pgroot @@ -249,7 +247,8 @@ RUN rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; COPY scripts bootstrap major_upgrade /scripts/ COPY launcher/postgres/launch.sh / -RUN ${PACKAGER} -y install glibc-devel glibc-static +RUN ${PACKAGER} -y install --nodocs --noplugins --setopt=install_weak_deps=0 \ + glibc-devel glibc-static RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install From 0c5de335f9ed6a1b8b5f0f4d1ca62d4ecd55db22 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:11:49 +0200 Subject: [PATCH 04/11] init --- docker/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docker/.gitignore diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 0000000..00c6541 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,3 @@ +# exclude some folders +!postgres-stage +|postgres-oracle \ No newline at end of file From 962ca93c467336e825ed3aef6d7dccefbe32b9ea Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:13:08 +0200 Subject: [PATCH 05/11] init --- .gitignore | 3 +++ docker/.gitignore | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 .gitignore delete mode 100644 docker/.gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..654f504 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# exclude some folders +!/docker/postgres-stage/* +!/docker/postgres-oracle/* \ No newline at end of file diff --git a/docker/.gitignore b/docker/.gitignore deleted file mode 100644 index 00c6541..0000000 --- a/docker/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# exclude some folders -!postgres-stage -|postgres-oracle \ No newline at end of file From 994531bd3056be9b19d91e15b8ac0d428a8be9c1 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:13:46 +0200 Subject: [PATCH 06/11] init --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 654f504..8ec0796 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ # exclude some folders -!/docker/postgres-stage/* -!/docker/postgres-oracle/* \ No newline at end of file +/docker/postgres-stage/* +/docker/postgres-oracle/* \ No newline at end of file From b1dec86a43356848254e714bce2eedc12f579a82 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:15:26 +0200 Subject: [PATCH 07/11] init --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8ec0796..c50ffd0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ # exclude some folders -/docker/postgres-stage/* -/docker/postgres-oracle/* \ No newline at end of file +docker/postgres-stage/ +docker/postgres-oracle/ \ No newline at end of file From bd00ae73b1076b27ba7980122242a718c163fd01 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:16:39 +0200 Subject: [PATCH 08/11] init --- {docker/benchmark => benchmark}/Dockerfile | 0 docker/newpg/Dockerfile | 134 --------- docker/postgres-oracle/Dockerfile | 275 ------------------ docker/postgres-stage/Dockerfile | 319 --------------------- 4 files changed, 728 deletions(-) rename {docker/benchmark => benchmark}/Dockerfile (100%) delete mode 100644 docker/newpg/Dockerfile delete mode 100644 docker/postgres-oracle/Dockerfile delete mode 100644 docker/postgres-stage/Dockerfile diff --git a/docker/benchmark/Dockerfile b/benchmark/Dockerfile similarity index 100% rename from docker/benchmark/Dockerfile rename to benchmark/Dockerfile diff --git a/docker/newpg/Dockerfile b/docker/newpg/Dockerfile deleted file mode 100644 index 3a1d7b3..0000000 --- a/docker/newpg/Dockerfile +++ /dev/null @@ -1,134 +0,0 @@ -ARG BASE_IMAGE=ubuntu:22.04 -ARG PGVERSION=15 -ARG TIMESCALEDB="1.7.5 2.3.1 2.9.2" -ARG DEMO=false -ARG COMPRESS=false -ARG ADDITIONAL_LOCALES= -ARG CONTAINERSUITE -ARG BUILD - - -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as base - -ARG ADDITIONAL_LOCALES - -COPY build_scripts/locales.sh /builddeps/ - -RUN bash /builddeps/locales.sh - - -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as dependencies-builder - -ARG DEMO - -ENV WALG_VERSION=v2.0.1 - -COPY build_scripts/dependencies.sh /builddeps/ - -COPY dependencies/debs /builddeps/ - -RUN bash /builddeps/dependencies.sh - - -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as builder-false - -ARG DEMO -ARG ADDITIONAL_LOCALES - -COPY build_scripts/prepare.sh build_scripts/locales.sh /builddeps/ - -RUN bash /builddeps/prepare.sh - -COPY --from=base /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.18 - -COPY cron_unprivileged.c dependencies/src /builddeps/ -COPY build_scripts/base.sh /builddeps/ -COPY --from=dependencies-builder /builddeps/*.deb /builddeps/ - -ARG PGVERSION -ARG TIMESCALEDB -ARG TIMESCALEDB_APACHE_ONLY=true -ARG TIMESCALEDB_TOOLKIT=true -ARG COMPRESS -ARG PGOLDVERSIONS="10 11 12 13 14" -ARG WITH_PERL=false - -ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" - -# Install PostgreSQL, extensions and contribs -ENV POSTGIS_VERSION=3.3 \ - POSTGIS_LEGACY=3.2 \ - BG_MON_COMMIT=241d8134e4dda4ffe6f21d40abf8d544d78bc9d8 \ - PG_AUTH_MON_COMMIT=439697fe2980cf48f1760f45e04c2d69b2748e73 \ - PG_MON_COMMIT=34c35a86e6dea26930bf1f770048cea2c695b80b \ - SET_USER=REL3_0_0 \ - PLPROFILER=REL4_2_1 \ - PG_PROFILE=4.1 \ - PAM_OAUTH2=v1.0.1 \ - PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \ - PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \ - PG_TM_AUX_COMMIT=51cf42f6043608919802fcc48458471d0ea3a127 - -WORKDIR /builddeps -RUN bash base.sh - -# Install wal-g -COPY --from=dependencies-builder /builddeps/wal-g /usr/local/bin/ - -COPY build_scripts/patroni_wale.sh build_scripts/compress_build.sh /builddeps/ - -# Install patroni and wal-e -ENV PATRONIVERSION=3.0.0 -ENV WALE_VERSION=1.1.1 - -WORKDIR / - -RUN bash /builddeps/patroni_wale.sh - -RUN if [ "$COMPRESS" = "true" ]; then bash /builddeps/compress_build.sh; fi - - -FROM scratch as builder-true -COPY --from=builder-false / / - - -FROM builder-${COMPRESS} - -LABEL maintainer="Polina Bungina " - -ARG PGVERSION -ARG TIMESCALEDB -ARG DEMO -ARG COMPRESS - -EXPOSE 5432 8008 8080 - -ENV LC_ALL=en_US.utf-8 \ - PATH=$PATH:/usr/lib/postgresql/$PGVERSION/bin \ - PGHOME=/home/postgres \ - RW_DIR=/run \ - TIMESCALEDB=$TIMESCALEDB \ - DEMO=$DEMO - -ENV WALE_ENV_DIR=$RW_DIR/etc/wal-e.d/env \ - LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ - PGROOT=$PGHOME/pgdata/pgroot - -ENV PGDATA=$PGROOT/data \ - PGLOG=$PGROOT/pg_log - -ENV USE_OLD_LOCALES=false - -WORKDIR $PGHOME - -COPY motd /etc/ -COPY runit /etc/service/ -COPY pgq_ticker.ini $PGHOME/ -COPY build_scripts/post_build.sh /builddeps/ - -RUN sh /builddeps/post_build.sh && rm -rf /builddeps/ - -COPY scripts bootstrap major_upgrade /scripts/ -COPY launch.sh / - -CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file diff --git a/docker/postgres-oracle/Dockerfile b/docker/postgres-oracle/Dockerfile deleted file mode 100644 index 49742de..0000000 --- a/docker/postgres-oracle/Dockerfile +++ /dev/null @@ -1,275 +0,0 @@ -ARG CONTAINERSUITE -ARG BUILD -ARG BASEOS - -FROM ${CONTAINERSUITE}/base:${BASEOS}-${BUILD} - -# Dockerfile specific informations -ARG PACKAGER -ARG BASEOS -ARG PGBACKREST_VERSION -ARG PATRONI_VERSION -ARG PGVERSION -ARG OLD_PG_VERSIONS -ARG PG_SUPPORTED_VERSIONS="$PGVERSION" -ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION" -ARG TIMESCALEDB="1.7.5 2.3.1 2.8.1" - -# Spilo-specific -ENV SET_USER=REL3_0_0 \ - #BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ - #PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ - #PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ - PLPROFILER=REL4_1 \ - PG_PROFILE=0.3.6 \ - PAM_OAUTH2=v1.0.1 \ - PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \ - PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \ - PG_TM_AUX_COMMIT=6c012d38a4c1b0ba4a36952d60b0ce3a22ac9c3d - -# Get some Standard-Stuff -RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ - openssh-clients \ - openssh-server \ - shadow-utils \ - tar \ - bzip2 \ - lz4 \ - python3 \ - python3-pip \ - python3-psycopg2 \ - python3-psutil \ - python3-requests \ - python3-etcd \ - #python3-pyyaml \ - git \ - patchutils \ - binutils \ - make \ - cmake \ - gcc \ - #curl \ - libcurl-devel \ - pam-devel \ - wget \ - mlocate \ - git-clang-format \ - openssl-devel \ - ccache \ - redhat-rpm-config \ - krb5-devel \ - busybox \ - dpkg \ - jq \ - rsync \ - && ${PACKAGER} -y clean all; - - -RUN if [ "$BASEOS" = "ubi8" ] ; then \ - ${PACKAGER} -y install --nodocs \ - pgbackrest-${PGBACKREST_VERSION} \ - && ${PACKAGER} -y clean all ; \ -else \ - ${PACKAGER} -y install --nodocs \ - --setopt=skip_missing_names_on_install=False \ - pgbackrest-${PGBACKREST_VERSION} \ - && ${PACKAGER} -y clean all ; \ -fi - -# install etcdctl -RUN ETCDVERSION=3.3.27 \ - && curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \ - | tar xz -C /bin --strip=1 --wildcards --no-anchored --no-same-owner etcdctl etcd; - -# Install Patroni -RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto -#swiftclient -RUN ${PACKAGER} -y install python3-etcd python3-consul \ - python3-gevent python3-greenlet python3-cachetools \ - python3-rsa python3-pyasn1-modules python3-cffi \ - && ${PACKAGER} -y clean all; - - # python3-kazoo python3-meld3 python3-boto python3-swiftclient - -RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION - -RUN mkdir /usr/lib/postgresql -ENV PATHBACKUP = $PATH -#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin -#RUN echo $PATH -# Install pam_oauth2.so -RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \ - && make -C pam-oauth2 install \ - #&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ - #&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz | tar xz \ - && git clone -b $SET_USER https://github.com/pgaudit/set_user.git \ - && git clone https://github.com/timescale/timescaledb.git \ - && git clone https://github.com/dimitri/pgextwlist.git \ - && git clone https://github.com/powa-team/pg_stat_kcache.git \ - && git clone https://github.com/crunchydata/pgnodemx \ - # Oracle-Client for oracle_fdw - && git clone https://github.com/laurenz/oracle_fdw.git \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-basic-21.9.0.0.0-1.el8.x86_64.rpm \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-sqlplus-21.9.0.0.0-1.el8.x86_64.rpm \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-devel-21.9.0.0.0-1.el8.x86_64.rpm \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-jdbc-21.9.0.0.0-1.el8.x86_64.rpm \ - && ${PACKAGER} install -y oracle-*.rpm \ - #&& git clone https://github.com/pgq/pgqd.git \ -\ - && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ - && ${PACKAGER} -y clean all\ -\ - # forbid creation of a main cluster when package is installed - #&& sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ -\ -# Install PostgreSQL - && for version in $PG_SUPPORTED_VERSIONS; do \ - ${PACKAGER} -y update && \ - ${PACKAGER} -y install postgresql${version}-pltcl \ -# postgresql${version}-dirtyread \ -# postgresql${version}-extra-window-functions \ -# postgresql${version}-first-last-agg \ -# postgresql${version}-hll \ -# postgresql${version}-hypopg \ - pgaudit*${version} \ -# postgresql${version}-pg-checksums \ -# postgresql${version}-pgl-ddl-deploy \ -# postgresql${version}-pglogical \ -# postgresql${version}-pglogical-ticker \ -# postgresql${version}-pgq-node \ -# postgresql${version}-pldebugger \ -# postgresql${version}-pllua \ -# postgresql${version}-plpgsql-check \ -# postgresql${version}-plproxy \ -# postgresql${version}-repack \ -# postgresql${version}-wal2json \ -# postgresql${version}-pgextwlist \ - #&& ${PACKAGER} -y clean all; \ - # Install PostgreSQL binaries, contrib, plproxy and multiple pl's - && ${PACKAGER} -y install -y postgresql${version}-contrib \ - postgresql${version}-plpython3 postgresql${version}-devel \ - pg_cron_${version} \ - #postgresql-${version}-pgq3 \ - #postgresql-${version}-pg-stat-kcache $EXTRAS \ - # Modify for using origial-spilo scripts - && ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \ - && export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \ -\ - # Install TimescaleDB - && cd /timescaledb \ - && for v in $TIMESCALEDB; do \ - git checkout $v \ - && sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt \ - && if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ - -DTAP_CHECKS=OFF -DPG_CONFIG=/usr/pgsql-$version/bin/pg_config \ - -DAPACHE_ONLY=$TIMESCALEDB_APACHE_ONLY -DSEND_TELEMETRY_DEFAULT=NO; then \ - make -C build install \ - && strip /usr/pgsql-$version/lib/timescaledb*.so; \ - fi \ - && git reset --hard \ - && git clean -f -d; \ - done \ - && cd .. \ - && for n in bg_mon-${BG_MON_COMMIT} pg_auth_mon-${PG_AUTH_MON_COMMIT} set_user pg_permissions-${PG_PERMISSIONS_COMMIT} pg_tm_aux-${PG_TM_AUX_COMMIT} pg_profile-${PG_PROFILE} $EXTRA_EXTENSIONS; do \ - make -C $n USE_PGXS=1 clean install-strip; \ - done \ - && cd /pgextwlist && make clean && make && make install \ - && cd /pg_stat_kcache && make clean && make && make install \ - && cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ - #oracle_fdw - && cd /oracle_fdw && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ - #&& cd /pgqd && ./configure --prefix=/opt && make && make install \ - && ${PACKAGER} -y clean all; \ - done - -# Clean Up git-repos -RUN rm -rf /pgextwlist \ - && rm -rf /pg_stat_kcache \ - && rm -rf /pgnodemx ; - -EXPOSE 5432 8008 8080 -ENV LC_ALL=en_US.utf-8 \ - PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ - PGHOME=/home/postgres \ - RW_DIR=/run \ - TIMESCALEDB=$TIMESCALEDB \ - DEMO=$DEMO - - - -ENV LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ - PGROOT=$PGHOME/pgdata/pgroot - -ENV PGDATA=$PGROOT/data \ - PGLOG=$PGROOT/pg_log - -WORKDIR $PGHOME - -COPY motd /etc/ -#COPY runit /etc/service/ - -COPY runit /etc/runit/runsvdir/default/ -COPY pgq_ticker.ini $PGHOME/ -RUN rm -rf /etc/service - -RUN sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \ - && chown -R postgres:postgres $PGHOME $RW_DIR \ - && rm -fr /var/spool/cron /var/tmp \ - && mkdir -p /var/spool \ - && ln -s $RW_DIR/cron /var/spool/cron \ - && ln -s $RW_DIR/tmp /var/tmp \ - && for d in /etc/runit/runsvdir/default/*; do \ - chmod 755 $d/* \ - && ln -s /run/supervise/$(basename $d) $d/supervise; \ - done \ - && ln -snf $RW_DIR/service /etc/service \ - #&& ln -s $RW_DIR/pam.d-postgresql /etc/pam.d/postgresql \ - && ln -s $RW_DIR/postgres.yml $PGHOME/postgres.yml \ - && ln -s $RW_DIR/.bash_history /root/.bash_history \ - && ln -s $RW_DIR/postgresql/.bash_history $PGHOME/.bash_history \ - && ln -s $RW_DIR/postgresql/.psql_history $PGHOME/.psql_history \ - && ln -s $RW_DIR/etc $PGHOME/etc \ - && for d in $PGHOME /root; do \ - d=$d/.config/patroni \ - && mkdir -p $d \ - && ln -s $PGHOME/postgres.yml $d/patronictl.yaml; \ - done \ - && sed -i 's/set compatible/set nocompatible/' /etc/vimrc \ - && echo "PATH=\"$PATH\"" > /etc/environment \ - && for e in TERM=linux LC_ALL=C.UTF-8 LANG=C.UTF-8 EDITOR=editor; \ - do echo "export $e" >> /etc/bash.bashrc; \ - done \ - && ln -s /etc/skel/.bashrc $PGHOME/.bashrc \ - && echo "source /etc/motd" >> /root/.bashrc \ - # Allow users in the root group to access the following files and dirs - && if [ "$COMPRESS" != "true" ]; then \ - chmod 664 /etc/passwd \ - && chmod o+r /etc/shadow \ - && chgrp -R 0 $PGHOME $RW_DIR \ - && chmod -R g=u $PGHOME $RW_DIR \ - && usermod -a -G root postgres; \ - fi - -# Remove default pgbackrest-config and spool-path -RUN rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; - -COPY scripts bootstrap major_upgrade /scripts/ -COPY launcher/postgres/launch.sh / - -#RUN dnf -y install 'dnf-command(config-manager)' -#RUN dnf -y config-manager --set-enabled crb -RUN ${PACKAGER} -y install glibc-devel glibc-static - -RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install - -# Modify for using origial-spilo scripts -RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir -RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ - -USER 26 - -#RUN chown -R postgres:root /var/lib/pgsql -CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file diff --git a/docker/postgres-stage/Dockerfile b/docker/postgres-stage/Dockerfile deleted file mode 100644 index 0ba0115..0000000 --- a/docker/postgres-stage/Dockerfile +++ /dev/null @@ -1,319 +0,0 @@ -ARG CONTAINERSUITE -ARG BASE_IMAGE -ARG BUILD -ARG PGVERSION - -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} as builder-false - -# Dockerfile specific informations -ARG PACKAGER -ARG BASEOS -ARG PGBACKREST_VERSION -ARG PATRONI_VERSION -ARG PGVERSION -ARG OLD_PG_VERSIONS -ARG PG_SUPPORTED_VERSIONS="$PGVERSION" -ARG PG_SUPPORTED_VERSIONS="$OLD_PG_VERSIONS $PGVERSION" -ARG TIMESCALEDB="1.7.5 2.3.1 2.8.1" - -# Spilo-specific -ENV SET_USER=REL3_0_0 \ - #BG_MON_COMMIT=e22182e129ce357b5aa80090fba2aacfc42b1996 \ - #PG_AUTH_MON_COMMIT=52d90bbaa6d3c61195cd9532ebe036b1183808c5 \ - #PG_MON_COMMIT=54fbdcc3cfe7e2a626bd96dda644d9a0c6866b58 \ - PLPROFILER=REL4_1 \ - PG_PROFILE=0.3.6 \ - PAM_OAUTH2=v1.0.1 \ - PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \ - PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \ - PG_TM_AUX_COMMIT=6c012d38a4c1b0ba4a36952d60b0ce3a22ac9c3d - -# Get some Standard-Stuff -RUN ${PACKAGER} -y update && ${PACKAGER} -y install --nodocs \ - openssh-clients \ - openssh-server \ - shadow-utils \ - tar \ - bzip2 \ - lz4 \ - python3 \ - python3-pip \ - python3-psycopg2 \ - python3-psutil \ - python3-requests \ - python3-etcd \ - #python3-pyyaml \ - git \ - patchutils \ - binutils \ - make \ - cmake \ - gcc \ - #curl \ - libcurl-devel \ - pam-devel \ - wget \ - mlocate \ - git-clang-format \ - openssl-devel \ - ccache \ - redhat-rpm-config \ - krb5-devel \ - busybox \ - dpkg \ - jq \ - rsync \ - && ${PACKAGER} -y clean all; - - -RUN if [ "$BASEOS" = "ubi8" ] ; then \ - ${PACKAGER} -y install --nodocs \ - pgbackrest-${PGBACKREST_VERSION} \ - && ${PACKAGER} -y clean all ; \ -else \ - ${PACKAGER} -y install --nodocs \ - --setopt=skip_missing_names_on_install=False \ - pgbackrest-${PGBACKREST_VERSION} \ - && ${PACKAGER} -y clean all ; \ -fi - -# install etcdctl -RUN ETCDVERSION=3.3.27 \ - && curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \ - | tar xz -C /bin --strip=1 --wildcards --no-anchored --no-same-owner etcdctl etcd; - -# Install Patroni -RUN pip3 install 'PyYAML<6.0' setuptools pystache loader dumb-init kazoo meld3 boto -#swiftclient -RUN ${PACKAGER} -y install python3-etcd python3-consul \ - python3-gevent python3-greenlet python3-cachetools \ - python3-rsa python3-pyasn1-modules python3-cffi \ - && ${PACKAGER} -y clean all; - - # python3-kazoo python3-meld3 python3-boto python3-swiftclient - -RUN pip3 install patroni[kubernetes$EXTRAS]==$PATRONI_VERSION - -RUN mkdir /usr/lib/postgresql -ENV PATHBACKUP = $PATH -#ENV PATH=$PATH:/usr/pgsql-$PGVERSION/bin -#RUN echo $PATH -# Install pam_oauth2.so -RUN git clone -b $PAM_OAUTH2 --recurse-submodules https://github.com/zalando-pg/pam-oauth2.git \ - && make -C pam-oauth2 install \ - #&& curl -sL https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz | tar xz \ - #&& curl -sL https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/x4m/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz | tar xz \ - && curl -sL https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz | tar xz \ - && git clone -b $SET_USER https://github.com/pgaudit/set_user.git \ - && git clone https://github.com/timescale/timescaledb.git \ - && git clone https://github.com/dimitri/pgextwlist.git \ - && git clone https://github.com/powa-team/pg_stat_kcache.git \ - && git clone https://github.com/crunchydata/pgnodemx \ - && git clone https://github.com/laurenz/oracle_fdw.git \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-basic-21.9.0.0.0-1.el8.x86_64.rpm \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-sqlplus-21.9.0.0.0-1.el8.x86_64.rpm \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-devel-21.9.0.0.0-1.el8.x86_64.rpm \ - && wget https://download.oracle.com/otn_software/linux/instantclient/219000/oracle-instantclient-jdbc-21.9.0.0.0-1.el8.x86_64.rpm \ - && ${PACKAGER} install -y oracle-*.rpm \ - #&& git clone https://github.com/pgq/pgqd.git \ -\ - && ${PACKAGER} -y install postgresql${PGVERSION} libevent-devel brotli-devel libbrotli \ - && ${PACKAGER} -y clean all\ -\ - # forbid creation of a main cluster when package is installed - #&& sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \ -\ -# Install PostgreSQL - && for version in $PG_SUPPORTED_VERSIONS; do \ - ${PACKAGER} -y update && \ - ${PACKAGER} -y install postgresql${version}-pltcl \ -# postgresql${version}-dirtyread \ -# postgresql${version}-extra-window-functions \ -# postgresql${version}-first-last-agg \ -# postgresql${version}-hll \ -# postgresql${version}-hypopg \ - pgaudit*${version} \ -# postgresql${version}-pg-checksums \ -# postgresql${version}-pgl-ddl-deploy \ -# postgresql${version}-pglogical \ -# postgresql${version}-pglogical-ticker \ -# postgresql${version}-pgq-node \ -# postgresql${version}-pldebugger \ -# postgresql${version}-pllua \ -# postgresql${version}-plpgsql-check \ -# postgresql${version}-plproxy \ -# postgresql${version}-repack \ -# postgresql${version}-wal2json \ -# postgresql${version}-pgextwlist \ - #&& ${PACKAGER} -y clean all; \ - # Install PostgreSQL binaries, contrib, plproxy and multiple pl's - && ${PACKAGER} -y install -y postgresql${version}-contrib \ - postgresql${version}-plpython3 postgresql${version}-devel \ - pg_cron_${version} \ - #postgresql-${version}-pgq3 \ - #postgresql-${version}-pg-stat-kcache $EXTRAS \ - # Modify for using origial-spilo scripts - && ln -s /usr/pgsql-${version} /usr/lib/postgresql/${version} \ - && export PATH=$PATHBACKUP:/usr/pgsql-${version}/bin \ -\ - # Install TimescaleDB - && cd /timescaledb \ - && for v in $TIMESCALEDB; do \ - git checkout $v \ - && sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt \ - && if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ - -DTAP_CHECKS=OFF -DPG_CONFIG=/usr/pgsql-$version/bin/pg_config \ - -DAPACHE_ONLY=$TIMESCALEDB_APACHE_ONLY -DSEND_TELEMETRY_DEFAULT=NO; then \ - make -C build install \ - && strip /usr/pgsql-$version/lib/timescaledb*.so; \ - fi \ - && git reset --hard \ - && git clean -f -d; \ - done \ - && cd .. \ - && for n in bg_mon-${BG_MON_COMMIT} pg_auth_mon-${PG_AUTH_MON_COMMIT} set_user pg_permissions-${PG_PERMISSIONS_COMMIT} pg_tm_aux-${PG_TM_AUX_COMMIT} pg_profile-${PG_PROFILE} $EXTRA_EXTENSIONS; do \ - make -C $n USE_PGXS=1 clean install-strip; \ - done \ - && cd /pgextwlist && make && make install \ - && cd /pg_stat_kcache && make clean && make && make install \ - && cd /pgnodemx && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ - && cd /oracle_fdw && make USE_PGXS=1 clean && make USE_PGXS=1 && make USE_PGXS=1 install \ - #&& cd /pgqd && ./configure --prefix=/opt && make && make install \ - && ${PACKAGER} -y clean all; \ - done \ - # Remove unnecessary packages - && ${PACKAGER} -y remove postgresql postgresql-all postgresql-server-dev-* libpq-dev=* libmagic1 bsdmainutils \ - && ${PACKAGER} -y autoremove \ - && ${PACKAGER} -y clean all \ - \ - # Clean up - && rm -rf /var/lib/apt/lists/* \ - /var/cache/debconf/* \ - /builddeps \ - /usr/share/doc \ - /usr/share/man \ - /usr/share/info \ - /usr/share/locale/?? \ - /usr/share/locale/??_?? \ - /usr/share/postgresql/*/man \ - /etc/pgbouncer/* \ - /usr/lib/postgresql/*/bin/createdb \ - /usr/lib/postgresql/*/bin/createlang \ - /usr/lib/postgresql/*/bin/createuser \ - /usr/lib/postgresql/*/bin/dropdb \ - /usr/lib/postgresql/*/bin/droplang \ - /usr/lib/postgresql/*/bin/dropuser \ - /usr/lib/postgresql/*/bin/pg_standby \ - /usr/lib/postgresql/*/bin/pltcl_* \ - && find /var/log -type f -exec truncate --size 0 {} \; - -# Install runit -RUN dnf -y install glibc-devel glibc-static - -RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install - -# Modify for using origial-spilo scripts -RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir - -# Clean Up git-repos -RUN rm -rf /pgextwlist /pg_stat_kcache /oracle_fdw /pgnodemx \ - && rm /etc/pgbackrest.conf && rm -rf /var/spool/pgbackrest; - -# Clean Up Packages -RUN ${PACKAGER} -y remove python3-pip python3-wheel python3-dev git patchutils binutils flatpak glibc-devel glibc-static \ - && ${PACKAGER} -y autoremove \ - && ${PACKAGER} -y clean all; - - -FROM builder-false - -ARG PG_VERSION -ARG PGVERSION -ARG TIMESCALEDB - -EXPOSE 5432 8008 8080 -ENV LC_ALL=en_US.utf-8 \ - PATH=$PATH:/usr/pgsql-$PGVERSION/bin \ - PGHOME=/home/postgres \ - RW_DIR=/run \ - TIMESCALEDB=$TIMESCALEDB \ - DEMO=$DEMO - - - -ENV LOG_ENV_DIR=$RW_DIR/etc/log.d/env \ - PGROOT=$PGHOME/pgdata/pgroot - -ENV PGDATA=$PGROOT/data \ - PGLOG=$PGROOT/pg_log - -WORKDIR $PGHOME - -COPY motd /etc/ -#COPY runit /etc/service/ - -COPY runit /etc/runit/runsvdir/default/ -COPY pgq_ticker.ini $PGHOME/ -RUN rm -rf /etc/service - -RUN sed -i "s|/var/lib/pgsql.*|$PGHOME:/bin/bash|" /etc/passwd \ - && chown -R postgres:postgres $PGHOME $RW_DIR \ - && rm -fr /var/spool/cron /var/tmp \ - && mkdir -p /var/spool \ - && ln -s $RW_DIR/cron /var/spool/cron \ - && ln -s $RW_DIR/tmp /var/tmp \ - && for d in /etc/runit/runsvdir/default/*; do \ - chmod 755 $d/* \ - && ln -s /run/supervise/$(basename $d) $d/supervise; \ - done \ - && ln -snf $RW_DIR/service /etc/service \ - #&& ln -s $RW_DIR/pam.d-postgresql /etc/pam.d/postgresql \ - && ln -s $RW_DIR/postgres.yml $PGHOME/postgres.yml \ - && ln -s $RW_DIR/.bash_history /root/.bash_history \ - && ln -s $RW_DIR/postgresql/.bash_history $PGHOME/.bash_history \ - && ln -s $RW_DIR/postgresql/.psql_history $PGHOME/.psql_history \ - && ln -s $RW_DIR/etc $PGHOME/etc \ - && for d in $PGHOME /root; do \ - d=$d/.config/patroni \ - && mkdir -p $d \ - && ln -s $PGHOME/postgres.yml $d/patronictl.yaml; \ - done \ - && sed -i 's/set compatible/set nocompatible/' /etc/vimrc \ - && echo "PATH=\"$PATH\"" > /etc/environment \ - && for e in TERM=linux LC_ALL=C.UTF-8 LANG=C.UTF-8 EDITOR=editor; \ - do echo "export $e" >> /etc/bash.bashrc; \ - done \ - && ln -s /etc/skel/.bashrc $PGHOME/.bashrc \ - && echo "source /etc/motd" >> /root/.bashrc \ - # Allow users in the root group to access the following files and dirs - && if [ "$COMPRESS" != "true" ]; then \ - chmod 664 /etc/passwd \ - && chmod o+r /etc/shadow \ - && chgrp -R 0 $PGHOME $RW_DIR \ - && chmod -R g=u $PGHOME $RW_DIR \ - && usermod -a -G root postgres; \ - fi - -# Remove default pgbackrest-config and spool-path -RUN rm -rf /var/spool/pgbackrest; - -COPY scripts bootstrap major_upgrade /scripts/ -COPY launcher/postgres/launch.sh / - -#1RUN dnf -y install 'dnf-command(config-manager)' -#RUN dnf -y config-manager --set-enabled crb -#RUN dnf -y install glibc-devel glibc-static - -#RUN mkdir /package && cd /package && wget http://smarden.org/runit/runit-2.1.2.tar.gz && tar -xvzf runit-2.1.2.tar.gz && rm runit-2.1.2.tar.gz && cd admin/runit-2.1.2 && package/install - -# Modify for using origial-spilo scripts -#RUN ln -s /usr/local/bin/dumb-init /usr/bin/dumb-init && ln -s /usr/local/bin/runsvdir /usr/bin/runsvdir - -#RUN curl -LO https://dl.k8s.io/release/v1.26.0/bin/linux/amd64/kubectl && chmod +x kubectl && mv ./kubectl /usr/local/bin/ - -USER 26 - -#RUN chown -R postgres:root /var/lib/pgsql -CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file From 8f075587123d1131f01db52154672d527fc220bd Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:17:14 +0200 Subject: [PATCH 09/11] init --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c50ffd0..20c22f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ # exclude some folders docker/postgres-stage/ -docker/postgres-oracle/ \ No newline at end of file +docker/postgres-oracle/ +docker/newpg +docker/benchmark \ No newline at end of file From 8c42ac8a27d4825e33a7a22cfaf32bb6b6f9afd6 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:17:25 +0200 Subject: [PATCH 10/11] init --- benchmark/Dockerfile | 49 -------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 benchmark/Dockerfile diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile deleted file mode 100644 index af147bf..0000000 --- a/benchmark/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -ARG CONTAINERSUITE -ARG BUILD - -FROM ${CONTAINERSUITE}/base:0.0.${BUILD} - -# Dockerfile specific informations -ARG PACKAGER -ARG BASEOS -ARG PGBACKREST_VERSION -ARG PG_MAJOR - -RUN if [ "$BASEOS" = "ubi8" ] ; then \ - ${PACKAGER} -y install --nodocs \ - openssh-clients \ - openssh-server \ - shadow-utils \ - tar \ - bzip2 \ - lz4 \ - && ${PACKAGER} -y clean all ; \ -else \ - ${PACKAGER} -y install --nodocs \ - --setopt=skip_missing_names_on_install=False \ - openssh-clients \ - openssh-server \ - bzip2 \ - lz4 \ - postgresql${PG_MAJOR}-server \ - - nano \ - && ${PACKAGER} -y clean all ; \ -fi - - -# add benbchmark files -#ADD scripts/benchmark/ /opt/benchmark/bin/ - -# set user and group ownership -R#UN chown -R postgres:postgres /opt/benchmark - -#COPY launcher/benchmark/launch.sh / - -USER 26 - -#ENTRYPOINT ["/opt/pgbackrest/bin/uid_postgres.sh"] - -#CMD ["/opt/pgbackrest/bin/start.sh"] - -#CMD ["/bin/sh", "/launch.sh", "init"] \ No newline at end of file From 16f812da044fb89435d3f41d2d962ee75ecde08a Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 6 Apr 2023 08:17:51 +0200 Subject: [PATCH 11/11] init --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 20c22f6..6220d73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # exclude some folders -docker/postgres-stage/ -docker/postgres-oracle/ +docker/postgres-stage +docker/postgres-oracle docker/newpg docker/benchmark \ No newline at end of file