diff --git a/.github/ISSUE_TEMPLATE/update-base-java.md b/.github/ISSUE_TEMPLATE/update-base-java.md index b29a47cb0..69efe017d 100644 --- a/.github/ISSUE_TEMPLATE/update-base-java.md +++ b/.github/ISSUE_TEMPLATE/update-base-java.md @@ -63,7 +63,7 @@ we should also make new versions of Java available for use. ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 # Test a product image can build, eg: ZooKeeper bake --product zookeeper=x.y.z # where x.y.z is a valid product version using the newly added Java version diff --git a/.github/ISSUE_TEMPLATE/update-base-stackable.md b/.github/ISSUE_TEMPLATE/update-base-stackable.md index 1d20147cf..ebb154ba2 100644 --- a/.github/ISSUE_TEMPLATE/update-base-stackable.md +++ b/.github/ISSUE_TEMPLATE/update-base-stackable.md @@ -52,7 +52,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product vector=x.y.z # where x.y.z is a valid version ``` diff --git a/.github/ISSUE_TEMPLATE/update-base-vector.md b/.github/ISSUE_TEMPLATE/update-base-vector.md index 43ea66ea6..2a8c28e55 100644 --- a/.github/ISSUE_TEMPLATE/update-base-vector.md +++ b/.github/ISSUE_TEMPLATE/update-base-vector.md @@ -70,7 +70,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product vector=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-airflow.md b/.github/ISSUE_TEMPLATE/update-product-airflow.md index 80f608ced..158b10e8f 100644 --- a/.github/ISSUE_TEMPLATE/update-product-airflow.md +++ b/.github/ISSUE_TEMPLATE/update-product-airflow.md @@ -63,7 +63,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product airflow=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-druid.md b/.github/ISSUE_TEMPLATE/update-product-druid.md index 2305bffb0..04e750553 100644 --- a/.github/ISSUE_TEMPLATE/update-product-druid.md +++ b/.github/ISSUE_TEMPLATE/update-product-druid.md @@ -67,7 +67,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product druid=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md index 88a5768db..633be0368 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md +++ b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md @@ -71,7 +71,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product hbase=x.y.z # where x.y.z is the new version added in this PR bake --product omid=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hdfs.md b/.github/ISSUE_TEMPLATE/update-product-hdfs.md index 1f9b13419..cf3b63350 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hdfs.md +++ b/.github/ISSUE_TEMPLATE/update-product-hdfs.md @@ -64,7 +64,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product hadoop=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hive.md b/.github/ISSUE_TEMPLATE/update-product-hive.md index 7afa60fca..1da166f1e 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hive.md +++ b/.github/ISSUE_TEMPLATE/update-product-hive.md @@ -64,7 +64,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product hive=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-kafka.md b/.github/ISSUE_TEMPLATE/update-product-kafka.md index cff61c59c..51d37ed77 100644 --- a/.github/ISSUE_TEMPLATE/update-product-kafka.md +++ b/.github/ISSUE_TEMPLATE/update-product-kafka.md @@ -76,7 +76,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product kafka=x.y.z # where x.y.z is the new version added in this PR bake --product kafka-testing-tools=1.0.0 # This version doesn't change diff --git a/.github/ISSUE_TEMPLATE/update-product-nifi.md b/.github/ISSUE_TEMPLATE/update-product-nifi.md index 078bc1680..16e3fdc75 100644 --- a/.github/ISSUE_TEMPLATE/update-product-nifi.md +++ b/.github/ISSUE_TEMPLATE/update-product-nifi.md @@ -63,7 +63,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product nifi=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-opa.md b/.github/ISSUE_TEMPLATE/update-product-opa.md index b49776b42..f0312208d 100644 --- a/.github/ISSUE_TEMPLATE/update-product-opa.md +++ b/.github/ISSUE_TEMPLATE/update-product-opa.md @@ -62,7 +62,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product opa=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-spark.md b/.github/ISSUE_TEMPLATE/update-product-spark.md index 30e4be6ff..0ea2a4ca4 100644 --- a/.github/ISSUE_TEMPLATE/update-product-spark.md +++ b/.github/ISSUE_TEMPLATE/update-product-spark.md @@ -64,7 +64,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product spark-k8s=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-superset.md b/.github/ISSUE_TEMPLATE/update-product-superset.md index c04169c4d..1f32cd504 100644 --- a/.github/ISSUE_TEMPLATE/update-product-superset.md +++ b/.github/ISSUE_TEMPLATE/update-product-superset.md @@ -65,7 +65,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product superset=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-trino.md b/.github/ISSUE_TEMPLATE/update-product-trino.md index 06656a38e..8f3683380 100644 --- a/.github/ISSUE_TEMPLATE/update-product-trino.md +++ b/.github/ISSUE_TEMPLATE/update-product-trino.md @@ -73,7 +73,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product trino=x.y.z # where x.y.z is the new version added in this PR bake --product trino-cli=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md index 0f2f5ca44..b520c2e6d 100644 --- a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md +++ b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md @@ -63,7 +63,7 @@ Add/Change/Remove anything that isn't applicable anymore ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.13 +pip install image-tools-stackabletech==0.0.16 bake --product zookeeper=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/workflows/mirror.yaml b/.github/workflows/mirror.yaml index bc03df794..f4cfc35a5 100644 --- a/.github/workflows/mirror.yaml +++ b/.github/workflows/mirror.yaml @@ -57,7 +57,7 @@ jobs: echo "IMAGE_REPOSITORY=$(.scripts/get_repo_name.sh)" | tee -a "$GITHUB_ENV" - name: Publish Container Image on oci.stackable.tech - uses: stackabletech/actions/publish-image@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/publish-image@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build @@ -84,7 +84,7 @@ jobs: echo "IMAGE_REPOSITORY=$(.scripts/get_repo_name.sh)" | tee -a "$GITHUB_ENV" - name: Publish and Sign Image Index Manifest to oci.stackable.tech - uses: stackabletech/actions/publish-index-manifest@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/publish-index-manifest@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$sdp+github-action-build diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml index ded4d5a31..c84a5aa00 100644 --- a/.github/workflows/pr_pre-commit.yaml +++ b/.github/workflows/pr_pre-commit.yaml @@ -16,7 +16,7 @@ jobs: with: persist-credentials: false fetch-depth: 0 - - uses: stackabletech/actions/run-pre-commit@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + - uses: stackabletech/actions/run-pre-commit@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: python-version: ${{ env.PYTHON_VERSION }} hadolint: ${{ env.HADOLINT_VERSION }} diff --git a/.github/workflows/preflight.yaml b/.github/workflows/preflight.yaml index 76c229987..ca2f6cbc7 100644 --- a/.github/workflows/preflight.yaml +++ b/.github/workflows/preflight.yaml @@ -80,7 +80,7 @@ jobs: - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: '3.x' - - run: pip install image-tools-stackabletech==0.0.13 + - run: pip install image-tools-stackabletech==0.0.16 - name: Install preflight run: | wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/1.10.0/preflight-linux-amd64 diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml index 33c63de71..7a54dbb7f 100644 --- a/.github/workflows/reusable_build_image.yaml +++ b/.github/workflows/reusable_build_image.yaml @@ -27,7 +27,7 @@ jobs: with: persist-credentials: false - id: shard - uses: stackabletech/actions/shard@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/shard@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: product-name: ${{ inputs.product-name }} outputs: @@ -53,18 +53,18 @@ jobs: persist-credentials: false - name: Free Disk Space - uses: stackabletech/actions/free-disk-space@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/free-disk-space@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 - name: Build Product Image id: build - uses: stackabletech/actions/build-product-image@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/build-product-image@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: product-name: ${{ inputs.product-name }} product-version: ${{ matrix.versions }} sdp-version: ${{ inputs.sdp-version }} - name: Publish Container Image on oci.stackable.tech - uses: stackabletech/actions/publish-image@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/publish-image@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build @@ -90,7 +90,7 @@ jobs: persist-credentials: false - name: Publish and Sign Image Index Manifest to oci.stackable.tech - uses: stackabletech/actions/publish-index-manifest@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # 0.8.1 + uses: stackabletech/actions/publish-index-manifest@9aae2d1c14239021bfa33c041010f6fb7adec815 # 0.8.2 with: image-registry-uri: oci.stackable.tech image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build diff --git a/airflow/Dockerfile b/airflow/Dockerfile index ed241ff18..c5bcfff4f 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -7,7 +7,7 @@ ARG GIT_SYNC # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8) FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC} AS gitsync-image -FROM stackable/image/statsd_exporter AS statsd_exporter-builder +FROM stackable/image/shared/statsd-exporter AS statsd_exporter-builder FROM python:3.12-bookworm AS opa-auth-manager-builder @@ -25,7 +25,7 @@ EOF FROM stackable/image/vector AS airflow-build-image ARG PRODUCT -ARG STATSD_EXPORTER +ARG SHARED_STATSD_EXPORTER ARG PYTHON ARG TARGETARCH ARG STACKABLE_USER_UID @@ -95,7 +95,7 @@ end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json EOF COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter -COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json +COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync RUN < /stackable/bin/hbck2 -chmod +x /stackable/bin/hbck2 -rm /stackable/bin/hbck2.env - -# We're removing these to make the intermediate layer smaller -# This can be necessary even though it's only a builder image because the GitHub Action Runners only have very limited space available -# and we are sometimes running into errors because we're out of space. -# Therefore, we try to clean up all layers as much as possible. -if [ "${DELETE_CACHES}" = "true" ] ; then - rm -rf /stackable/.m2/repository/* - rm -rf /stackable/.npm/* - rm -rf /stackable/.cache/* -fi - -# set correct groups -chmod --recursive g=u /stackable -EOF - - # Splitting this out into its own builder so that Hadoop & HBase can be built in parallel # envsubst is only available in java-devel which is why we don't just do this in the final image FROM stackable/image/java-devel AS hadoop-s3-builder @@ -240,76 +138,14 @@ rm /stackable/bin/export-snapshot-to-s3.env chmod --recursive g=u /stackable EOF - -FROM stackable/image/java-devel AS phoenix-builder - -ARG PRODUCT -ARG ASYNC_PROFILER -ARG PHOENIX -ARG HBASE_PROFILE -ARG HADOOP -ARG STACKABLE_USER_UID - -# Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) -# This can be used to speed up builds when disk space is of no concern. -ARG DELETE_CACHES="true" - -COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/patchable.toml /stackable/src/phoenix/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PHOENIX} /stackable/src/phoenix/stackable/patches/${PHOENIX} -USER ${STACKABLE_USER_UID} -WORKDIR /stackable - -RUN --mount=type=cache,id=maven-phoenix-${PHOENIX}-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository < /stackable/bin/hbck2 +chmod +x /stackable/bin/hbck2 +rm /stackable/bin/hbck2.env + +# We're removing these to make the intermediate layer smaller +# This can be necessary even though it's only a builder image because the GitHub Action Runners only have very limited space available +# and we are sometimes running into errors because we're out of space. +# Therefore, we try to clean up all layers as much as possible. +if [ "${DELETE_CACHES}" = "true" ] ; then + rm -rf /stackable/.m2/repository/* + rm -rf /stackable/.npm/* + rm -rf /stackable/.cache/* +fi + +# set correct groups +chmod --recursive g=u /stackable +EOF diff --git a/hbase/hbase-operator-tools/versions.py b/hbase/hbase-operator-tools/versions.py new file mode 100644 index 000000000..22a6aaf41 --- /dev/null +++ b/hbase/hbase-operator-tools/versions.py @@ -0,0 +1,16 @@ +versions = [ + { + "product": "1.2.0", + "hbase_thirdparty": "4.1.5", + "hbase_version": "2.4.18", + "java-devel": "11", + "delete_caches": "true", + }, + { + "product": "1.3.0-fd5a5fb", + "hbase_thirdparty": "4.1.9", + "hbase_version": "2.6.1", + "java-devel": "11", + "delete_caches": "true", + }, +] diff --git a/hbase/phoenix/Dockerfile b/hbase/phoenix/Dockerfile new file mode 100644 index 000000000..2d9273b3c --- /dev/null +++ b/hbase/phoenix/Dockerfile @@ -0,0 +1,57 @@ +FROM stackable/image/java-devel + +ARG PRODUCT +ARG ASYNC_PROFILER +ARG HBASE_VERSION +ARG HBASE_PROFILE +ARG HADOOP_VERSION +ARG STACKABLE_USER_UID + +# Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) +# This can be used to speed up builds when disk space is of no concern. +ARG DELETE_CACHES="true" + +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/patchable.toml /stackable/src/phoenix/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 hbase/phoenix/stackable/patches/${PRODUCT} /stackable/src/phoenix/stackable/patches/${PRODUCT} +USER ${STACKABLE_USER_UID} +WORKDIR /stackable + +RUN --mount=type=cache,id=maven-phoenix-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <