diff --git a/docs/modules/demos/pages/jupyterhub-keycloak.adoc b/docs/modules/demos/pages/jupyterhub-keycloak.adoc index f78403d1..93d2ba1e 100644 --- a/docs/modules/demos/pages/jupyterhub-keycloak.adoc +++ b/docs/modules/demos/pages/jupyterhub-keycloak.adoc @@ -103,7 +103,7 @@ This setup is ideal for interactive data processing. === Spark Configuration -* **Executor Image**: Uses a custom image `oci.stackable.tech/sandbox/spark:3.5.2-python311` (built on the standard Spark image) for the executors, matching the Python version of the notebook. +* **Executor Image**: Uses a custom image `oci.stackable.tech/sandbox/spark:3.5.5-python311` (built on the standard Spark image) for the executors, matching the Python version of the notebook. * **Resource Allocation**: Configures Spark executor instances, memory, and cores through settings defined in the notebook. * **Hadoop and AWS Libraries**: Includes necessary Hadoop and AWS libraries for S3 operations, matching the notebook image version. diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 46b6e761..36aeaa51 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -274,7 +274,7 @@ data: spec: version: "1.0" sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.5 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py job: diff --git a/stacks/jupyterhub-keycloak/Dockerfile b/stacks/jupyterhub-keycloak/Dockerfile index 874bbd17..d847f41e 100644 --- a/stacks/jupyterhub-keycloak/Dockerfile +++ b/stacks/jupyterhub-keycloak/Dockerfile @@ -1,9 +1,9 @@ -# docker build -t oci.stackable.tech/sandbox/spark:3.5.2-python311 -f Dockerfile . -# kind load docker-image oci.stackable.tech/sandbox/spark:3.5.2-python311 -n stackable-data-platform +# docker build -t oci.stackable.tech/sandbox/spark:3.5.5-python311 -f Dockerfile . +# kind load docker-image oci.stackable.tech/sandbox/spark:3.5.5-python311 -n stackable-data-platform # or: -# docker push oci.stackable.tech/sandbox/spark:3.5.2-python311 +# docker push oci.stackable.tech/sandbox/spark:3.5.5-python311 -FROM spark:3.5.2-scala2.12-java17-ubuntu +FROM spark:3.5.5-scala2.12-java17-ubuntu USER root diff --git a/stacks/jupyterhub-keycloak/jupyterhub.yaml b/stacks/jupyterhub-keycloak/jupyterhub.yaml index 375763e1..ed11b3dc 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub.yaml @@ -209,7 +209,7 @@ options: image: display_name: Image choices: -{% for image in ["quay.io/jupyter/pyspark-notebook:python-3.11.9", "quay.io/jupyter/pyspark-notebook:spark-3.5.2"] %} +{% for image in ["quay.io/jupyter/pyspark-notebook:python-3.11.9", "quay.io/jupyter/pyspark-notebook:spark-3.5.5"] %} "{{image}}": display_name: "{{image}}" kubespawner_override: diff --git a/stacks/jupyterhub-keycloak/process-s3.ipynb b/stacks/jupyterhub-keycloak/process-s3.ipynb index ac36f6ad..f13dd25f 100644 --- a/stacks/jupyterhub-keycloak/process-s3.ipynb +++ b/stacks/jupyterhub-keycloak/process-s3.ipynb @@ -49,8 +49,8 @@ "acts as the driver. It is important that the versions of spark and python match across the driver (running in the juypyterhub image)\n", "and the executor(s) (running in a separate image, specified below with the `spark.kubernetes.container.image` setting).\n", "\n", - "The jupyterhub image `quay.io/jupyter/pyspark-notebook:spark-3.5.2` uses a base ubuntu image (like the spark images).\n", - "The versions of java match exactly. Python versions can differ at patch level, and the image used below `oci.stackable.tech/sandbox/spark:3.5.2-python311` is built from a `spark:3.5.2-scala2.12-java17-ubuntu` base image with python 3.11 (the same major/minor version as the notebook) installed.\n", + "The jupyterhub image `quay.io/jupyter/pyspark-notebook:spark-3.5.5` uses a base ubuntu image (like the spark images).\n", + "The versions of java match exactly. Python versions can differ at patch level, and the image used below `oci.stackable.tech/sandbox/spark:3.5.5-python311` is built from a `spark:3.5.2-scala2.12-java17-ubuntu` base image with python 3.11 (the same major/minor version as the notebook) installed.\n", "\n", "## S3\n", "As we will be reading data from an S3 bucket, we need to add the necessary `hadoop` and `aws` libraries in the same hadoop version as the\n", @@ -69,7 +69,7 @@ "NAMESPACE = os.environ.get(\"NAMESPACE\", \"default\")\n", "POD_NAME = os.environ.get(\"HOSTNAME\", f\"jupyter-{os.environ.get('USER', 'default')}-{NAMESPACE}\")\n", "\n", - "EXECUTOR_IMAGE = \"oci.stackable.tech/sandbox/spark:3.5.2-python311\" \n", + "EXECUTOR_IMAGE = \"oci.stackable.tech/sandbox/spark:3.5.5-python311\" \n", "\n", "spark = (\n", " SparkSession.builder\n",