ci(docker): set legacy IPC format to fix e2e tests (#1201)

Signed-off-by: Sajid Alam <[email protected]>
Co-authored-by: Deepyaman Datta <[email protected]>

Author: SajidAlamQB

kedro-docker/kedro_docker/template/Dockerfile.spark

@@ -5,6 +5,9 @@ FROM $BASE_IMAGE as runtime-environment
 RUN apt-get update && mkdir -p /usr/share/man/man1 && \
     apt-get install -y procps default-jre-headless && rm -rf /var/lib/apt/lists/*
 
+# set Spark configuration to handle Arrow properly
+ENV ARROW_PRE_0_15_IPC_FORMAT=1
+
 # update pip and install uv
 RUN python -m pip install -U "pip>=21.2"
 RUN pip install uv