[DOP-30579] Improve ivy2 package caching

Author: dolfinus

docker/Dockerfile.worker

@@ -41,23 +41,25 @@ RUN --mount=type=cache,target=/root/.cache/pypoetry \
     && python -m compileall -j 4 .venv
 
 
-FROM builder AS maven_packages
+FROM builder AS ivy2_packages
 
 RUN --mount=type=bind,source=./syncmaster/worker/ivy2.py,target=/app/syncmaster/worker/ivy2.py \
-    --mount=type=bind,source=./docker/download_maven_packages.py,target=/app/docker/download_maven_packages.py \
-    mkdir /root/.ivy2 && \
+    --mount=type=bind,source=./docker/download_ivy2_packages.py,target=/app/docker/download_ivy2_packages.py \
+    --mount=type=cache,target=/root/.ivy2 \
     # Try to download all dependencies at once.
     # If multiple packages depends on the same transitive dependency, Spark uses maximum version of this dependency.
-    python /app/docker/download_maven_packages.py all && \
+    python /app/docker/download_ivy2_packages.py all && \
     # Then try to download specific connectors to fetch exact dependency version specified within connector.
     # Yes, this is slow, but overwise using worker without internet access will fail, unless custom ivysettings.xml is used
-    python /app/docker/download_maven_packages.py s3 && \
-    python /app/docker/download_maven_packages.py hdfs && \
-    python /app/docker/download_maven_packages.py clickhouse && \
-    python /app/docker/download_maven_packages.py postgres && \
-    python /app/docker/download_maven_packages.py oracle && \
-    python /app/docker/download_maven_packages.py mssql && \
-    python /app/docker/download_maven_packages.py mysql
+    python /app/docker/download_ivy2_packages.py s3 && \
+    python /app/docker/download_ivy2_packages.py hdfs && \
+    python /app/docker/download_ivy2_packages.py clickhouse && \
+    python /app/docker/download_ivy2_packages.py postgres && \
+    python /app/docker/download_ivy2_packages.py oracle && \
+    python /app/docker/download_ivy2_packages.py mssql && \
+    python /app/docker/download_ivy2_packages.py mysql && \
+    mkdir -p /home/syncmaster/.ivy2 && \
+    cp --recursive /root/.ivy2/ /home/syncmaster/.ivy2/
     # if someone uses custom worker image, they should download jars on their own
 
 
@@ -72,7 +74,9 @@ RUN useradd syncmaster && \
 # We don't need poetry and compilers in final image
 COPY --from=builder /app/.venv/ /app/.venv/
 # custom Spark session function may download different jars, so syncmaster have to own them
-COPY --from=maven_packages --chown=syncmaster:syncmaster /root/.ivy2/ /home/syncmaster/.ivy2/
+COPY --from=ivy2_packages --chown=syncmaster:syncmaster /home/syncmaster/.ivy2/ /home/syncmaster/.ivy2/
+# If someone needs to use worker image with root user, use the same jars
+RUN ln -s /home/syncmaster/.ivy2 /root/.ivy2
 
 COPY ./syncmaster/ /app/syncmaster/
 RUN python -m compileall syncmaster