From 72523b127bc1fe1b9455ae2ae7cdac03aa368c12 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 17 Sep 2024 11:12:13 +0200 Subject: [PATCH 1/4] Add a curlrc file This adds a curlrc file with sane defaults including retries etc. which can be used for all our Docker builds without having to specify all flags (e.g. fail) every time --- airflow/Dockerfile | 2 +- druid/Dockerfile | 4 +-- hadoop/Dockerfile | 10 ++++---- hbase/Dockerfile | 10 ++++---- hello-world/Dockerfile | 2 +- hive/Dockerfile | 4 +-- kafka/Dockerfile | 6 ++--- kcat/Dockerfile | 2 +- nifi/Dockerfile | 16 ++++++------ omid/Dockerfile | 2 +- opa/Dockerfile | 2 +- spark-k8s/Dockerfile | 22 ++++++++--------- stackable-base/Dockerfile | 5 ++++ stackable-base/stackable/curlrc | 43 +++++++++++++++++++++++++++++++++ statsd_exporter/Dockerfile | 2 +- testing-tools/Dockerfile | 2 +- tools/Dockerfile | 4 +-- trino-cli/Dockerfile | 2 +- trino/Dockerfile | 6 ++--- ubi8-rust-builder/Dockerfile | 2 +- ubi9-rust-builder/Dockerfile | 2 +- zookeeper/Dockerfile | 4 +-- 22 files changed, 101 insertions(+), 53 deletions(-) create mode 100644 stackable-base/stackable/curlrc diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 6c30adc66..00b5a951c 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -101,7 +101,7 @@ RUN mkdir -pv ${AIRFLOW_HOME} && \ mkdir -pv ${AIRFLOW_HOME}/dags && \ mkdir -pv ${AIRFLOW_HOME}/logs && \ chown --recursive stackable:stackable ${AIRFLOW_HOME} && \ - curl --fail -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}" + curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}" COPY airflow/stackable/utils/entrypoint.sh /entrypoint.sh COPY airflow/stackable/utils/run-airflow.sh /run-airflow.sh diff --git a/druid/Dockerfile b/druid/Dockerfile index 62e0456eb..e0e3eea26 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -48,7 +48,7 @@ RUN --mount=type=cache,id=maven-${PRODUCT},uid=1000,target=/stackable/.m2/reposi --mount=type=cache,id=npm-${PRODUCT},uid=1000,target=/stackable/.npm \ --mount=type=cache,id=cache-${PRODUCT},uid=1000,target=/stackable/.cache \ <> /stackable/.profi chown stackable:stackable /stackable/.bashrc chown stackable:stackable /stackable/.profile +cp /root/.curlrc /stackable/.curlrc +chown stackable:0 /stackable/.curlrc + # CVE-2023-37920: Remove "e-Tugra" root certificates # e-Tugra's root certificates were subject to an investigation prompted by reporting of security issues in their systems # Until they are removed by default from ca-certificates, we should remove them manually diff --git a/stackable-base/stackable/curlrc b/stackable-base/stackable/curlrc new file mode 100644 index 000000000..284b6694e --- /dev/null +++ b/stackable-base/stackable/curlrc @@ -0,0 +1,43 @@ +# Stackable curl config file +# https://everything.curl.dev/cmdline/configfile.html + +# Follow up to 30 redirects (-L), 0 by default +# https://everything.curl.dev/http/redirects.html#tell-curl-to-follow-redirects +location + +# Retry the request up to 5 times on transient errors (e.g., network issues). +# https://everything.curl.dev/usingcurl/downloads/retry.html#retry +retry = 10 + +# Retry on ECONNREFUSED in addition to all the other transient errors +# https://everything.curl.dev/usingcurl/downloads/retry.html#connection-refused +retry-connrefused + +# Wait 15 seconds between retry attempts, this disables the exponential backoff +# https://everything.curl.dev/usingcurl/downloads/retry.html#tweak-your-retries +retry-delay = 15 + +# Limit the total retry time to 1200 seconds (20 minutes), this is across all retries. +# https://everything.curl.dev/usingcurl/downloads/retry.html#tweak-your-retries +retry-max-time = 1200 + +# Fail the request if the server returns an error (4xx or 5xx HTTP status code). +# https://everything.curl.dev/http/response.html#http-response-codes +fail + +# Set an overall timeout of 600 seconds (10 minutes) for a single download process +# https://everything.curl.dev/usingcurl/timeouts.html#maximum-time-allowed-to-spend +max-time = 600 + +# Set the connection timeout to 30 seconds to avoid hanging too long on connection attempts. +# https://everything.curl.dev/usingcurl/timeouts.html#never-spend-more-than-this-to-connect +connect-timeout = 30 + +# (Optional) Protect against downloading files larger than 100GB +# https://everything.curl.dev/usingcurl/downloads/max-filesize.html#maximum-filesize +# max-filesize = 107374182400 + +# Suppress progress output, but still show errors +# https://everything.curl.dev/usingcurl/verbose/index.html?highlight=silent#silence +silent +show-error diff --git a/statsd_exporter/Dockerfile b/statsd_exporter/Dockerfile index 94c28ad5f..0a084a1fc 100644 --- a/statsd_exporter/Dockerfile +++ b/statsd_exporter/Dockerfile @@ -19,7 +19,7 @@ microdnf clean all rm -rf /var/cache/yum export GOPATH=/go_cache -curl --fail -L "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC . +curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC . ( cd "statsd_exporter-${PRODUCT}" || exit go build -o ../statsd_exporter diff --git a/testing-tools/Dockerfile b/testing-tools/Dockerfile index ae39b88eb..3efb417d2 100644 --- a/testing-tools/Dockerfile +++ b/testing-tools/Dockerfile @@ -55,7 +55,7 @@ COPY testing-tools/python /stackable/python COPY testing-tools/licenses /licenses ENV PATH=/stackable/keycloak/bin:$PATH -RUN curl --fail https://repo.stackable.tech/repository/packages/keycloak/keycloak-${KEYCLOAK_VERSION}.tar.gz | tar -xzC /stackable && \ +RUN curl --fail -L https://repo.stackable.tech/repository/packages/keycloak/keycloak-${KEYCLOAK_VERSION}.tar.gz | tar -xzC /stackable && \ ln -s /stackable/keycloak-${KEYCLOAK_VERSION} /stackable/keycloak RUN pip install --no-cache-dir --upgrade pip && \ diff --git a/tools/Dockerfile b/tools/Dockerfile index 838d58cc2..f953fbb52 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -34,10 +34,10 @@ WORKDIR /stackable/bin ENV PATH=/stackable/bin:$PATH # Get latest stable version from curl -L -s https://dl.k8s.io/release/stable.txt -RUN curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ +RUN curl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ -o /stackable/bin/kubectl && chmod +x /stackable/bin/kubectl -RUN curl -L https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 \ +RUN curl https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 \ -o /stackable/bin/jq && \ chmod +x /stackable/bin/jq diff --git a/trino-cli/Dockerfile b/trino-cli/Dockerfile index aa5a8a06e..75f3c48f1 100644 --- a/trino-cli/Dockerfile +++ b/trino-cli/Dockerfile @@ -28,7 +28,7 @@ COPY --chown=stackable:stackable trino-cli/licenses /licenses WORKDIR /stackable/trino-cli -RUN curl --fail -L -O https://repo.stackable.tech/repository/packages/trino-cli/trino-cli-${PRODUCT}-executable.jar \ +RUN curl -O https://repo.stackable.tech/repository/packages/trino-cli/trino-cli-${PRODUCT}-executable.jar \ && ln -s trino-cli-${PRODUCT}-executable.jar trino-cli-executable.jar ENTRYPOINT ["java", "-jar", "/stackable/trino-cli/trino-cli-executable.jar"] diff --git a/trino/Dockerfile b/trino/Dockerfile index 22a28f212..9f2ce1584 100644 --- a/trino/Dockerfile +++ b/trino/Dockerfile @@ -20,7 +20,7 @@ WORKDIR /stackable COPY --chown=stackable:stackable trino/stackable/patches/apply_patches.sh /stackable/trino-storage-${STORAGE_CONNECTOR}-src/patches/apply_patches.sh COPY --chown=stackable:stackable trino/stackable/patches/trino-storage/${STORAGE_CONNECTOR} /stackable/trino-storage-${STORAGE_CONNECTOR}-src/patches/${STORAGE_CONNECTOR} -RUN curl --fail -L "https://repo.stackable.tech/repository/packages/trino-storage/trino-storage-${STORAGE_CONNECTOR}-src.tar.gz" | tar -xzC . +RUN curl "https://repo.stackable.tech/repository/packages/trino-storage/trino-storage-${STORAGE_CONNECTOR}-src.tar.gz" | tar -xzC . # adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980 # hadolint ignore=SC2215 RUN --mount=type=cache,target=/root/.m2/repository cd trino-storage-${STORAGE_CONNECTOR}-src && \ @@ -48,7 +48,7 @@ EOF WORKDIR /stackable -RUN curl --fail -L "https://repo.stackable.tech/repository/packages/trino-server/trino-server-${PRODUCT}-src.tar.gz" | tar -xzC . +RUN curl "https://repo.stackable.tech/repository/packages/trino-server/trino-server-${PRODUCT}-src.tar.gz" | tar -xzC . COPY --chown=stackable:stackable trino/stackable/patches/apply_patches.sh /stackable/trino-server-${PRODUCT}-src/patches/apply_patches.sh COPY --chown=stackable:stackable trino/stackable/patches/${PRODUCT} /stackable/trino-server-${PRODUCT}-src/patches/${PRODUCT} @@ -108,7 +108,7 @@ WORKDIR /stackable COPY --chown=stackable:stackable trino/stackable/patches/apply_patches.sh /stackable/jmx_prometheus-${JMX_EXPORTER}-src/patches/apply_patches.sh COPY --chown=stackable:stackable trino/stackable/patches/jmx-exporter/${JMX_EXPORTER} /stackable/jmx_prometheus-${JMX_EXPORTER}-src/patches/${JMX_EXPORTER} -RUN curl --fail -L "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus-${JMX_EXPORTER}-src.tar.gz" | tar -xzC . +RUN curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus-${JMX_EXPORTER}-src.tar.gz" | tar -xzC . # adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980 # hadolint ignore=SC2215 RUN --mount=type=cache,target=/root/.m2/repository cd jmx_prometheus-${JMX_EXPORTER}-src && \ diff --git a/ubi8-rust-builder/Dockerfile b/ubi8-rust-builder/Dockerfile index 1fc80a459..8df839f4a 100644 --- a/ubi8-rust-builder/Dockerfile +++ b/ubi8-rust-builder/Dockerfile @@ -67,7 +67,7 @@ WORKDIR /opt/protoc # Prost does not document which version of protoc it expects (https://docs.rs/prost-build/0.12.4/prost_build/), so this should be the latest upstream version # (within reason). RUN ARCH=$(arch | sed 's/^aarch64$/aarch_64/') \ - && curl --location --output protoc.zip "https://repo.stackable.tech/repository/packages/protoc/protoc-${PROTOC_VERSION}-linux-${ARCH}.zip" \ + && curl --fail --location --output protoc.zip "https://repo.stackable.tech/repository/packages/protoc/protoc-${PROTOC_VERSION}-linux-${ARCH}.zip" \ && unzip protoc.zip \ && rm protoc.zip ENV PROTOC=/opt/protoc/bin/protoc diff --git a/ubi9-rust-builder/Dockerfile b/ubi9-rust-builder/Dockerfile index 8d16f6b90..6d3019dc7 100644 --- a/ubi9-rust-builder/Dockerfile +++ b/ubi9-rust-builder/Dockerfile @@ -66,7 +66,7 @@ WORKDIR /opt/protoc # Prost does not document which version of protoc it expects (https://docs.rs/prost-build/0.12.4/prost_build/), so this should be the latest upstream version # (within reason). RUN ARCH=$(arch | sed 's/^aarch64$/aarch_64/') \ - && curl --location --output protoc.zip "https://repo.stackable.tech/repository/packages/protoc/protoc-${PROTOC_VERSION}-linux-${ARCH}.zip" \ + && curl --fail --location --output protoc.zip "https://repo.stackable.tech/repository/packages/protoc/protoc-${PROTOC_VERSION}-linux-${ARCH}.zip" \ && unzip protoc.zip \ && rm protoc.zip ENV PROTOC=/opt/protoc/bin/protoc diff --git a/zookeeper/Dockerfile b/zookeeper/Dockerfile index 42eced6f8..21d87b91e 100644 --- a/zookeeper/Dockerfile +++ b/zookeeper/Dockerfile @@ -15,7 +15,7 @@ USER stackable WORKDIR /stackable # Download ZooKeeper sources from our own repo -RUN curl --fail -L "https://repo.stackable.tech/repository/packages/zookeeper/apache-zookeeper-${PRODUCT}.tar.gz" | tar -xzC . && \ +RUN curl "https://repo.stackable.tech/repository/packages/zookeeper/apache-zookeeper-${PRODUCT}.tar.gz" | tar -xzC . && \ # Apply any required patches patches/apply_patches.sh ${PRODUCT} && \ cd /stackable/apache-zookeeper-${PRODUCT}/ && \ @@ -31,7 +31,7 @@ RUN curl --fail -L "https://repo.stackable.tech/repository/packages/zookeeper/ap rm -rf /stackable/apache-zookeeper-${PRODUCT}-bin/docs && \ rm /stackable/apache-zookeeper-${PRODUCT}-bin/README_packaging.md && \ # Download the JMX exporter jar from our own repo - curl --fail "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" \ + curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" \ -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \ chmod +x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar From a2afa606b6cb8b443eaa50f81c54bbe75acdd061 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 17 Sep 2024 11:23:32 +0200 Subject: [PATCH 2/4] Update tools/Dockerfile Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Dockerfile b/tools/Dockerfile index f953fbb52..81946690d 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -34,7 +34,7 @@ WORKDIR /stackable/bin ENV PATH=/stackable/bin:$PATH # Get latest stable version from curl -L -s https://dl.k8s.io/release/stable.txt -RUN curl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ +RUN curl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ -o /stackable/bin/kubectl && chmod +x /stackable/bin/kubectl RUN curl https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 \ From 3041fea7c05d226447c6cdd5a9122eb25eb19322 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 17 Sep 2024 11:23:39 +0200 Subject: [PATCH 3/4] Update tools/Dockerfile Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- tools/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Dockerfile b/tools/Dockerfile index 81946690d..f4fed14c3 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -37,7 +37,7 @@ ENV PATH=/stackable/bin:$PATH RUN curl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl \ -o /stackable/bin/kubectl && chmod +x /stackable/bin/kubectl -RUN curl https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 \ +RUN curl https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 \ -o /stackable/bin/jq && \ chmod +x /stackable/bin/jq From 164d3e821b15ef4b4dfbb05c65794babaa56e7c6 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 17 Sep 2024 11:31:34 +0200 Subject: [PATCH 4/4] Fix formatting/order etc. --- stackable-base/stackable/curlrc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stackable-base/stackable/curlrc b/stackable-base/stackable/curlrc index 284b6694e..32ca5fc92 100644 --- a/stackable-base/stackable/curlrc +++ b/stackable-base/stackable/curlrc @@ -5,7 +5,7 @@ # https://everything.curl.dev/http/redirects.html#tell-curl-to-follow-redirects location -# Retry the request up to 5 times on transient errors (e.g., network issues). +# Retry the request up to 10 times on transient errors (e.g., network issues) # https://everything.curl.dev/usingcurl/downloads/retry.html#retry retry = 10 @@ -17,11 +17,11 @@ retry-connrefused # https://everything.curl.dev/usingcurl/downloads/retry.html#tweak-your-retries retry-delay = 15 -# Limit the total retry time to 1200 seconds (20 minutes), this is across all retries. +# Limit the total retry time to 1200 seconds (20 minutes), this is across all retries # https://everything.curl.dev/usingcurl/downloads/retry.html#tweak-your-retries retry-max-time = 1200 -# Fail the request if the server returns an error (4xx or 5xx HTTP status code). +# Fail the request if the server returns an error (4xx or 5xx HTTP status code) # https://everything.curl.dev/http/response.html#http-response-codes fail @@ -29,15 +29,15 @@ fail # https://everything.curl.dev/usingcurl/timeouts.html#maximum-time-allowed-to-spend max-time = 600 -# Set the connection timeout to 30 seconds to avoid hanging too long on connection attempts. +# Set the connection timeout to 30 seconds to avoid hanging too long on connection attempts # https://everything.curl.dev/usingcurl/timeouts.html#never-spend-more-than-this-to-connect connect-timeout = 30 -# (Optional) Protect against downloading files larger than 100GB -# https://everything.curl.dev/usingcurl/downloads/max-filesize.html#maximum-filesize -# max-filesize = 107374182400 - # Suppress progress output, but still show errors # https://everything.curl.dev/usingcurl/verbose/index.html?highlight=silent#silence silent show-error + +# (Optional) Protect against downloading files larger than 100GB +# https://everything.curl.dev/usingcurl/downloads/max-filesize.html#maximum-filesize +# max-filesize = 107374182400