Merge pull request #1491 from Altinity/cicd-fixes-25.3.8-fips

25.3.8 fips - Packaging and cicd fixes

Author: Enmk

docker/keeper/Dockerfile

@@ -35,11 +35,8 @@ RUN arch=${TARGETARCH:-amd64} \
         arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
     esac
 
-# lts / testing / prestable / etc
-ARG REPO_CHANNEL="stable"
-ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="25.2.2.39"
-ARG PACKAGES="clickhouse-keeper"
+# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
+
 ARG DIRECT_DOWNLOAD_URLS=""
 
 # user/group precreated explicitly with fixed uid/gid on purpose.
@@ -63,12 +60,7 @@ RUN arch=${TARGETARCH:-amd64} \
             && wget -c -q "$url" \
         ; done \
     else \
-        for package in ${PACKAGES}; do \
-            cd /tmp \
-            && echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
-            && wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
-            && wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz.sha512" \
-        ; done \
+        exit 1; \
     fi \
     && cat *.tgz.sha512 | sha512sum -c \
     && for file in *.tgz; do \

docker/server/Dockerfile.alpine

@@ -32,11 +32,10 @@ RUN arch=${TARGETARCH:-amd64} \
         arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
     esac
 
-# lts / testing / prestable / etc
-ARG REPO_CHANNEL="stable"
-ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="25.2.2.39"
-ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
+
+
+# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
+
 ARG DIRECT_DOWNLOAD_URLS=""
 
 # user/group precreated explicitly with fixed uid/gid on purpose.
@@ -59,11 +58,7 @@ RUN arch=${TARGETARCH:-amd64} \
             && wget -c -q "$url" \
         ; done \
     else \
-        for package in ${PACKAGES}; do \
-            echo "Get ${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
-            && wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz" \
-            && wget -c -q "${REPOSITORY}/${package}-${VERSION}-${arch}.tgz.sha512" \
-        ; done \
+        exit 1; \
     fi \
     && cat *.tgz.sha512 | sed 's:/output/:/tmp/:' | sha512sum -c \
     && for file in *.tgz; do \

docker/server/Dockerfile.ubuntu

@@ -26,23 +26,12 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
         wget \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-ARG REPO_CHANNEL="stable"
-ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="25.2.2.39"
-ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
-
 #docker-official-library:off
 # The part between `docker-official-library` tags is related to our builds
 
-# set non-empty deb_location_url url to create a docker image
-# from debs created by CI build, for example:
-# docker build . --network host --build-arg version="21.4.1.6282" --build-arg deb_location_url="https://..." -t ...
-ARG deb_location_url=""
-ARG DIRECT_DOWNLOAD_URLS=""
+# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
 
-# set non-empty single_binary_location_url to create docker image
-# from a single binary url (useful for non-standard builds - with sanitizers, for arm64).
-ARG single_binary_location_url=""
+ARG DIRECT_DOWNLOAD_URLS=""
 
 ARG TARGETARCH
 
@@ -58,64 +47,7 @@ RUN if [ -n "${DIRECT_DOWNLOAD_URLS}" ]; then \
         && rm -rf /tmp/* ; \
     fi
 
-# install from a web location with deb packages
-RUN arch="${TARGETARCH:-amd64}" \
-    && if [ -n "${deb_location_url}" ]; then \
-        echo "installing from custom url with deb packages: ${deb_location_url}" \
-        && rm -rf /tmp/clickhouse_debs \
-        && mkdir -p /tmp/clickhouse_debs \
-        && for package in ${PACKAGES}; do \
-            { wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_${arch}.deb" -P /tmp/clickhouse_debs || \
-                wget --progress=bar:force:noscroll "${deb_location_url}/${package}_${VERSION}_all.deb" -P /tmp/clickhouse_debs ; } \
-            || exit 1 \
-        ; done \
-        && dpkg -i /tmp/clickhouse_debs/*.deb \
-        && rm -rf /tmp/* ; \
-    fi
-
-# install from a single binary
-RUN if [ -n "${single_binary_location_url}" ]; then \
-        echo "installing from single binary url: ${single_binary_location_url}" \
-        && rm -rf /tmp/clickhouse_binary \
-        && mkdir -p /tmp/clickhouse_binary \
-        && wget --progress=bar:force:noscroll "${single_binary_location_url}" -O /tmp/clickhouse_binary/clickhouse \
-        && chmod +x /tmp/clickhouse_binary/clickhouse \
-        && /tmp/clickhouse_binary/clickhouse install --user "clickhouse" --group "clickhouse" \
-        && rm -rf /tmp/* ; \
-    fi
-
-# The rest is the same in the official docker and in our build system
-#docker-official-library:on
-
-# A fallback to installation from ClickHouse repository
-# It works unless the clickhouse binary already exists
-RUN clickhouse local -q 'SELECT 1' >/dev/null 2>&1 && exit 0 || : \
-    ; apt-get update \
-    && apt-get install --yes --no-install-recommends \
-        dirmngr \
-        gnupg2 \
-    && mkdir -p /etc/apt/sources.list.d \
-    && GNUPGHOME=$(mktemp -d) \
-    && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
-        --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
-        --keyserver hkp://keyserver.ubuntu.com:80 \
-        --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
-    && rm -rf "$GNUPGHOME" \
-    && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
-    && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
-    && echo "installing from repository: ${REPOSITORY}" \
-    && apt-get update \
-    && for package in ${PACKAGES}; do \
-        packages="${packages} ${package}=${VERSION}" \
-    ; done \
-    && apt-get install --yes --no-install-recommends ${packages} || exit 1 \
-    && rm -rf \
-        /var/lib/apt/lists/* \
-        /var/cache/debconf \
-        /tmp/* \
-    && apt-get autoremove --purge -yq dirmngr gnupg2 \
-    && chmod ugo+Xrw -R /etc/clickhouse-server /etc/clickhouse-client
-# The last chmod is here to make the next one is No-op in docker official library Dockerfile
+# NOTE (strtgbb): Removed install methods other than direct URL install to tidy the Dockerfile
 
 # post install
 # we need to allow "others" access to clickhouse folder, because docker container

programs/server/config.xml

@@ -1687,7 +1687,7 @@
         <anonymize>false</anonymize>
         <!-- Default endpoint should be changed to different Sentry DSN only if you have -->
         <!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
-        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+        <endpoint></endpoint>
         <!-- Send LOGICAL_ERRORs as well (default: false) -->
         <send_logical_errors>false</send_logical_errors>
     </send_crash_reports>

programs/server/config.yaml.example

@@ -939,6 +939,6 @@ send_crash_reports:
     anonymize: false
     # Default endpoint should be changed to different Sentry DSN only if you have
     # some in-house engineers or hired consultants who're going to debug ClickHouse issues for you
-    endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
+    endpoint: ''
     # Uncomment to disable ClickHouse internal DNS caching.
     # disable_internal_dns_cache: 1

tests/ci/docker_server.py

@@ -184,8 +184,8 @@ def buildx_args(
     args = [
         f"--platform=linux/{arch}",
         f"--label=build-url={GITHUB_RUN_URL}",
-        f"--label=com.clickhouse.build.githash={git.sha}",
-        f"--label=com.clickhouse.build.version={version}",
+        f"--label=com.altinity.build.githash={git.sha}",
+        f"--label=com.altinity.build.version={version}",
     ]
     if direct_urls:
         args.append(f"--build-arg=DIRECT_DOWNLOAD_URLS='{' '.join(direct_urls)}'")

tests/integration/test_config_xml_full/configs/config.xml

@@ -995,7 +995,7 @@
         <anonymize>false</anonymize>
         <!-- Default endpoint should be changed to different Sentry DSN only if you have -->
         <!-- some in-house engineers or hired consultants who're going to debug ClickHouse issues for you -->
-        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+        <endpoint></endpoint>
     </send_crash_reports>
 
     <!-- Uncomment to disable ClickHouse internal DNS caching. -->

tests/integration/test_config_xml_main/configs/config.xml

@@ -195,6 +195,6 @@
     <send_crash_reports>
         <enabled>false</enabled>
         <anonymize>false</anonymize>
-        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+        <endpoint></endpoint>
     </send_crash_reports>
 </clickhouse>

tests/integration/test_config_xml_yaml_mix/configs/config.xml

@@ -195,7 +195,7 @@
     <send_crash_reports>
         <enabled>false</enabled>
         <anonymize>false</anonymize>
-        <endpoint>https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277</endpoint>
+        <endpoint></endpoint>
     </send_crash_reports>
 
     <mark_cache_size>123451234</mark_cache_size>

tests/integration/test_config_yaml_full/configs/config.yaml

@@ -133,5 +133,5 @@ query_masking_rules:
 send_crash_reports:
   enabled: false
   anonymize: false
-  endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
+  endpoint: ''