Merge pull request ClickHouse#78667 from azat/bump-gdb

cctools: update gdb to 16.2 (6x faster backtraces on CI) and fix Apple TAPI/cctools port

Author: azat

ci/defs/defs.py

@@ -101,15 +101,12 @@ class CIFiles:
     #     platforms=Docker.Platforms.arm_amd,
     #     depends_on=[],
     # ),
-    # TODO: fix build failure:
-    # 7 58.76 In file included from ./../code-sign-blobs/superblob.h:7:
-    # 7 58.76 ./../code-sign-blobs/blob.h:185:60: error: no member named 'clone' in 'Security::BlobCore'
-    # Docker.Config(
-    #     name="clickhouse/cctools",
-    #     path="./docker/packager/cctools",
-    #     platforms=Docker.Platforms.arm_amd,
-    #     depends_on=["clickhouse/fasttest"],
-    # ),
+    Docker.Config(
+        name="clickhouse/cctools",
+        path="./docker/packager/cctools",
+        platforms=Docker.Platforms.arm_amd,
+        depends_on=["clickhouse/fasttest"],
+    ),
     Docker.Config(
         name="clickhouse/test-util",
         path="./docker/test/util",

ci/docker/binary-builder/Dockerfile

@@ -5,7 +5,7 @@ ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
 
 # If the cctools is updated, then first build it in the CI, then update here in a different commit
-COPY --from=clickhouse/cctools:d9e3596e706b /cctools /cctools
+COPY --from=clickhouse/cctools:13cfebfd8f1bbc65ab49 /cctools /cctools
 
 # A cross-linker for RISC-V 64 (we need it, because LLVM's LLD does not work):
 RUN apt-get update \

ci/docker/fasttest/Dockerfile

@@ -67,7 +67,7 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb
+COPY --from=clickhouse/cctools:13cfebfd8f1bbc65ab49 /opt/gdb /opt/gdb
 # Give suid to gdb to grant it attach permissions
 RUN chmod u+s /opt/gdb/bin/gdb
 ENV PATH="/opt/gdb/bin:${PATH}"

docker/packager/cctools/Dockerfile

@@ -4,24 +4,27 @@
 # It's based on the assumption that we don't care of the cctools version so much
 # It event does not depend on the clickhouse/fasttest in the `docker/images.json`
 ARG FROM_TAG=latest
-FROM clickhouse/fasttest:$FROM_TAG as builder
+FROM clickhouse/fasttest:$FROM_TAG AS builder
 
 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
 
 RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
     && cd apple-libtapi \
     && git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9 \
-    && INSTALLPREFIX=/cctools ./build.sh \
+    && INSTALLPREFIX=/cctools NINJA=1 ./build.sh \
     && ./install.sh \
     && cd .. \
     && rm -rf apple-libtapi
 
 # Build and install tools for cross-linking to Darwin (x86-64)
 # Build and install tools for cross-linking to Darwin (aarch64)
-RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+RUN git clone https://github.com/tpoechtrager/cctools-port.git --branch 986-ld64-711 \
+    && apt-get update \
+    && apt-get install --yes \
+        make \
+    && apt-get clean \
     && cd cctools-port/cctools \
-    && git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7 \
     && ./configure --prefix=/cctools --with-libtapi=/cctools \
         --target=x86_64-apple-darwin \
     && make install -j$(nproc) \
@@ -40,7 +43,7 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
 #     DWARF error: invalid or unhandled FORM value: 0x23
 #
 ENV LD=ld.lld-${LLVM_VERSION}
-ARG GDB_VERSION=15.1
+ARG GDB_VERSION=16.2
 RUN apt-get update \
     && apt-get install --yes \
         libgmp-dev \
@@ -58,3 +61,5 @@ RUN wget https://sourceware.org/pub/gdb/releases/gdb-$GDB_VERSION.tar.gz \
 FROM scratch
 COPY --from=builder /cctools /cctools
 COPY --from=builder /opt/gdb /opt/gdb
+
+# NOTE: this file should be kept in sync with other docker images, since they use COPY from this image

docker/test/integration/base/Dockerfile

@@ -73,5 +73,5 @@ maxClientCnxns=80' > /opt/zookeeper/conf/zoo.cfg && \
 ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb
+COPY --from=clickhouse/cctools:13cfebfd8f1bbc65ab49 /opt/gdb /opt/gdb
 ENV PATH="/opt/gdb/bin:${PATH}"

docker/test/integration/runner/Dockerfile

@@ -85,7 +85,7 @@ COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/
 COPY misc/ /misc/
 
-COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb
+COPY --from=clickhouse/cctools:13cfebfd8f1bbc65ab49 /opt/gdb /opt/gdb
 ENV PATH="/opt/gdb/bin:${PATH}"
 
 # Same options as in test/base/Dockerfile

docker/test/performance-comparison/Dockerfile

@@ -41,7 +41,7 @@ RUN pip3 --no-cache-dir install -r requirements.txt
 
 COPY run.sh /
 
-COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb
+COPY --from=clickhouse/cctools:13cfebfd8f1bbc65ab49 /opt/gdb /opt/gdb
 ENV PATH="/opt/gdb/bin:${PATH}"
 
 CMD ["bash", "/run.sh"]

docker/test/util/Dockerfile

@@ -56,5 +56,5 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
-COPY --from=clickhouse/cctools:0d6b90a7a490 /opt/gdb /opt/gdb
+COPY --from=clickhouse/cctools:13cfebfd8f1bbc65ab49 /opt/gdb /opt/gdb
 ENV PATH="/opt/gdb/bin:${PATH}"