steineggerlab
diff --git a/‎lib/foldseek/.github/workflows/Dockerfile.GPU-manylinux2014‎
Lines changed: 83 additions & 0 deletions b/‎lib/foldseek/.github/workflows/Dockerfile.GPU-manylinux2014‎
Lines changed: 83 additions & 0 deletions
diff --git a/‎lib/foldseek/.github/workflows/docker.yml‎
Lines changed: 62 additions & 1 deletion b/‎lib/foldseek/.github/workflows/docker.yml‎
Lines changed: 62 additions & 1 deletion
diff --git a/‎lib/foldseek/.gitmodules‎
Lines changed: 4 additions & 0 deletions b/‎lib/foldseek/.gitmodules‎
Lines changed: 4 additions & 0 deletions
diff --git a/‎lib/foldseek/CMakeLists.txt‎
Lines changed: 61 additions & 21 deletions b/‎lib/foldseek/CMakeLists.txt‎
Lines changed: 61 additions & 21 deletions
diff --git a/‎lib/foldseek/Dockerfile‎
Lines changed: 45 additions & 21 deletions b/‎lib/foldseek/Dockerfile‎
Lines changed: 45 additions & 21 deletions
@@ -0,0 +1,83 @@
+ARG APP=foldseek
+FROM quay.io/pypa/manylinux2014_x86_64
+ARG APP
+ARG VER="12-6"
+ARG ARCH="x86_64"
+
+# CUDA
+RUN yum install -y yum-utils
+RUN yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
+RUN yum -y install cuda-nvcc-${VER}.${ARCH} cuda-cudart-devel-${VER}.${ARCH} libcublas-devel-${VER}.${ARCH}
+RUN echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/999_nvidia_cuda.conf
+ENV PATH="/usr/local/cuda/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"
+ENV CUDA_HOME=/usr/local/cuda
+ENV CUDA_ROOT=/usr/local/cuda
+ENV CUDA_PATH=/usr/local/cuda
+ENV CUDADIR=/usr/local/cuda
+
+# Build system
+RUN yum install -y git curl wget vim zlib-devel bzip2-devel ninja-build centos-release-scl
+RUN mv /etc/yum.repos.d/CentOS-SCLo-scl.repo /etc/yum.repos.d/CentOS-SCLo-scl.repo.disabled
+RUN yum install -y devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-libatomic-devel
+ENV CC=/opt/rh/devtoolset-11/root/bin/gcc
+ENV CXX=/opt/rh/devtoolset-11/root/bin/g++
+ENV CUDAHOSTCXX=/opt/rh/devtoolset-11/root/bin/g++
+ENV CUDACXX=/usr/local/cuda/bin/nvcc
+ENV LIBGCC=/opt/rh/devtoolset-11/root/usr/lib/gcc/x86_64-redhat-linux/11
+
+# cmake
+RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.0/cmake-3.31.0-linux-x86_64.sh; \
+    chmod +x cmake-3.31.0-linux-x86_64.sh; \
+    ./cmake-3.31.0-linux-x86_64.sh --skip-license --prefix=/usr/local;
+
+RUN mkdir /deps; \
+    cd /deps; \
+    wget https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz; \
+    tar xzvf bzip2-1.0.8.tar.gz; \
+    cd bzip2-1.0.8; \
+    make install PREFIX=/deps;
+
+RUN cd /deps; \
+    wget https://www.zlib.net/zlib-1.3.1.tar.gz; \
+    tar xzvf zlib-1.3.1.tar.gz; \
+    cd zlib-1.3.1; \
+    ./configure --prefix=/deps --static; \
+    make install;
+
+# install toolchain
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --default-toolchain stable -y
+ENV PATH=/root/.cargo/bin:$PATH
+
+# compile
+WORKDIR /work
+ADD . .
+RUN mkdir -p /work/build && cd /work/build; \
+    if [ -e "${LIBGCC}/libgomp.so" ]; then \
+        mv -f -- "${LIBGCC}/libgomp.so" "${LIBGCC}/libgomp.so.disabled"; \
+    fi; \
+    /usr/local/bin/cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DHAVE_TESTS=1 -DENABLE_WERROR=1 -DHAVE_AVX2=1 \
+        -DOpenMP_C_FLAGS="-fopenmp -I${LIBGCC} -L${LIBGCC}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGCC} -L${LIBGCC}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY="${LIBGCC}/libgomp.a" \
+        -DATOMIC_LIB_OVERRIDE="${LIBGCC}/libatomic.a" \
+        -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_POLICY_DEFAULT_CMP0144=NEW \
+        -DZLIB_ROOT=/deps -DBZIP2_ROOT=/deps \
+        -DFORCE_STATIC_DEPS=1 -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" ..; \
+    cmake --build . -j$(nproc --all) -v;
+
+RUN if ldd /work/build/src/${APP} | grep -P -v "linux-vdso.so|/lib64/(ld-linux-x86-64|libc|libm|libdl|librt|libpthread|libutil).so" | grep -q .; then \
+        echo "Error: unwanted libraries found"; \
+        ldd /work/build/src/${APP}; \
+        exit 1; \
+    fi; \
+    if readelf -Ws /work/build/src/${APP} | grep -q GLIBC_PRIVATE; then \
+        echo "Error: binary contains private glibc symbols"; \
+        readelf -Ws /work/build/src/${APP}; \
+        exit 1; \
+    fi; \
+    LIBC_V=$(readelf -V /work/build/src/${APP} | awk '$3 ~ /^GLIBC_/ { print $3 }' | sort -V | tail -n1); \
+    if [[ "$LIBC_V" > "GLIBC_2.17" ]]; then \
+        echo "Error: glibc too new"; \
+        readelf -V /work/build/src/${APP}; \
+        exit 1; \
+    fi;
@@ -11,9 +11,11 @@ on:
       tag:
         required: true
         type: string
+        description: "Docker tag"
       latest:
         default: false
         type: boolean
+        description: "Mark as latest"
 
 
 env:
@@ -45,7 +47,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
@@ -77,4 +79,63 @@ jobs:
             ${{ steps.dispatch_tag.outputs.tag }}
             ${{ steps.dispatch_tag.outputs.latest }}
           labels: ${{ steps.meta.outputs.labels }}
+  build-and-push-gpu-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=-cuda12,onlatest=true
+
+      - name: Tag for workflow_dispatch
+        id: dispatch_tag
+        run: |
+          if [ x"$TAG" != x"" ];then
+            echo "::set-output name=tag::${FULL_TAG}"
+          fi
+          if [ x"$LATEST" = x"true" ]; then
+            echo "::set-output name=latest::${LATEST_TAG}"
+          fi
+        env:
+          FULL_TAG: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.tag }}
+          LATEST_TAG: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          TAG: ${{ github.event.inputs.tag }}
+          LATEST: ${{ github.event.inputs.latest }}
 
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-args: |
+            GPU=1
+          tags: |
+            ${{ steps.meta.outputs.tags }}
+            ${{ steps.dispatch_tag.outputs.tag }}
+            ${{ steps.dispatch_tag.outputs.latest }}
+          labels: ${{ steps.meta.outputs.labels }}
@@ -2,6 +2,10 @@
 	path = lib/mmseqs/util/regression
 	url = https://github.com/soedinglab/MMseqs2-Regression.git
 	update = none
+[submodule "lib/prostt5/ggml/src/ggml-kompute/kompute"]
+	path = lib/prostt5/ggml/src/ggml-kompute/kompute
+	url =
+	update = none
 [submodule "regression"]
 	path = regression
 	url = https://github.com/steineggerlab/foldseek-regression
@@ -2,17 +2,31 @@ cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
 # keep old policy for setting implicit link libraries
 # zlib causes issues in static builds otherwise
 cmake_policy(SET CMP0060 OLD)
+cmake_policy(SET CMP0074 NEW)
 project(foldseek C CXX)
-#set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/lib/mmseqs/cmake")
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/lib/mmseqs/cmake")
 
 set(ENABLE_PROSTT5 1 CACHE BOOL "Enable ProstT5")
 set(ENABLE_CUDA 0 CACHE BOOL "Enable CUDA")
 set(IGNORE_RUST_VERSION 0 CACHE BOOL "Ignore Rust version check")
+set(FORCE_STATIC_DEPS 0 CACHE BOOL "Force static linking of deps")
 
-if (NOT CMAKE_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE Release)
-endif ()
+if(FORCE_STATIC_DEPS)
+    if(ENABLE_CUDA)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES .a .so CACHE INTERNAL "" FORCE)
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
+    else()
+        set(CMAKE_FIND_LIBRARY_SUFFIXES .a CACHE INTERNAL "" FORCE)
+        set(CMAKE_LINK_SEARCH_START_STATIC ON CACHE INTERNAL "" FORCE)
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++")
+    endif()
+    set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
+endif()
+
+if(NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+    set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
 
 set(FRAMEWORK_ONLY 1 CACHE INTERNAL "" FORCE)
 include(MMseqsSetupDerivedTarget)
@@ -38,6 +52,7 @@ set(ENV{CARGO_NET_OFFLINE} true)
 add_subdirectory(lib/corrosion)
 # don't try to link to gcc_s, its not needed anyway
 list(REMOVE_ITEM Rust_CARGO_TARGET_LINK_NATIVE_LIBS "gcc_s")
+set(Rust_CARGO_TARGET_LINK_NATIVE_LIBS ${Rust_CARGO_TARGET_LINK_NATIVE_LIBS} CACHE INTERNAL "")
 
 if(Rust_VERSION VERSION_LESS "1.68.0" AND NOT IGNORE_RUST_VERSION)
     message(FATAL_ERROR "Rust >= 1.68.0 is required")
@@ -80,27 +95,52 @@ if(EMSCRIPTEN)
     corrosion_add_target_local_rustflags(block_aligner_c "-Clink-args=--no-entry -sRELOCATABLE=1")
 endif()
 
-set(CANDLE_FEATURE "")
-if(ENABLE_CUDA)
-    if(NOT DEFINED CUDAToolkit_ROOT)
-        message(FATAL_ERROR "please set -DCUDAToolkit_ROOT=path-to-cuda")
-    endif()
-    set(CANDLE_FEATURE cuda)
+if(HAVE_SSE2)
+    message(WARNING "ProstT5 support requires at least SSE4.1, disabling ProstT5")
+    set(ENABLE_PROSTT5 OFF)
 endif()
 
 if(ENABLE_PROSTT5)
-    corrosion_import_crate(
-        MANIFEST_PATH lib/prostt5/c/Cargo.toml
-        CRATE_TYPES staticlib
-        FEATURES "${CANDLE_FEATURE}"
-    )
-    if(ENABLE_CUDA)
-        corrosion_set_env_vars(cprostt5 CUDA_ROOT=${CUDAToolkit_ROOT})
+    set(CMAKE_SYSTEM_PROCESSOR_TMP ${CMAKE_SYSTEM_PROCESSOR})
+    # circumvent ggml's host CPU detection
+    if(ARM)
+        set(CMAKE_SYSTEM_PROCESSOR aarch64)
     endif()
-    include_directories(lib/prostt5/c)
-    if(EMSCRIPTEN)
-        corrosion_add_target_local_rustflags(cprostt5 "-Clink-args=--no-entry -sRELOCATABLE=1")
+    if(X86 OR X64)
+        set(CMAKE_SYSTEM_PROCESSOR x86_64)
+    endif()
+    if(PPC64)
+        set(CMAKE_SYSTEM_PROCESSOR ppc64)
+    endif()
+    set(GGML_STATIC ON)
+    set(BUILD_SHARED_LIBS OFF)
+    set(GGML_BLAS OFF)
+    set(GGML_OPENMP OFF)
+    if (NOT NATIVE_ARCH)
+        set(GGML_NATIVE OFF)
+        if (HAVE_AVX2)
+            set(GGML_AVX2 ON)
+            set(GGML_AVX ON)
+            set(GGML_FMA ON)
+            set(GGML_F16C ON)
+        elseif(HAVE_SSE4_1)
+            set(GGML_AVX2 OFF)
+            set(GGML_AVX OFF)
+            set(GGML_FMA OFF)
+            set(GGML_F16C OFF)
+        endif()
+    endif()
+    if (ENABLE_CUDA)
+        set(GGML_CUDA ON)
+        set(GGML_CUDA_F16 ON)
+        set(GGML_CUDA_FORCE_CUBLAS ON)
+        set(GGML_CUDA_NO_VMM ON)
+    endif()
+    if (APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+        set(GGML_METAL OFF)
     endif()
+    add_subdirectory(lib/prostt5 EXCLUDE_FROM_ALL)
+    set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR_TMP})
 endif()
 
 add_subdirectory(src)
@@ -1,15 +1,21 @@
 ARG APP=foldseek
-FROM --platform=$BUILDPLATFORM debian:stable-backports as builder
+FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS builder
 ARG TARGETARCH
 ARG APP
+ARG GPU
 
 RUN dpkg --add-architecture $TARGETARCH \
     && apt-get update \
     && apt-get install -y \
       build-essential curl xxd git cmake \
       zlib1g-dev libbz2-dev libatomic1 \
-      crossbuild-essential-$TARGETARCH zlib1g-dev:$TARGETARCH libbz2-dev:$TARGETARCH \
-    && rm -rf /var/lib/apt/lists/*
+      crossbuild-essential-$TARGETARCH zlib1g-dev:$TARGETARCH libbz2-dev:$TARGETARCH; \
+    if [ "$GPU" = "1" ]; then \
+      curl -O https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb; \
+      dpkg -i cuda-keyring_1.1-1_all.deb; \
+      apt-get update && apt-get install -y cuda-nvcc-12-6 cuda-cudart-dev-12-6 libcublas-dev-12-6 ninja-build; \
+    fi; \
+    rm -rf /var/lib/apt/lists/*;
 
 RUN if [ "$TARGETARCH" = "arm64" ]; then \
       ARCH=aarch64; \
@@ -29,31 +35,49 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
       CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ cmake -DHAVE_ARM8=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
       make -j $(nproc --all); \
       mv src/${APP} /opt/build/${APP}_arch; \
-      touch /opt/build/${APP}_sse2 /opt/build/${APP}_avx2; \
+      touch /opt/build/${APP}_sse41 /opt/build/${APP}_avx2; \
     else \
-      mkdir -p build_sse2/src && mkdir -p build_avx2/src; \
-      cd /opt/build/build_sse2; \
-      cmake -DHAVE_SSE2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
-      make -j $(nproc --all); \
-      mv src/${APP} /opt/build/${APP}_sse2; \
-      cd /opt/build/build_avx2; \
-      cmake -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
-      make -j $(nproc --all); \
-      mv src/${APP} /opt/build/${APP}_avx2; \
-      touch /opt/build/${APP}_arch; \
+      if [ "$GPU" = "1" ]; then \
+        export CUDACXX=/usr/local/cuda/bin/nvcc; \
+        mkdir -p build_avx2/src; \
+        cd /opt/build/build_avx2; \
+        LIBGOMP=/usr/lib/gcc/x86_64-linux-gnu/12/; \
+        cmake -GNinja -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DFORCE_STATIC_DEPS=1 \
+          -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES="75-real;80-real;86-real;89-real;90" \
+          -DOpenMP_C_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_C_LIB_NAMES=gomp -DOpenMP_CXX_FLAGS="-fopenmp -I${LIBGOMP}" -DOpenMP_CXX_LIB_NAMES=gomp -DOpenMP_gomp_LIBRARY=${LIBGOMP}/libgomp.a \
+          -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        cmake --build . -j$(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_avx2; \
+        touch /opt/build/${APP}_arch /opt/build/${APP}_sse41; \
+      else \
+        mkdir -p build_sse41/src && mkdir -p build_avx2/src; \
+        cd /opt/build/build_sse41; \
+        cmake -DHAVE_SSE4_1=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_sse41; \
+        cd /opt/build/build_avx2; \
+        cmake -DHAVE_AVX2=1 -DHAVE_MPI=0 -DHAVE_TESTS=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. ..; \
+        make -j $(nproc --all); \
+        mv src/${APP} /opt/build/${APP}_avx2; \
+        touch /opt/build/${APP}_arch; \
+      fi; \
     fi
 
-FROM debian:stable-slim
+FROM debian:bookworm-slim
 ARG TARGETARCH
 ARG APP
+ARG GPU
+
+COPY --from=builder /opt/build/${APP}_arch /opt/build/${APP}_sse41 /opt/build/${APP}_avx2 /usr/local/bin/
+ADD util/${APP}_wrapper.sh /usr/local/bin/entrypoint
 
 RUN apt-get update && apt-get install -y \
       gawk bash grep libstdc++6 libgomp1 libatomic1 zlib1g libbz2-1.0 wget tar aria2 \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --from=builder /opt/build/${APP}_arch /opt/build/${APP}_sse2 /opt/build/${APP}_avx2 /usr/local/bin/
-ADD util/${APP}_wrapper.sh /usr/local/bin/entrypoint
-RUN if [ "$TARGETARCH" = "arm64" ]; then rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_arch /usr/local/bin/entrypoint; fi
+    && rm -rf /var/lib/apt/lists/*; \
+    if [ "$TARGETARCH" = "arm64" ]; then \
+      rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_arch /usr/local/bin/entrypoint; \
+    elif [ "$GPU" = "1" ]; then \
+      rm -f /usr/local/bin/entrypoint; ln -s /usr/local/bin/${APP}_avx2 /usr/local/bin/entrypoint; \
+    fi
 
 ENTRYPOINT ["/usr/local/bin/entrypoint"]
-