Update to debian bookworm and python 3.12

ostefano · ostefano · commit b06bac9b12ce · 2024-07-15T13:49:37.000+01:00
diff --git a/core/Dockerfile b/core/Dockerfile
@@ -1,25 +1,42 @@
 ARG DOCKER_HUB_PROXY=""
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
+
+FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as php-base
+    ENV DEBIAN_FRONTEND noninteractive
+
+    # Uncomment when building in corporate environments
+    # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem
+    # COPY ./rootca.crt /usr/lib/ssl/cert.pem
+
+    RUN apt-get update; apt-get install -y --no-install-recommends \
+        lsb-release \
+        ca-certificates \
+        curl
+    RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb
+    RUN dpkg -i /tmp/debsuryorg-archive-keyring.deb
+    RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list
+    RUN apt-get update
+
+
+FROM php-base as composer-build
     ENV DEBIAN_FRONTEND noninteractive
     ENV COMPOSER_ALLOW_SUPERUSER 1
     ARG CORE_TAG
     ARG CORE_COMMIT
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
-        ca-certificates \
-        php \
-        php-apcu \
-        php-curl \
-        php-xml \
-        php-intl \
-        php-bcmath \
-        php-mbstring \
-        php-mysql \
-        php-redis \
-        php-gd \
-        php-fpm \
-        php-zip \
+    RUN apt-get install -y --no-install-recommends \
+        php7.4 \
+        php7.4-apcu \
+        php7.4-curl \
+        php7.4-xml \
+        php7.4-intl \
+        php7.4-bcmath \
+        php7.4-mbstring \
+        php7.4-mysql \
+        php7.4-redis \
+        php7.4-gd \
+        php7.4-fpm \
+        php7.4-zip \
         unzip \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
@@ -29,41 +46,53 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
     RUN composer config --no-interaction allow-plugins.composer/installers true
     RUN composer install
     RUN composer require --with-all-dependencies --no-interaction \
-            supervisorphp/supervisor:^4.0 \
-            guzzlehttp/guzzle \
-            lstrojny/fxmlrpc \
-            php-http/message \
-            php-http/message-factory \
-            # docker image specific dependencies
-            elasticsearch/elasticsearch:^8.7.0 \
-            jakub-onderka/openid-connect-php:^1.0.0 \
-            aws/aws-sdk-php
-
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
+        supervisorphp/supervisor:^4.0 \
+        guzzlehttp/guzzle \
+        lstrojny/fxmlrpc \
+        php-http/message \
+        php-http/message-factory \
+        # docker image specific dependencies
+        elasticsearch/elasticsearch:^8.7.0 \
+        jakub-onderka/openid-connect-php:^1.0.0 \
+        aws/aws-sdk-php
+
+
+FROM php-base as php-build
     ENV DEBIAN_FRONTEND noninteractive
     ENV TZ Etc/UTC
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
+    RUN apt-get install -y --no-install-recommends \
         gcc \
         g++ \
         make \
+        php7.4 \
+        php7.4-dev \
+        php7.4-xml \
         libfuzzy-dev \
-        ca-certificates \
-        php \
-        php-dev \
-        php-xml \
-        php-pear \
         librdkafka-dev \
         libsimdjson-dev \
         libzstd-dev \
         git \
+        php-pear \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
-    RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson && pecl install zstd
-    RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
-        cd php-ext-brotli && phpize && ./configure && make && make install
+    RUN apt-cache search pecl
+
+    RUN update-alternatives --set php /usr/bin/php7.4
+    RUN update-alternatives --set php-config /usr/bin/php-config7.4
+    RUN update-alternatives --set phpize /usr/bin/phpize7.4
+
+    RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib
+    RUN pecl channel-update pecl.php.net && \
+        pecl install ssdeep && \
+        pecl install rdkafka && \
+        pecl install simdjson && \
+        pecl install zstd
+     RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
+         cd php-ext-brotli && phpize && ./configure && make && make install
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
+
+FROM php-base as python-build
     ENV DEBIAN_FRONTEND noninteractive   
     ARG CORE_TAG
     ARG CORE_COMMIT
@@ -77,8 +106,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
     ARG PYPI_CYBOX_VERSION
     ARG PYPI_PYMISP_VERSION
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
-        python3-pip \
+    RUN apt-get install -y --no-install-recommends \
         git \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
     
@@ -89,7 +117,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
         else
             git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP
         fi
-
         cd /var/www/MISP || exit; git submodule update --init --recursive .
 EOF
 
@@ -124,7 +151,7 @@ EOF
             fi
         done;
 
-        pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
+        pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
         
         # Remove files we do not care for
         rm -r /var/www/MISP/PyMISP
@@ -134,13 +161,14 @@ EOF
         find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} +
 EOF
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
+
+FROM php-base
     ENV DEBIAN_FRONTEND noninteractive
     ARG CORE_TAG
     ARG CORE_COMMIT
     ARG PHP_VER
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
+    RUN apt-get install -y --no-install-recommends \
         gettext \
         procps \
         sudo \
@@ -150,42 +178,41 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
         openssl \
         gpg \
         gpg-agent \
-        ssdeep \
-        libfuzzy2 \
         mariadb-client \
         rsync \
-        # Python Requirements
-        python3 \
-        python3-setuptools \
-        python3-pip \
         # PHP Requirements
-        php \
-        php-apcu \
-        php-curl \
-        php-xml \
-        php-intl \
-        php-bcmath \
-        php-mbstring \
-        php-mysql \
-        php-redis \
-        php-gd \
-        php-fpm \
-        php-zip \
-        php-ldap \
+        php7.4 \
+        php7.4-apcu \
+        php7.4-curl \
+        php7.4-xml \
+        php7.4-intl \
+        php7.4-bcmath \
+        php7.4-mbstring \
+        php7.4-mysql \
+        php7.4-redis \
+        php7.4-gd \
+        php7.4-fpm \
+        php7.4-zip \
+        php7.4-ldap \
+        libmagic1 \
         libldap-common \
         librdkafka1 \
         libbrotli1 \
-        libsimdjson5 \
+        libsimdjson14 \
         libzstd1 \
+        ssdeep \
+        libfuzzy2 \
         # Unsure we need these
         zip unzip \
         # Require for advanced an unattended configuration
         curl jq \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
+    RUN update-alternatives --set php /usr/bin/php7.4
+
     # Install python modules
     COPY --from=python-build /wheels /wheels
-    RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels
+    RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels
 
     # PHP: install prebuilt libraries, then install the app's PHP deps
     COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"]
@@ -208,7 +235,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
 EOF
 
     # nginx
-    RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs
+    RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs
 
     # Make a copy of the file and configuration stores, so we can sync from it
 
diff --git a/modules/Dockerfile b/modules/Dockerfile
@@ -1,6 +1,6 @@
 ARG DOCKER_HUB_PROXY=""
 
-FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
+FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build
     ENV DEBIAN_FRONTEND noninteractive
     ARG MODULES_TAG
     ARG MODULES_COMMIT
@@ -9,72 +9,82 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
     RUN apt-get update && apt-get install -y --no-install-recommends \
         cmake \
         git \
-        python3-dev \
-        python3-pip \
-        python3-wheel \
         build-essential \
-        pkg-config \
         libpoppler-cpp-dev \
         libfuzzy-dev \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev  \
         libssl-dev \
         ninja-build \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
     
     RUN mkdir /wheels
-    WORKDIR /srv
 
     RUN <<-EOF
         if [ ! -z ${MODULES_COMMIT} ]; then
             git clone https://github.com/MISP/misp-modules.git /srv/misp-modules && cd /srv/misp-modules && git checkout ${MODULES_COMMIT}
         else
             git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules
         fi
-
-        cd /srv/misp-modules
-        sed -i "s/numpy.*/numpy<2/" REQUIREMENTS
-        echo "pyeti" >> REQUIREMENTS
-        echo "greynoise" >> REQUIREMENTS
-        echo "Google-Search-API" >> REQUIREMENTS
-        pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/
-        rm -rf /srv/misp-modules
 EOF
 
+    WORKDIR /srv/misp-modules
+    RUN pip install pipenv
+    RUN sed -i "s/python_version.*/python_version = \"3.12\"/" Pipfile
+    RUN sed -i "/\[packages\]/a numpy = \"<2.0.0\"" Pipfile
+    RUN sed -i "/\[packages\]/a pysafebrowsing = \"*\"" Pipfile
+    RUN sed -i "/\[packages\]/a sigmf = \"*\"" Pipfile
+    RUN sed -i "/\[packages\]/a matplotlib = \"*\"" Pipfile
+    RUN pipenv lock
+    RUN pipenv requirements > requirements.txt
+    RUN pip wheel -r requirements.txt --no-cache-dir -w /wheels/
+    
+    WORKDIR /srv/
+    RUN rm -rf /srv/misp-modules
+
     RUN <<-EOF
         git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup
         cd /srv/faup
         if [ ! -z ${LIBFAUP_COMMIT} ]; then
             git checkout ${LIBFAUP_COMMIT}
         fi
-
-        cd /srv/faup/build
-        cmake -G "Ninja" ../
-        ninja
-        ninja install
-        cd /srv/faup/src/lib/bindings/python
-        pip3 wheel --no-cache-dir --no-dependencies -w /wheels/ .
-        rm -rf /srv/faup
 EOF
+    
+    WORKDIR /srv/faup/build
+    RUN cmake -G "Ninja" ../
+    RUN ninja
+    RUN ninja install
+    WORKDIR /srv/faup/src/lib/bindings/python
+    RUN pip wheel --no-cache-dir --no-dependencies -w /wheels/ .
+
+    WORKDIR /srv/
+    RUN rm -rf /srv/faup
 
-FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye"
+
+FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm"
     ENV DEBIAN_FRONTEND noninteractive
 
     RUN apt-get update && apt-get install -y --no-install-recommends \
         libglib2.0-0 \
-        libzbar0 \
-        libxrender1 \
-        libxext6 \
         libpoppler-cpp0v5 \
         libgl1 \
+        libfuzzy2 \
+        libffi8 \
+        libxext6 \
+        libxml2 \
+        libxslt1.1  \
+        libzbar0 \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
     
     COPY --from=python-build /wheels /wheels
     COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/
-    RUN pip3 install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels
+    RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels
     
     # Since we compile faup ourselves and lua is not required anymore, we can load our own library 
     #   and skip the pre-compiled blob to improve compatibility with other architectures like ARM
     RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
-        /usr/local/lib/python3.11/site-packages/pyfaup/__init__.py
+        /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py
 
     # Disable (all) warnings raised when using 'future'
     RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \
