diff --git a/.github/workflows/call-build-macos-packages.yaml b/.github/workflows/call-build-macos-packages.yaml index bc774099..656d42ba 100644 --- a/.github/workflows/call-build-macos-packages.yaml +++ b/.github/workflows/call-build-macos-packages.yaml @@ -44,7 +44,7 @@ jobs: run: | brew update # We ignore errors here as some packages may already be installed - brew install bison flex gnu-sed libyaml openssl pkgconfig libgit2 cyrus-sasl rocksdb || true + brew install bison flex gnu-sed libyaml openssl pkgconfig libgit2 cyrus-sasl rocksdb libtool || true # Remove dynamic linkage for libyaml rm -fv $(brew --prefix libyaml)/lib/*.dylib # Remove dynamic linkage for zstd added by curl install @@ -96,12 +96,6 @@ jobs: DYLD_PRINT_LIBRARIES=1 DYLD_PRINT_LIBRARIES_POST_LAUNCH=1 DYLD_PRINT_RPATHS=1 bin/fluentdo-agent --dry-run working-directory: source/build - - name: Check config - run: testing/verify-config.sh - env: - FLUENT_BIT_BINARY: ${{ github.workspace }}/source/build/bin/fluentdo-agent - shell: bash - - name: Upload build packages uses: actions/upload-artifact@v5 with: diff --git a/.github/workflows/call-build-windows-packages.yaml b/.github/workflows/call-build-windows-packages.yaml index e6099eab..f3abc63a 100644 --- a/.github/workflows/call-build-windows-packages.yaml +++ b/.github/workflows/call-build-windows-packages.yaml @@ -115,6 +115,12 @@ jobs: C:\vcpkg\vcpkg install --recurse libgit2 --triplet ${{ matrix.config.vcpkg_triplet }} shell: cmd + # Unfortunately static builds are not supported and trigger link errors + # - name: Build libsasl with vcpkg + # run: | + # C:\vcpkg\vcpkg install --recurse cyrus-sasl --allow-unsupported --triplet ${{ matrix.config.vcpkg_triplet }} + # shell: cmd + - name: Upgrade any outdated vcpkg packages run: | C:\vcpkg\vcpkg upgrade --no-dry-run diff --git a/build.sh b/build.sh new file mode 120000 index 00000000..421f7e08 --- /dev/null +++ b/build.sh @@ -0,0 +1 @@ +source/packaging/build.sh \ No newline at end of file diff --git a/scripts/add-ignored-files.sh b/scripts/add-ignored-files.sh index b836c288..3036fa35 100755 --- a/scripts/add-ignored-files.sh +++ b/scripts/add-ignored-files.sh @@ -15,7 +15,9 @@ while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli done SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) SOURCE_DIR=${SOURCE_DIR:-$SCRIPT_DIR/../source} -export FLUENT_BIT_VERSION=${FLUENT_BIT_VERSION:-v4.0.5} + +# Retrieve the version to use if not specified +export FLUENT_BIT_VERSION=${FLUENT_BIT_VERSION:-$(cat "${SOURCE_DIR}"/oss_version.txt)} while IFS= read -r ignoredFile do diff --git a/scripts/create-sync-pr.sh b/scripts/create-sync-pr.sh index d6deb023..814bc4e7 100755 --- a/scripts/create-sync-pr.sh +++ b/scripts/create-sync-pr.sh @@ -7,8 +7,19 @@ set -euo pipefail # This script creates a GitHub PR for upstream sync commits # ============================================================================= -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -REPO_ROOT="${SCRIPT_DIR}/.." +# This does not work with a symlink to this script +# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# See https://stackoverflow.com/a/246128/24637657 +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) + SOURCE=$(readlink "$SOURCE") + # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE +done +SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) + +REPO_ROOT=${REPO_ROOT:-${SCRIPT_DIR}/..} cd "$REPO_ROOT" # Colors for output diff --git a/scripts/sync-upstream-patches.sh b/scripts/sync-upstream-patches.sh index ce1ef38e..f1df8f6c 100755 --- a/scripts/sync-upstream-patches.sh +++ b/scripts/sync-upstream-patches.sh @@ -11,8 +11,19 @@ set -euo pipefail # we use patch files to apply changes selectively. # ============================================================================= -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -REPO_ROOT="${SCRIPT_DIR}/.." +# This does not work with a symlink to this script +# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# See https://stackoverflow.com/a/246128/24637657 +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) + SOURCE=$(readlink "$SOURCE") + # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE +done +SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) + +REPO_ROOT=${REPO_ROOT:-${SCRIPT_DIR}/..} cd "$REPO_ROOT" || exit 1 # Configuration diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 51b92fc1..57e06cda 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -193,6 +193,8 @@ option(FLB_CONFIG_YAML "Enable YAML config format" Yes) # List of plugins available and defaults for each option include(cmake/plugins_options.cmake) +# Add any special Enterprise support +include(cmake/enterprise.cmake) # Event loop backend (advanced users only) # ---------------------------------------- @@ -777,12 +779,15 @@ if(FLB_AWS AND FLB_KAFKA) set(FLB_HAVE_AWS_MSK_IAM ON) # Set CMake variable FLB_DEFINITION(FLB_HAVE_AWS_MSK_IAM) # Set preprocessor definition message(STATUS "AWS MSK IAM authentication: ENABLED") - else() + elseif(FLB_SYSTEM_WINDOWS) + # Windows has an issue with providing a static build of cyrus-sasl message(STATUS "AWS MSK IAM authentication: DISABLED (requires Kafka with libsasl2)") + else() + message(FATAL_ERROR "AWS MSK IAM authentication: DISABLED (requires Kafka with libsasl2)") endif() else() if(FLB_AWS AND NOT FLB_KAFKA) - message(STATUS "AWS MSK IAM authentication: DISABLED (requires FLB_KAFKA=ON)") + message(FATAL_ERROR "AWS MSK IAM authentication: DISABLED (requires FLB_KAFKA=ON)") endif() endif() @@ -1589,3 +1594,5 @@ message(STATUS "Libraries install to: ${CMAKE_INSTALL_LIBDIR}") file(GLOB_RECURSE INSTALL_FILES "${CMAKE_INSTALL_PREFIX}/*") message(STATUS "Files to package: ${INSTALL_FILES}") +# Verify all required options are set +validate_required_options(FLB_HAVE_LIBYAML) diff --git a/source/cmake/enterprise.cmake b/source/cmake/enterprise.cmake index b2b99c25..a7e97aac 100644 --- a/source/cmake/enterprise.cmake +++ b/source/cmake/enterprise.cmake @@ -5,3 +5,16 @@ FLB_DEFINITION(FLB_ENTERPRISE) if(CMAKE_INSTALL_PREFIX MATCHES "/opt/td-agent-bit") set(FLB_TD ON) endif() + +# Ensure we have specific options enabled (they may get disabled implicitly due to missing dependencies) +function(validate_required_options) + set(REQUIRED_OPTIONS ${ARGV}) + + foreach(OPT ${REQUIRED_OPTIONS}) + if(NOT ${OPT}) + message(FATAL_ERROR "ERROR: ${OPT} is required but disabled.") + endif() + endforeach() + + message(STATUS "All required options validated successfully") +endfunction() diff --git a/source/cmake/kafka.cmake b/source/cmake/kafka.cmake index 09e98dd9..a4653168 100644 --- a/source/cmake/kafka.cmake +++ b/source/cmake/kafka.cmake @@ -14,17 +14,20 @@ if(PkgConfig_FOUND) if(SASL_FOUND) message(STATUS "Found libsasl2: ${SASL_VERSION}") set(FLB_SASL_ENABLED ON) - else() - message(WARNING "libsasl2 not found - SASL authentication will be disabled") endif() else() message(WARNING "pkg-config not available - trying fallback SASL detection") - # Fallback detection +endif() + +# Fallback when pkgconfig is not available or not working properly (centos 6) +if(NOT FLB_SASL_ENABLED) find_library(SASL2_LIB NAMES sasl2) find_path(SASL2_INCLUDE NAMES sasl/sasl.h) if(SASL2_LIB AND SASL2_INCLUDE) set(FLB_SASL_ENABLED ON) message(STATUS "Found libsasl2 via fallback: ${SASL2_LIB}") + else() + message(WARNING "libsasl2 not found - SASL authentication will be disabled") endif() endif() diff --git a/source/cpack/scripts/sysv-init-install.sh b/source/cpack/scripts/sysv-init-install.sh index 5ff7f917..4088ac2f 100755 --- a/source/cpack/scripts/sysv-init-install.sh +++ b/source/cpack/scripts/sysv-init-install.sh @@ -6,6 +6,7 @@ elif [ -f /etc/init.d/fluent-bit ]; then NAME=fluent-bit else echo "ERROR: No init.d script found" + ls -l /etc/init.d/ exit 1 fi chmod a+x /etc/init.d/"$NAME" diff --git a/source/lib/librdkafka-2.10.1/src/rdhttp.c b/source/lib/librdkafka-2.10.1/src/rdhttp.c index e5a9ad27..98c308fc 100644 --- a/source/lib/librdkafka-2.10.1/src/rdhttp.c +++ b/source/lib/librdkafka-2.10.1/src/rdhttp.c @@ -139,7 +139,8 @@ rd_http_error_t *rd_http_req_init(rd_http_req_t *hreq, const char *url) { hreq->hreq_buf = rd_buf_new(1, 1024); curl_easy_setopt(hreq->hreq_curl, CURLOPT_URL, url); -#if CURL_AT_LEAST_VERSION(7, 85, 0) +//#if CURL_AT_LEAST_VERSION(7, 85, 0) +#if LIBCURL_VERSION_NUM >= 0x075500 curl_easy_setopt(hreq->hreq_curl, CURLOPT_PROTOCOLS_STR, "http,https"); #else /* As of 06/10/2025 Debian 10 and CentOS Stream 9 ship with diff --git a/source/packaging/build.sh b/source/packaging/build.sh index 995721b9..80835f55 100755 --- a/source/packaging/build.sh +++ b/source/packaging/build.sh @@ -1,13 +1,22 @@ #!/bin/bash # Build a specific Linux target using the local source code via a container image -set -eux - -# Never rely on PWD so we can invoke from anywhere -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +set -euo pipefail + +# This does not work with a symlink to this script +# SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# See https://stackoverflow.com/a/246128/24637657 +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) + SOURCE=$(readlink "$SOURCE") + # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE +done +SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) # Allow us to specify in the caller or pass variables FLB_DISTRO=${FLB_DISTRO:-} -FLB_OUT_DIR=${FLB_OUT_DIR:-} +FLB_OUT_DIR=${FLB_OUT_DIR:-agent} FLB_NIGHTLY_BUILD=${FLB_NIGHTLY_BUILD:-} DOCKER=${FLB_DOCKER_CLI:-docker} CACHE_ID=${CACHE_ID:-main} diff --git a/source/packaging/distros/almalinux/Dockerfile b/source/packaging/distros/almalinux/Dockerfile index a2b233d6..0856607a 100644 --- a/source/packaging/distros/almalinux/Dockerfile +++ b/source/packaging/distros/almalinux/Dockerfile @@ -23,7 +23,7 @@ ARG CACHE_ID RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-8-${CACHE_ID} yum -y update && \ yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into separate layer to allow us to cache above RUN yum clean all @@ -42,7 +42,7 @@ ARG CACHE_ID RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-8-arm64-${CACHE_ID} yum -y update && \ yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into separate layer to allow us to cache above RUN yum clean all @@ -62,7 +62,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-9-${CAC yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into separate layer to allow us to cache above RUN yum clean all @@ -81,7 +81,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-9-arm64 yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into separate layer to allow us to cache above RUN yum clean all @@ -101,7 +101,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-10-${CA yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into separate layer to allow us to cache above RUN yum clean all @@ -120,7 +120,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=almalinux-10-arm6 yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into separate layer to allow us to cache above RUN yum clean all diff --git a/source/packaging/distros/centos/6/Dockerfile b/source/packaging/distros/centos/6/Dockerfile index e278fc4c..3b61c962 100644 --- a/source/packaging/distros/centos/6/Dockerfile +++ b/source/packaging/distros/centos/6/Dockerfile @@ -27,7 +27,7 @@ ARG CACHE_ID # hadolint ignore=DL3032, DL3033 RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-6-${CACHE_ID} yum -y update && \ yum install -y rpm-build curl ca-certificates gcc gcc-c++ make wget tar \ - openssl openss-libs openssl-devel cyrus-sasl-devel \ + openssl openss-libs openssl-devel cyrus-sasl-lib cyrus-sasl-devel \ libyaml-devel \ perl-core libtemplate-perl zlib-devel \ libicu-devel scl-utils-build devtoolset-9 rh-python35 && \ diff --git a/source/packaging/distros/centos/7/Dockerfile b/source/packaging/distros/centos/7/Dockerfile new file mode 100644 index 00000000..bb42af53 --- /dev/null +++ b/source/packaging/distros/centos/7/Dockerfile @@ -0,0 +1,123 @@ +ARG BASE_BUILDER=centos-7-base +ARG CMAKE_VERSION=3.31.6 +# Used to differentiate in CI from main/PR builds +ARG CACHE_ID=main +# Multiarch support +FROM multiarch/qemu-user-static:x86_64-aarch64 AS multiarch-aarch64 + +# centos/7 base image +FROM centos:7 AS centos-7-base + +ENV CMAKE_HOME="/opt/cmake" +ARG CMAKE_VERSION +ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download" + +COPY packaging/distros/centos/7/*.repo /etc/yum.repos.d/ + +# Enable vault repos now main are EOL +RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ + sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo + +ARG CACHE_ID +# hadolint ignore=DL3033,DL3032 +RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-7-${CACHE_ID} \ + yum -y update && \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \ + libyaml-devel libgit2-devel \ + tar gzip && \ + yum install -y epel-release && \ + yum install -y cmake3 && \ + yum install -y scl-utils-build devtoolset-9 && \ + yum install -y perl-IPC-Cmd perl-Data-Dumper +# Split into separate layer to allow us to cache above +RUN yum clean all && scl enable devtoolset-9 bash && echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + +# hadolint ignore=DL4006 +RUN mkdir -p "${CMAKE_HOME}" && \ + cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \ + echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \ + curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1 + +ENV PATH="${CMAKE_HOME}/bin:${PATH}" + +# centos/7.arm64v8 base image +FROM arm64v8/centos:7 AS centos-7.arm64v8-base + +COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static + +ENV CMAKE_HOME="/opt/cmake" +ARG CMAKE_VERSION +ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download" + +# Install DevToolset 9 as it is required for RocksDB to use C++17 +COPY packaging/distros/centos/7/*.repo /etc/yum.repos.d/ + +# Enable vault repos now main are EOL +RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ + sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo + +ARG CACHE_ID +# hadolint ignore=DL3033,DL3032 +RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-7-arm64-${CACHE_ID} \ + yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \ + wget unzip systemd-devel wget flex bison \ + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \ + libyaml-devel libgit2-devel \ + tar gzip && \ + yum install -y epel-release && \ + yum install -y cmake3 && \ + yum install -y scl-utils-build devtoolset-9 && \ + yum install -y perl-IPC-Cmd perl-Data-Dumper +# Split into separate layer to allow us to cache above +RUN yum clean all && scl enable devtoolset-9 bash && echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + +# hadolint ignore=DL4006 +RUN mkdir -p "${CMAKE_HOME}" && \ + cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \ + echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \ + curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1 + +ENV PATH="${CMAKE_HOME}/bin:${PATH}" + +# Need larger page size +ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" +ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS + +# Common build for all distributions now +# hadolint ignore=DL3006 +FROM $BASE_BUILDER AS builder + +# We need a more recent OpenSSL as well so build from source +ADD https://github.com/openssl/openssl/releases/download/openssl-3.5.2/openssl-3.5.2.tar.gz /openssl/ +RUN tar -xzvf /openssl/openssl-3.5.2.tar.gz -C /openssl +WORKDIR /openssl/openssl-3.5.2 +RUN ./Configure && make -j "$(getconf _NPROCESSORS_ONLN)" && make install + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +# Docker context must be the base of the repo +WORKDIR /tmp/fluent-bit/ +COPY . ./ + +WORKDIR /tmp/fluent-bit/build/ +# CMake configuration variables +ARG CMAKE_INSTALL_PREFIX=/opt/fluentdo-agent/ +ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ + +RUN PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH \ + cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ + -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \ + -DOPENSSL_ROOT_DIR=/openssl/openssl-3.5.2 \ + -DOPENSSL_USE_STATIC_LIBS=On \ + ../ + +VOLUME [ "/output" ] +# Ensure we run a login shell to trigger the devtoolset enabling +CMD [ "/bin/bash", "-l", "-c", "scl enable devtoolset-9 bash && make --no-print-directory -j 4 && cpack -G RPM && cp *.rpm /output/" ] diff --git a/source/packaging/distros/centos/7/scl.repo b/source/packaging/distros/centos/7/scl.repo new file mode 100644 index 00000000..5e39b4ea --- /dev/null +++ b/source/packaging/distros/centos/7/scl.repo @@ -0,0 +1,12 @@ +# Used for devtoolset installation +[centos-sclo-rh] +name=CentOS-7 - SCLo rh +baseurl=http://vault.centos.org/centos/7/sclo/$basearch/rh/ +gpgcheck=0 +enabled=1 + +[centos-sclo-sclo] +name=CentOS-7 - SCLo sclo +baseurl=http://vault.centos.org/centos/7/sclo/$basearch/sclo/ +gpgcheck=0 +enabled=1 diff --git a/source/packaging/distros/centos/Dockerfile b/source/packaging/distros/centos/Dockerfile index cd5c9a5e..de7e141f 100644 --- a/source/packaging/distros/centos/Dockerfile +++ b/source/packaging/distros/centos/Dockerfile @@ -12,117 +12,6 @@ ARG CACHE_ID=main # Multiarch support FROM multiarch/qemu-user-static:x86_64-aarch64 AS multiarch-aarch64 -# centos/7 base image -FROM centos:7 AS centos-7-base - -ENV CMAKE_HOME="/opt/cmake" -ARG CMAKE_VERSION="3.31.6" -ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download" - -# Install DevToolset 9 as it is required for RocksDB to use C++17 -RUN cat > /etc/yum.repos.d/scl.repo <<'EOF' -[centos-sclo-rh] -name=CentOS-7 - SCLo rh -baseurl=http://vault.centos.org/centos/7/sclo/$basearch/rh/ -gpgcheck=0 -enabled=1 - -[centos-sclo-sclo] -name=CentOS-7 - SCLo sclo -baseurl=http://vault.centos.org/centos/7/sclo/$basearch/sclo/ -gpgcheck=0 -enabled=1 -EOF - -# Enable vault repos now main are EOL -RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo - -ARG CACHE_ID -# hadolint ignore=DL3033,DL3032 -RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-7-${CACHE_ID} \ - yum -y update && \ - yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \ - wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \ - libyaml-devel libgit2-devel \ - tar gzip && \ - yum install -y epel-release && \ - yum install -y cmake3 && \ - yum install -y scl-utils-build devtoolset-9 -# Split into separate layer to allow us to cache above -RUN yum clean all && scl enable devtoolset-9 bash && echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc -SHELL ["/bin/bash", "--login", "-c"] - -# hadolint ignore=DL4006 -RUN mkdir -p "${CMAKE_HOME}" && \ - cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \ - echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \ - curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1 - -ENV PATH="${CMAKE_HOME}/bin:${PATH}" - -ARG FLB_KAFKA=Off -ENV FLB_KAFKA=${FLB_KAFKA} - -# centos/7.arm64v8 base image -FROM arm64v8/centos:7 AS centos-7.arm64v8-base - -COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static - -ENV CMAKE_HOME="/opt/cmake" -ARG CMAKE_VERSION="3.31.6" -ARG CMAKE_URL="https://github.com/Kitware/CMake/releases/download" - -# Install DevToolset 9 as it is required for RocksDB to use C++17 -RUN cat > /etc/yum.repos.d/scl.repo <<'EOF' -[centos-sclo-rh] -name=CentOS-7 - SCLo rh -baseurl=http://vault.centos.org/centos/7/sclo/$basearch/rh/ -gpgcheck=0 -enabled=1 - -[centos-sclo-sclo] -name=CentOS-7 - SCLo sclo -baseurl=http://vault.centos.org/centos/7/sclo/$basearch/sclo/ -gpgcheck=0 -enabled=1 -EOF - -# Enable vault repos now main are EOL -RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo - -ARG CACHE_ID -# hadolint ignore=DL3033,DL3032 -RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=centos-7-arm64-${CACHE_ID} \ - yum install -y rpm-build curl ca-certificates gcc gcc-c++ make bash \ - wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libcurl-devel pkgconf-pkg-config \ - libyaml-devel libgit2-devel \ - tar gzip && \ - yum install -y epel-release && \ - yum install -y cmake3 && \ - yum install -y scl-utils-build devtoolset-9 -# Split into separate layer to allow us to cache above -RUN yum clean all && scl enable devtoolset-9 bash && echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc -SHELL ["/bin/bash", "--login", "-c"] - -# hadolint ignore=DL4006 -RUN mkdir -p "${CMAKE_HOME}" && \ - cmake_download_url="${CMAKE_URL}/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" && \ - echo "Downloading CMake ${CMAKE_VERSION}: ${cmake_download_url} -> ${CMAKE_HOME}" && \ - curl -jksSL "${cmake_download_url}" | tar -xzf - -C "${CMAKE_HOME}" --strip-components 1 - -ENV PATH="${CMAKE_HOME}/bin:${PATH}" - -# Need larger page size -ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" -ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS - -ARG FLB_KAFKA=Off -ENV FLB_KAFKA=${FLB_KAFKA} - # centos/8 base image FROM centos:8 AS centos-8-base @@ -157,9 +46,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \ ENV PATH="${CMAKE_HOME}/bin:${PATH}" -ARG FLB_KAFKA=On -ENV FLB_KAFKA=${FLB_KAFKA} - # centos/8.arm64v8 base image FROM arm64v8/centos:8 AS centos-8.arm64v8-base @@ -200,9 +86,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}" ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS -ARG FLB_KAFKA=On -ENV FLB_KAFKA=${FLB_KAFKA} - FROM quay.io/centos/centos:stream9 AS centos-9-base ENV CMAKE_HOME="/opt/cmake" @@ -232,9 +115,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \ ENV PATH="${CMAKE_HOME}/bin:${PATH}" -ARG FLB_KAFKA=On -ENV FLB_KAFKA=${FLB_KAFKA} - # hadolint ignore=DL3029 FROM --platform=arm64 quay.io/centos/centos:stream9 AS centos-9.arm64v8-base @@ -270,9 +150,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}" ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS -ARG FLB_KAFKA=On -ENV FLB_KAFKA=${FLB_KAFKA} - FROM quay.io/centos/centos:stream10 AS centos-10-base ENV CMAKE_HOME="/opt/cmake" @@ -301,9 +178,6 @@ RUN mkdir -p "${CMAKE_HOME}" && \ ENV PATH="${CMAKE_HOME}/bin:${PATH}" -ARG FLB_KAFKA=On -ENV FLB_KAFKA=${FLB_KAFKA} - # hadolint ignore=DL3029 FROM --platform=arm64 quay.io/centos/centos:stream10 AS centos-10.arm64v8-base @@ -339,9 +213,6 @@ ENV PATH="${CMAKE_HOME}/bin:${PATH}" ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS -ARG FLB_KAFKA=On -ENV FLB_KAFKA=${FLB_KAFKA} - # Common build for all distributions now # hadolint ignore=DL3006 FROM $BASE_BUILDER AS builder @@ -362,7 +233,6 @@ RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \ - -DFLB_KAFKA="$FLB_KAFKA" \ ../ VOLUME [ "/output" ] diff --git a/source/packaging/distros/rockylinux/Dockerfile b/source/packaging/distros/rockylinux/Dockerfile index d0bdf6d7..8c6fc044 100644 --- a/source/packaging/distros/rockylinux/Dockerfile +++ b/source/packaging/distros/rockylinux/Dockerfile @@ -24,7 +24,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-8-${CA yum install -y epel-release && \ yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into a separate layer to cache above RUN yum clean all @@ -43,7 +43,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-8-arm6 yum install -y epel-release && \ yum install -y rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into a separate layer to cache above RUN yum clean all @@ -62,7 +62,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-9-${CA yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into a separate layer to cache above RUN yum clean all @@ -80,7 +80,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-9-arm6 yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into a separate layer to cache above RUN yum clean all @@ -99,7 +99,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-10-${C yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into a separate layer to cache above RUN yum clean all @@ -117,7 +117,7 @@ RUN --mount=type=cache,target=/var/cache/yum,sharing=locked,id=rockylinux-10-arm yum install -y epel-release && \ yum install -y --allowerasing rpm-build curl ca-certificates gcc gcc-c++ cmake make bash \ wget unzip systemd-devel wget flex bison \ - cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config + cyrus-sasl-lib cyrus-sasl-devel openssl openssl-libs openssl-devel libyaml-devel libgit2-devel pkgconf-pkg-config # Split into a separate layer to cache above RUN yum clean all diff --git a/source/packaging/distros/suse/Dockerfile b/source/packaging/distros/suse/Dockerfile index 37175273..a169bfa0 100644 --- a/source/packaging/distros/suse/Dockerfile +++ b/source/packaging/distros/suse/Dockerfile @@ -17,51 +17,25 @@ ARG CACHE_ID=main # Multiarch support FROM multiarch/qemu-user-static:x86_64-aarch64 as multiarch-aarch64 -# suse/12 base image -# Based on Google ops agent to use OpenSUSE Leap 42.3 to emulate SLES 12: -# https://en.opensuse.org/openSUSE:Build_Service_cross_distribution_howto#Detect_a_distribution_flavor_for_special_code -FROM opensuse/archive:42.3 AS suse-12-base -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ARG CACHE_ID -# zypper/libcurl has a use-after-free bug that causes segfaults for particular download sequences. -# If this bug happens to trigger in the future, adding a "zypper -n download" of a subset of the packages can avoid the segfault. -# hadolint ignore=DL3036,DL3037 -RUN --mount=type=cache,target=/var/cache/zypp,sharing=locked,id=suse-12-${CACHE_ID} \ - set -x; \ - # The repo signatures are no longer valid - # TODO: use checksum verification on updates - zypper --no-gpg-check refresh 'OSS Update' && \ - zypper --no-gpg-check refresh 'NON OSS Update' && \ - zypper -n install curl tar git systemd autoconf automake flex libtool libcurl-devel libopenssl-devel libyajl-devel \ - gcc gcc-c++ glibc-devel zlib-devel rpm-build expect systemd-devel systemd-rpm-macros libgit2-devel && \ - # Remove expired root certificate. - mv /var/lib/ca-certificates/pem/DST_Root_CA_X3.pem /etc/pki/trust/blacklist/ && \ - update-ca-certificates && \ - # Allow fluent-bit to find systemd - ln -fs /usr/lib/systemd /lib/systemd - -# Unfortunately this currently fails with missing dependencies and the requirement for C++17. -ARG FLB_PROCESSOR_DEDUP=Off -ENV FLB_PROCESSOR_DEDUP=$FLB_PROCESSOR_DEDUP - -ARG FLB_JEMALLOC_OPTIONS -ENV FLB_JEMALLOC_OPTIONS=${FLB_JEMALLOC_OPTIONS} - # suse/15 base image -FROM opensuse/leap:15.3 AS suse-15-base +FROM opensuse/leap:15.6 AS suse-15-base ARG CACHE_ID # hadolint ignore=DL3036,DL3037 RUN --mount=type=cache,target=/var/cache/zypp,sharing=locked,id=suse-15-${CACHE_ID} \ - set -x; \ - zypper -n install curl tar git systemd autoconf automake flex libtool libcurl-devel libopenssl-devel libyajl-devel \ - gcc gcc-c++ zlib-devel rpm-build expect systemd-devel systemd-rpm-macros libgit2-devel && \ - # Allow fluent-bit to find systemd - ln -fs /usr/lib/systemd /lib/systemd - -ARG FLB_PROCESSOR_DEDUP=On -ENV FLB_PROCESSOR_DEDUP=$FLB_PROCESSOR_DEDUP + zypper up -y && \ + zypper install -y --no-recommends \ + rpm-build \ + curl ca-certificates wget unzip flex bison \ + gcc gcc-c++ \ + cmake-full \ + make \ + bash \ + systemd-devel \ + libgit2-devel \ + cyrus-sasl cyrus-sasl-devel \ + libopenssl3 libopenssl-devel \ + libyaml-devel # suse/15.arm64v8 base image # hadolint ignore=DL3029 @@ -70,18 +44,23 @@ COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64 ARG CACHE_ID # hadolint ignore=DL3036,DL3037 -RUN --mount=type=cache,target=/var/cache/zypp,sharing=locked,id=suse-15-arm64-${CACHE_ID} \ - set -x; \ - zypper -n install curl tar git systemd autoconf automake flex libtool libcurl-devel libopenssl-devel libyajl-devel \ - gcc gcc-c++ zlib-devel rpm-build expect systemd-devel systemd-rpm-macros libgit2-devel && \ - # Allow fluent-bit to find systemd - ln -fs /usr/lib/systemd /lib/systemd - -ARG FLB_PROCESSOR_DEDUP=On -ENV FLB_PROCESSOR_DEDUP=$FLB_PROCESSOR_DEDUP - -ARG FLB_JEMALLOC_OPTIONS -ENV FLB_JEMALLOC_OPTIONS=${FLB_JEMALLOC_OPTIONS} +RUN --mount=type=cache,target=/var/cache/zypp,sharing=locked,id=suse-15-${CACHE_ID} \ + zypper up -y && \ + zypper install -y --no-recommends \ + rpm-build \ + curl ca-certificates wget unzip flex bison \ + gcc gcc-c++ \ + cmake-full \ + make \ + bash \ + systemd-devel \ + libgit2-devel \ + cyrus-sasl cyrus-sasl-devel \ + libopenssl3 libopenssl-devel \ + libyaml-devel + +ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" +ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS # Common build for all distributions now # hadolint ignore=DL3006 @@ -95,27 +74,6 @@ RUN CMAKE_ARCH=$(uname -m); \ ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake && \ ln -sf /opt/cmake/bin/cpack /usr/local/bin/cpack -# Bison needs to be built from source unfortunately as at least one target does not provide it so we want to be common for all -ARG BISON_VER=3.8.2 -ARG BISON_URL=https://www.mirrorservice.org/sites/ftp.gnu.org/gnu/bison/ -ARG FLEX_VER=2.6.4 -ARG FLEX_URL=https://github.com/westes/flex/files/981163 -ADD ${BISON_URL}/bison-${BISON_VER}.tar.gz /bison/ -ADD ${FLEX_URL}/flex-${FLEX_VER}.tar.gz /flex/ -RUN tar -xzvf /bison/bison-${BISON_VER}.tar.gz -C /bison/ && tar -xzvf /flex/flex-${FLEX_VER}.tar.gz -C /flex/ - -# Flex needs Bison so do first -WORKDIR /bison/bison-${BISON_VER}/ -RUN ./configure && make -j "$(getconf _NPROCESSORS_ONLN)" && make install -WORKDIR /flex/flex-${FLEX_VER}/ -RUN ./configure && make -j "$(getconf _NPROCESSORS_ONLN)" && make install - -# We need a more recent OpenSSL as well so build from source -ADD https://github.com/openssl/openssl/releases/download/openssl-3.5.2/openssl-3.5.2.tar.gz /openssl/ -RUN tar -xzvf /openssl/openssl-3.5.2.tar.gz -C /openssl -WORKDIR /openssl/openssl-3.5.2 -RUN ./Configure && make -j "$(getconf _NPROCESSORS_ONLN)" && make install - ARG FLB_NIGHTLY_BUILD ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD @@ -131,21 +89,11 @@ WORKDIR /source/fluent-bit/build/ ARG CMAKE_INSTALL_PREFIX=/opt/fluentdo-agent/ ARG CMAKE_INSTALL_SYSCONFDIR=/etc/ -# No YAML support -ARG FLB_CONFIG_YAML=Off -# Triggers errors on librdkafka if enabled -ARG WITH_CURL=Off - RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DCMAKE_RULE_MESSAGES:BOOL=OFF -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DCMAKE_INSTALL_SYSCONFDIR="$CMAKE_INSTALL_SYSCONFDIR" \ -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \ - -DFLB_CONFIG_YAML="${FLB_CONFIG_YAML}" \ - -DWITH_CURL="${WITH_CURL}" \ - -DFLB_PROCESSOR_DEDUP="${FLB_PROCESSOR_DEDUP}" \ - -DOPENSSL_ROOT_DIR=/openssl/openssl-3.5.2 \ - -DOPENSSL_USE_STATIC_LIBS=On \ ../ VOLUME [ "/output" ] diff --git a/testing/bats/run-package-functional-tests.sh b/testing/bats/run-package-functional-tests.sh index f4c81446..36c941bf 100755 --- a/testing/bats/run-package-functional-tests.sh +++ b/testing/bats/run-package-functional-tests.sh @@ -6,12 +6,12 @@ set -euo pipefail # See https://stackoverflow.com/a/246128/24637657 SOURCE=${BASH_SOURCE[0]} while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) - SOURCE=$(readlink "$SOURCE") - # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE + SCRIPT_DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + SOURCE=$(readlink "$SOURCE") + # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE=$SCRIPT_DIR/$SOURCE done -SCRIPT_DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd ) +SCRIPT_DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker} export BASE_IMAGE=${BASE_IMAGE:-dokken/almalinux-8} @@ -26,14 +26,25 @@ export FLUENTDO_AGENT_VERSION=${FLUENTDO_AGENT_VERSION:-25.11.2} export DOWNLOAD_DIR=${DOWNLOAD_DIR:-$PWD/downloads} mkdir -p "$DOWNLOAD_DIR" +# We have to break into two separate steps as first it will look for *.rpm then *.deb +# so if we have .deb files it will fail to find any *.rpm and attempt to check for the +# existence of the glob +FOUND_FILES=false for f in "$DOWNLOAD_DIR"/*.{rpm,deb}; do - - ## Check if the glob gets expanded to existing files. - ## If not, f here will be exactly the pattern above - ## and the exists test will evaluate to false. - if [ -e "$f" ]; then + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then echo "INFO: Found package in $DOWNLOAD_DIR" - elif [[ -n "${CI:-}" ]]; then + FOUND_FILES=true + break + else + echo "DEBUG: skipping $f" + fi +done + +if [[ $FOUND_FILES == false ]]; then + if [[ -n "${CI:-}" ]]; then # For CI we want to use local packages so ensure they are present echo "ERROR: Unable to find package in $DOWNLOAD_DIR" exit 1 @@ -45,19 +56,15 @@ for f in "$DOWNLOAD_DIR"/*.{rpm,deb}; do # almalinux/8 becomes DISTRO_ID=almalinux, DISTRO_VERSION=8 # debian/bookworm becomes DISTRO_ID=debian, DISTRO_VERSION=bookworm # ubuntu/24 becomes DISTRO_ID=ubuntu, DISTRO_VERSION=24 - DISTRO_ID=$(echo "$DISTRO" | cut -d'/' -f1) + DISTRO_ID=$(echo "$DISTRO" | cut -d'/' -f1) export DISTRO_ID - DISTRO_VERSION=$(echo "$DISTRO" | cut -d'/' -f2) + DISTRO_VERSION=$(echo "$DISTRO" | cut -d'/' -f2) export DISTRO_VERSION # Use the install script to just download the image "$SCRIPT_DIR"/../../install.sh --debug --download fi - - ## This is all we needed to know, so we can break after the first iteration - break -done - +fi echo "INFO: building test container 'bats/test/$DISTRO'" "${CONTAINER_RUNTIME}" build -t "bats/test/$DISTRO" \