docker: add arrow/parquet support

Signed-off-by: Arbin <[email protected]>

Author: kalavt

dockerfiles/Dockerfile

@@ -21,6 +21,10 @@ FROM multiarch/qemu-user-static:x86_64-aarch64 AS qemu-arm64
 
 FROM debian:trixie-slim AS builder-base
 
+# EXTRA_CMAKE_FLAGS can be used to enable optional features like Parquet
+ARG EXTRA_CMAKE_FLAGS
+ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS}
+
 COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/
 COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/
 
@@ -34,7 +38,7 @@ RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# hadolint ignore=DL3008
+# hadolint ignore=DL3008,SC2015
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     build-essential \
@@ -54,6 +58,13 @@ RUN apt-get update && \
     flex \
     bison \
     libyaml-dev \
+    && if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \
+        curl -fsSL https://packages.apache.org/artifactory/arrow/debian/apache-arrow-apt-source-latest-trixie.deb -o apache-arrow-apt-source.deb && \
+        apt-get install -y -V ./apache-arrow-apt-source.deb && \
+        apt-get update && \
+        apt-get install -y -V libarrow-dev libparquet-dev && \
+        rm -f apache-arrow-apt-source.deb; \
+    fi \
     && apt-get satisfy -y cmake "cmake (<< 4.0)" \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
@@ -122,12 +133,23 @@ FROM debian:trixie-slim AS deb-extractor
 COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/
 COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/
 
+ARG EXTRA_CMAKE_FLAGS
+ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS}
+
 # We download all debs locally then extract them into a directory we can use as the root for distroless.
 # We also include some extra handling for the status files that some tooling uses for scanning, etc.
 WORKDIR /tmp
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# hadolint ignore=SC2015,DL3008
 RUN apt-get update && \
-    apt-get download \
+    if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \
+        apt-get install -y --no-install-recommends curl ca-certificates && \
+        curl -fsSL https://packages.apache.org/artifactory/arrow/debian/apache-arrow-apt-source-latest-trixie.deb -o apache-arrow-apt-source.deb && \
+        apt-get install -y -V ./apache-arrow-apt-source.deb && \
+        apt-get update && \
+        rm -f apache-arrow-apt-source.deb; \
+    fi \
+    && apt-get download \
     libssl3t64 \
     libcurl4t64 \
     libnghttp2-14 \
@@ -167,6 +189,19 @@ RUN apt-get update && \
     libyaml-0-2 \
     libcap2 \
     libldap2 \
+    # MAINTAINER: Arrow SONAME format: libarrow{major*100+minor*10} (e.g., 22.0.0 -> libarrow2200)
+    # Only update when upgrading Arrow version or if packages become unavailable in apt source.
+    && if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \
+        apt-get download \
+            libarrow2200 \
+            libparquet2200 \
+            libsnappy1v5 \
+            libabsl20240722 \
+            libbz2-1.0 \
+            libprotobuf32t64 \
+            libthrift-0.19.0t64 \
+            libxml2; \
+    fi \
     && \
     mkdir -p /dpkg/var/lib/dpkg/status.d/ && \
     for deb in *.deb; do \
@@ -231,9 +266,12 @@ LABEL description="Fluent Bit multi-architecture debug container image" \
 
 COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/
 COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/
+
+ARG EXTRA_CMAKE_FLAGS
+ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS}
 ENV DEBIAN_FRONTEND=noninteractive
 
-# hadolint ignore=DL3008
+# hadolint ignore=DL3008,SC2015
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     libssl3t64 \
@@ -263,6 +301,22 @@ RUN apt-get update && \
     htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \
     make tar flex bison \
     libssl-dev libsasl2-dev libsystemd-dev zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \
+    # MAINTAINER: Keep Arrow packages in sync with deb-extractor stage.
+    && if echo "${EXTRA_CMAKE_FLAGS}" | grep -q "FLB_PARQUET_ENCODER=On"; then \
+        curl -fsSL https://packages.apache.org/artifactory/arrow/debian/apache-arrow-apt-source-latest-trixie.deb -o apache-arrow-apt-source.deb && \
+        apt-get install -y -V ./apache-arrow-apt-source.deb && \
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            libarrow2200 \
+            libparquet2200 \
+            libsnappy1v5 \
+            libabsl20240722 \
+            libbz2-1.0 \
+            libprotobuf32t64 \
+            libthrift-0.19.0t64 \
+            libxml2 && \
+        rm -f apache-arrow-apt-source.deb; \
+    fi \
     && apt-get satisfy -y cmake "cmake (<< 4.0)" \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

dockerfiles/Dockerfile.windows

@@ -16,6 +16,9 @@ ARG WINDOWS_VERSION=ltsc2025
 # Builder Image - Windows Server Core
 FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION AS builder-base
 
+# Unified build argument for extra CMake flags (e.g., -DFLB_PARQUET_ENCODER=On)
+ARG EXTRA_CMAKE_FLAGS=""
+
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
 # Clean up any existing temp files and create fresh temp directory
@@ -154,6 +157,16 @@ RUN vcpkg install --recurse libyaml --triplet x64-windows-static; `
     Remove-Item -Path $env:TEMP\* -Recurse -Force -ErrorAction SilentlyContinue; `
     Remove-Item -Path C:\dev\vcpkg\buildtrees -Recurse -Force -ErrorAction SilentlyContinue;
 
+# Install Apache Arrow with Parquet support if enabled via EXTRA_CMAKE_FLAGS
+RUN if ($env:EXTRA_CMAKE_FLAGS -like '*-DFLB_PARQUET_ENCODER=On*') { `
+        Write-Host 'Installing Apache Arrow with Parquet support...'; `
+        vcpkg install --recurse arrow[parquet] --triplet x64-windows-static; `
+        Remove-Item -Path $env:TEMP\* -Recurse -Force -ErrorAction SilentlyContinue; `
+        Remove-Item -Path C:\dev\vcpkg\buildtrees -Recurse -Force -ErrorAction SilentlyContinue; `
+    } else { `
+        Write-Host 'Skipping Apache Arrow installation (FLB_PARQUET_ENCODER not in EXTRA_CMAKE_FLAGS)'; `
+    }
+
 # Final vcpkg cleanup to remove all build artifacts and save space
 RUN Remove-Item -Path C:\dev\vcpkg\downloads -Recurse -Force -ErrorAction SilentlyContinue; `
     Remove-Item -Path C:\dev\vcpkg\buildtrees -Recurse -Force -ErrorAction SilentlyContinue; `
@@ -167,11 +180,20 @@ WORKDIR /src/build
 COPY . /src/
 
 ARG BUILD_PARALLEL=1
+ARG EXTRA_CMAKE_FLAGS
+
+# Optional feature detection: Add cmake paths based on EXTRA_CMAKE_FLAGS
 SHELL ["cmd", "/S", "/C"]
 RUN call "%MSVS_HOME%\VC\Auxiliary\Build\vcvars64.bat" && `
+    set "OPTIONAL_CMAKE_ARGS=" && `
+    (echo %EXTRA_CMAKE_FLAGS% | findstr /C:"-DFLB_PARQUET_ENCODER=On" >nul && ( `
+        set "OPTIONAL_CMAKE_ARGS=%OPTIONAL_CMAKE_ARGS% -DArrow_DIR=C:\dev\vcpkg\packages\arrow_x64-windows-static\share\arrow -DParquet_DIR=C:\dev\vcpkg\packages\arrow_x64-windows-static\share\parquet" `
+    ) || cd .) & `
     cmake -G "NMake Makefiles" `
-      -DOPENSSL_ROOT_DIR='C:\dev\vcpkg\packages\openssl_x64-windows-static' `
-      -DFLB_LIBYAML_DIR='C:\dev\vcpkg\packages\libyaml_x64-windows-static' `
+      -DOPENSSL_ROOT_DIR=C:\dev\vcpkg\packages\openssl_x64-windows-static `
+      -DFLB_LIBYAML_DIR=C:\dev\vcpkg\packages\libyaml_x64-windows-static `
+      %OPTIONAL_CMAKE_ARGS% `
+      %EXTRA_CMAKE_FLAGS% `
       -DFLB_SIMD=On `
       -DCMAKE_BUILD_TYPE=Release `
       -DFLB_SHARED_LIB=Off `

dockerfiles/README.md

@@ -21,24 +21,31 @@ With QEMU set up and buildkit support, you can build all targets in one simple c
 To set up for Ubuntu 20.04 development PC as an example:
 
 1. Add QEMU: https://askubuntu.com/a/1369504
+
 ```
 sudo add-apt-repository ppa:jacob/virtualisation
 sudo apt-get update && sudo apt-get install qemu qemu-user qemu-user-static
 ```
+
 2. Install buildkit: https://docs.docker.com/buildx/working-with-buildx/#install
+
 ```
 wget https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64
 mv buildx-v0.7.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx
 chmod a+x ~/.docker/cli-plugins/docker-buildx
 ```
+
 3. Configure and use: https://stackoverflow.com/a/60667468
+
 ```
 docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 docker buildx rm builder
 docker buildx create --name builder --use
 docker buildx inspect --bootstrap
 ```
+
 4. Build Fluent Bit from the **root of the Git repo (not from this directory)**:
+
 ```
 docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" --target=production -f dockerfiles/Dockerfile .
 ```
@@ -47,10 +54,25 @@ docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x
 
 1. Checkout the branch you want, e.g. 1.8 for 1.8.X containers.
 2. Build Fluent Bit from the **root of the Git repo (not from this directory)**:
+
 ```
 $ docker build -t fluent/fluent-bit --target=production -f dockerfiles/Dockerfile .
 ```
+
+### Optional: Build with Parquet encoder support
+
+To enable Parquet encoding support (requires Apache Arrow/Parquet libraries), use the `EXTRA_CMAKE_FLAGS` build argument:
+
+```
+$ docker build -t fluent/fluent-bit --target=production \
+  --build-arg EXTRA_CMAKE_FLAGS="-DFLB_PARQUET_ENCODER=On" \
+  -f dockerfiles/Dockerfile .
+```
+
+Note: Enabling Parquet support will increase the image size by approximately 49MB due to Apache Arrow dependencies.
+
 3. Test the container.
+
 ```
 $ docker run --rm -it fluent/fluent-bit:latest
 ```
@@ -90,10 +112,11 @@ ______ _                  _    ______ _ _             ___   _____
 ## ghcr.io topology
 
 Containers are "staged" prior to release in the following ways to `ghcr.io`:
-* `ghcr.io/fluent/fluent-bit` - official releases, identical to DockerHub
-* `ghcr.io/fluent/fluent-bit/staging` - all architectures staging images used for testing prior to release
-* `ghcr.io/fluent/fluent-bit/master` - x86_64/AMD64 only images built on each push to master, used for integration tests
-* `ghcr.io/fluent/fluent-bit/pr-X` - x86_64/AMD64 only PR images where `X` is the PR number
+
+- `ghcr.io/fluent/fluent-bit` - official releases, identical to DockerHub
+- `ghcr.io/fluent/fluent-bit/staging` - all architectures staging images used for testing prior to release
+- `ghcr.io/fluent/fluent-bit/master` - x86_64/AMD64 only images built on each push to master, used for integration tests
+- `ghcr.io/fluent/fluent-bit/pr-X` - x86_64/AMD64 only PR images where `X` is the PR number
 
 ## Windows
 
@@ -109,20 +132,34 @@ More information is available at:
 In addition, metadata as defined in OCI image spec annotations, is leveraged in the generated image. This is the reason for the additional `--build-arg` parameters.
 
 ### Minimum set of build-args
+
 ```powershell
 docker build --no-cache `
   --build-arg WINDOWS_VERSION=ltsc2019 `
   -t fluent/fluent-bit:master-windows -f ./dockerfiles/Dockerfile.windows .
 ```
 
+### Optional: Build with Parquet encoder support
+
+To enable Parquet encoding support on Windows, add the `EXTRA_CMAKE_FLAGS` build argument:
+
+```powershell
+docker build --no-cache `
+  --build-arg WINDOWS_VERSION=ltsc2019 `
+  --build-arg EXTRA_CMAKE_FLAGS="-DFLB_PARQUET_ENCODER=On" `
+  -t fluent/fluent-bit:master-windows -f ./dockerfiles/Dockerfile.windows .
+```
+
+Note: Parquet support uses vcpkg to install Apache Arrow with static linking.
+
 ## Contact
 
 Feel free to join us on our Mailing List or IRC:
 
- - Slack: http://slack.fluentd.org / channel #fluent-bit
- - Mailing List: https://groups.google.com/forum/#!forum/fluent-bit
- - IRC: irc.freenode.net #fluent-bit
- - Twitter: http://twitter.com/fluentbit
+- Slack: http://slack.fluentd.org / channel #fluent-bit
+- Mailing List: https://groups.google.com/forum/#!forum/fluent-bit
+- IRC: irc.freenode.net #fluent-bit
+- Twitter: http://twitter.com/fluentbit
 
 ## License