Feat: Add Kafka integration for Parseable server #936 .

.github/workflows/build-push-edge-kafka.yaml

@@ -0,0 +1,45 @@
+name: Build and push edge kafka tag
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths-ignore:
+      - 'docs/**'
+      - 'helm/**'
+      - 'assets/**'
+      - '**.md'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: parseable/parseable
+
+      - name: Build and push
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          file: ./Dockerfile.kafka
+          push: true
+          tags: parseable/parseable:edge-kafka
+          platforms: linux/amd64,linux/arm64

.github/workflows/build.yaml

@@ -1,3 +1,5 @@
+name: Ensure parseable builds on all release targets
+
 on:
   pull_request:
     paths-ignore:
@@ -6,68 +8,141 @@ on:
       - "assets/**"
       - "**.md"
 
-name: Ensure parseable builds on all release targets
 jobs:
-  build-linux:
-    name: Build for ${{matrix.target}}
-    runs-on: ubuntu-latest
+  # Default build without Kafka
+  build-default:
+    name: Build Default ${{matrix.target}}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        target:
-          - aarch64-unknown-linux-gnu # linux(arm)
-          - x86_64-unknown-linux-gnu # linux(64 bit)
+        include:
+          # Linux builds
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+          # macOS builds  
+          - os: macos-latest
+            target: x86_64-apple-darwin
+          - os: macos-latest
+            target: aarch64-apple-darwin
+          # Windows build
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
 
-      - uses: actions-rs/toolchain@v1
+      - name: Cache dependencies
+        uses: actions/cache@v4
         with:
-          toolchain: stable
-          profile: minimal # minimal component installation (ie, no documentation)
-          target: ${{ matrix.target }}
-          override: true
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ matrix.target }}-default-${{ hashFiles('**/Cargo.lock') }}
 
-      - uses: actions-rs/cargo@v1
+      - name: Build
+        uses: actions-rs/cargo@v1
         with:
-          use-cross: true
+          use-cross: ${{ runner.os == 'Linux' }}
           command: build
-          args: --target ${{matrix.target}}
+          args: --target ${{ matrix.target }} --release
 
-  build-windows:
-    name: Build for windows
-    runs-on: windows-latest
+  # Kafka build for supported platforms
+  build-kafka:
+    name: Build Kafka ${{matrix.target}}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Linux builds
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            target: aarch64-apple-darwin
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal # minimal component installation (ie, no documentation)
-          default: true
-          override: true
+      # Linux-specific dependencies
+      - name: Install Linux dependencies
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential \
+            pkg-config \
+            cmake \
+            clang \
+            zlib1g-dev \
+            libzstd-dev \
+            liblz4-dev \
+            libssl-dev \
+            libsasl2-dev \
+            python3 \
+            gcc-aarch64-linux-gnu \
+            g++-aarch64-linux-gnu
+
+          # Install cross-compilation specific packages
+          if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then
+            sudo apt-get install -y \
+              gcc-aarch64-linux-gnu \
+              g++-aarch64-linux-gnu \
+              libc6-dev-arm64-cross \
+              libsasl2-dev:arm64 \
+              libssl-dev:arm64 \
+              pkg-config-aarch64-linux-gnu
+          fi
 
-      - name: Build on windows
-        run: cargo build --target x86_64-pc-windows-msvc
 
-  build-macos:
-    name: Build for ${{matrix.target}}
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        target:
-          - aarch64-apple-darwin # macos(arm)
-          - x86_64-apple-darwin # macos(intel 64 bit)
+      # macOS-specific dependencies
+      - name: Install macOS dependencies
+        if: runner.os == 'macOS'
+        run: |
+          brew install \
+            cmake \
+            llvm \
+            pkg-config \
+            zstd \
+            lz4 \
+            [email protected] \
+            cyrus-sasl \
+            python3
 
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: stable
-          profile: minimal
-          target: ${{ matrix.target }}
-          override: true
+          targets: ${{ matrix.target }}
 
-      - name: Build on ${{ matrix.target }}
-        run: cargo build --target ${{ matrix.target }}
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ matrix.target }}-kafka-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Build with Kafka
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: ${{ runner.os == 'Linux' }}
+          command: build
+          args: --target ${{ matrix.target }} --features kafka --release
+        env:
+          LIBRDKAFKA_SSL_VENDORED: 1
+          PKG_CONFIG_ALLOW_CROSS: "1"
+          PKG_CONFIG_PATH: "/usr/lib/aarch64-linux-gnu/pkgconfig"
+          SASL2_DIR: "/usr/lib/aarch64-linux-gnu"
+          OPENSSL_DIR: "/usr/lib/aarch64-linux-gnu"
+          OPENSSL_ROOT_DIR: "/usr/lib/aarch64-linux-gnu"
+          OPENSSL_STATIC: "1"
+          SASL2_STATIC: "0"

.github/workflows/coverage.yaml

@@ -11,7 +11,7 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
@@ -24,13 +24,23 @@ jobs:
         with:
           tool: cargo-hack, cargo-llvm-cov, nextest
 
+      - name: Install System Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            libsasl2-dev \
+            libssl-dev \
+            pkg-config \
+            build-essential
+        if: runner.os == 'Linux'
+
       - name: Check with clippy
         run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings
 
       - name: Tests
         run: cargo hack --each-feature llvm-cov --no-report nextest
 
-      - name: Genrate coverage report
+      - name: Generate coverage report
         run: cargo llvm-cov report --lcov --output-path coverage.lcov
 
       - name: Upload Coverage Report

.github/workflows/integration-test.yaml

@@ -14,21 +14,45 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Start compose
         run: docker compose -f docker-compose-test.yaml up --build --exit-code-from quest
       - name: Stop compose
         if: always()
-        run: docker compose -f docker-compose-test.yaml down
+        run: docker compose -f docker-compose-test.yaml down -v
 
   docker-compose-distributed-test:
     name: Quest Smoke and Load Tests for Distributed deployments
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Start compose
         run: docker compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest
       - name: Stop compose
         if: always()
-        run: docker compose -f docker-compose-distributed-test.yaml down
+        run: docker compose -f docker-compose-distributed-test.yaml down -v
+
+  docker-compose-test-with-kafka:
+    name: Quest Smoke and Load Tests for Standalone deployments with Kafka
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Start compose
+        run: docker compose -f docker-compose-test-with-kafka.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker compose -f docker-compose-test-with-kafka.yaml down -v
+
+  docker-compose-distributed-test-with-kafka:
+    name: Quest Smoke and Load Tests for Distributed deployments with Kafka
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Start compose
+        run: docker compose -f docker-compose-distributed-test-with-kafka.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker compose -f docker-compose-distributed-test-with-kafka.yaml down -v

.gitignore

@@ -14,3 +14,4 @@ parseable
 parseable_*
 parseable-env-secret
 cache
+.idea