Merge pull request #1439 from stratosphereips/develop

Slips v1.1.9

Author: AlyaGomaa

.github/workflows/install-slips-dependencies.yml

@@ -1,10 +1,7 @@
 name: Install Slips Dependencies
 
 on:
-  # workflow_call make this workflow re-usable
   workflow_call:
-    # these are like variables to make the workflow more clean
-    # we can pass these variable from another workflows if we want
     inputs:
       zeek-repo-url:
         description: 'Zeek repository URL'
@@ -25,6 +22,8 @@ on:
 jobs:
   install-dependencies:
     runs-on: ubuntu-22.04
+    outputs:
+      dependencies_installed: ${{ steps.mark_installed.outputs.installed }}
     steps:
     - uses: actions/checkout@v4
       with:
@@ -62,3 +61,7 @@ jobs:
         curl -fsSL ${{ inputs.zeek-key-url }} | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/security_zeek.gpg
         sudo apt update && sudo apt install -y --no-install-recommends zeek
         sudo ln -s /opt/zeek/bin/zeek /usr/local/bin/bro
+
+    - name: Mark dependencies as installed
+      id: mark_installed
+      run: echo "installed=true" >> $GITHUB_OUTPUT

.github/workflows/publish-slips-images.yml

@@ -3,64 +3,85 @@ name: CI-production-publishing-slips-images
 on:
   push:
     branches:
-    - 'master'
-    - '!develop'
+      - 'master'
+      - '!develop'
 
 jobs:
-  publish_slips_docker_image:
-    # runs the tests in a docker(built by this job) on stop of a GH VM
-    runs-on: ubuntu-20.04
+  setup:
+    runs-on: ubuntu-22.04
     # 2 hours timeout
-    timeout-minutes: 7200
+    timeout-minutes: 120
+    outputs:
+      slips_version: ${{ steps.get_version.outputs.slips_version }}
+      builder_name: ${{ steps.create_builder.outputs.name }}
 
     steps:
-    - name: Get slips version
-      run: |
-        VER=$(curl -s  https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/VERSION)
-        echo "SLIPS_VERSION=$VER" >> $GITHUB_ENV
+      - name: Get slips version
+        id: get_version
+        run: |
+          ver=$(curl -s https://raw.githubusercontent.com/stratosphereips/StratosphereLinuxIPS/develop/VERSION)
+          echo "slips_version=$ver" >> $GITHUB_OUTPUT
 
-    # clone slips and checkout branch
-    # By default it checks out only one commit
-    - uses: actions/checkout@v4
-      with:
-        ref: 'master'
-        # Fetch all history for all tags and branches
-        fetch-depth: ''
-        # submodules are needed for local and global p2p
-        submodules: true
+      # should come before buildx action
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
+      # Set up Docker Buildx with docker-container driver is required
+      # at the moment to be able to use a subdirectory with Git context
+      - name: Create and use Buildx builder
+        id: create_builder
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container  #  allows multi-platform builds
+          #  allows you to enable special permissions for the build environment that are normally considered insecure,
+          # such as:network.host
+          buildkitd-flags: --allow-insecure-entitlement network.host
+          name: slips_multi_arch_builder
+          use: true
 
-    - name: Login to DockerHub
-      uses: docker/login-action@v3
-      with:
-        username: stratosphereips
-        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+  publish_images:
+    needs: setup
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        image_type:
+          - name: slips
+            dockerfile: ./docker/Dockerfile
+          - name: slips_light
+            dockerfile: ./docker/light/Dockerfile
 
-    # Set up Docker Buildx with docker-container driver is required
-    # at the moment to be able to use a subdirectory with Git context
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+    steps:
+      - name: Use existing Buildx builder
+        uses: docker/setup-buildx-action@v3
+        with:
+          name: slips_multi_arch_builder
+          install: true
+          use: true
+
+      # clone slips and checkout branch
+      # By default it checks out only one commit
+      - uses: actions/checkout@v4
+        with:
+          ref: 'master'
+          # Fetch all history for all tags and branches
+          fetch-depth: 0
+          # submodules are needed for local and global p2p
+          submodules: true
 
-    - name: Build and push the main Slips image
-      id: docker_build_slips
-      uses: docker/build-push-action@v6
-      with:
-        allow: network.host
-        context: ./
-        file: ./docker/Dockerfile
-        tags: |
-          stratosphereips/slips:latest
-          stratosphereips/slips:${{ env.SLIPS_VERSION }}
-        push: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: stratosphereips
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
-    - name: Build and push the light Slips image
-      id: docker_build_light_slips
-      uses: docker/build-push-action@v6
-      with:
-        allow: network.host
-        context: ./
-        file: ./docker/light/Dockerfile
-        tags: |
-          stratosphereips/slips_light:latest
-          stratosphereips/slips_light:${{ env.SLIPS_VERSION }}
-        push: true
+      - name: Build and push ${{ matrix.image_type.name }} image
+        uses: docker/build-push-action@v6
+        with:
+          allow: network.host
+          context: ./
+          file: ${{ matrix.image_type.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            stratosphereips/${{ matrix.image_type.name }}:latest
+            stratosphereips/${{ matrix.image_type.name }}:${{ needs.setup.outputs.slips_version }}
+          push: true

.github/workflows/unit-tests.yml

@@ -6,27 +6,24 @@ on:
       - 'master'
       - 'develop'
 
-
 jobs:
   # uses the common workflow that builds slips
   install-dependencies-using-reusable-workflow:
     uses: ./.github/workflows/install-slips-dependencies.yml
 
   unit-tests:
     runs-on: ubuntu-22.04
-    timeout-minutes: 1800
-    # make this job depend on the first job
+    timeout-minutes: 120
     needs: install-dependencies-using-reusable-workflow
 
-    # suppress tensorflow warnings
     env:
       TF_CPP_MIN_LOG_LEVEL: 3
       TF_ENABLE_ONEDNN_OPTS: 0
 
     strategy:
       matrix:
         test_file:
-          - test_inputProc.py
+          - test_input.py
           - test_main.py
           - test_conn.py
           - test_downloaded_file.py
@@ -86,13 +83,19 @@ jobs:
           - test_idea_format.py
           - test_fides_sqlite_db.py
           - test_fides_module.py
+          - test_fides_queues.py
+          - test_fides_bridge.py
 
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.ref }}
-        fetch-depth: ''
+        fetch-depth: 0
 
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.10.12
 
     - name: Restore APT cache
       id: apt-cache
@@ -104,20 +107,29 @@ jobs:
         key: apt-cache
 
     - if: ${{ steps.apt-cache.outputs.cache-hit == 'true' }}
-      name: Echo restored from cache
+      name: Echo restored APT cache
       continue-on-error: true
       run: echo "Restored APT dependencies from cache successfully"
 
-    - name: Install Python dependencies (from cache if possible)
-      run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install -r install/requirements.txt
-
-    - name: Install apt dependencies (from cache if possible)
+    - name: Install apt dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y $(cat install/apt_dependencies.txt)
 
+    - name: Restore Python dependencies
+      id: python-cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('install/requirements.txt') }}
+
+    - if: ${{ steps.python-cache.outputs.cache-hit == 'true' }}
+      name: Echo restored Python cache
+      continue-on-error: true
+      run: echo "Restored Python dependencies from cache successfully"
+
+    - name: Install Python dependencies
+      run: python3 -m pip install -r install/requirements.txt
 
     - name: Start redis server
       run: redis-server --daemonize yes

CHANGELOG.md

@@ -1,3 +1,15 @@
+1.1.9 (April 30, 2025)
+- Add bootstrapping node mode for the global P2P. Thanks to @d-strat
+- Add support for ARM64 architecture in Docker images.
+- Fix issues getting domain registrants.
+- Fix the "Database is locked" SQLite error.
+- Fix the issue of Slips hanging when shutting down.
+- Ignore URLs when found in threat intelligence feeds.
+- Improve handling of Zeek tab-separated log files. Logs from Zeek old versions are now read correctly.
+- Optimize IP Info module.
+- Print flows processed per minute in the stats printed to the CLI.
+- Support reading labeled Zeek logs and using their labels in Slips modules.
+
 1.1.8 (Mar 31st, 2025)
 - Fix SQLite database errors.
 - Fix CPU and RAM profilers.

README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-Slips v1.1.7
+Slips v1.1.9
 </h1>
 
 

VERSION

@@ -1 +1 @@
-1.1.8
+1.1.9

config/iris_config.yaml

@@ -1,12 +1,14 @@
 Identity:
   GenerateNewKey: true
 Server:
-  port: 9010
-  Host: 127.0.0.1
-  DhtServerMode: true
+  Port: 9010
+  Host: 0.0.0.0
+  DhtServerMode: 'true'
 Redis:
   Host: 127.0.0.1
-  Port: 6644
+  Port: 6379
   Tl2NlChannel: iris_internal
 PeerDiscovery:
-  DisableBootstrappingNodes: true
+  DisableBootstrappingNodes: false
+  ListOfMultiAddresses:
+  - /dns/melchior.slips.stratosphere.fel.cvut.cz/udp/6437/quic 12D3KooWJJa9PpMFVP7s3TQs2sedypJXxtMVkphRhgkjGH9EYMfM

config/slips.yaml

@@ -507,6 +507,8 @@ global_p2p:
    # running slips on an interface
    use_global_p2p: False
    iris_conf: config/iris_config.yaml
+   bootstrapping_node: False
+   bootstrapping_modules: ["fidesModule", "irisModule"]
 
 #############################
 local_p2p:

docker/Dockerfile

@@ -56,14 +56,14 @@ RUN apt update && apt install -y --no-install-recommends \
     python3-dev \
     python3-tzlocal \
     python3-pip \
+    nano \
+    tree \
+    tmux \
     && echo 'deb http://download.opensuse.org/repositories/security:/zeek/xUbuntu_22.04/ /' |  tee /etc/apt/sources.list.d/security:zeek.list \
     && curl -fsSL https://download.opensuse.org/repositories/security:zeek/xUbuntu_22.04/Release.key | gpg --dearmor |  tee /etc/apt/trusted.gpg.d/security_zeek.gpg > /dev/null \
-    && curl -fsSL https://packages.redis.io/gpg |  gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg \
-    && echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/redis.list \
     && apt update \
     && apt install -y --no-install-recommends --fix-missing \
     zeek \
-    redis \
     npm \
     && ln -s /opt/zeek/bin/zeek /usr/local/bin/bro \
     && apt clean \
@@ -73,8 +73,17 @@ RUN apt update && apt install -y --no-install-recommends \
     && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  \
     && nvm install 22
 
+# why are we compiling redis instead od just using apt?
+# to support running slips on the rpi (arm64). the rpi uses jemmalloc by default, which expects a different page size
+# than the default on x86_64
+RUN pip3 install --no-cache-dir --upgrade pip \
+    && git clone https://github.com/redis/redis \
+    && cd redis \
+    && make distclean \
+    && make MALLOC=libc
+
+ENV PATH="$PATH:/redis/src"
 
-RUN pip3 install --no-cache-dir --upgrade pip
 
 # Switch to Slips installation dir on login.
 WORKDIR ${SLIPS_DIR}
@@ -90,14 +99,11 @@ RUN  pip install --ignore-installed --no-cache-dir -r install/requirements.txt \
     && git init \
     && git remote add origin https://github.com/stratosphereips/StratosphereLinuxIPS.git \
     && cd modules/kalipso \
-    && npm install
-
-
-# build the pigeon and add pigeon to path
-RUN cd p2p4slips \
-    && go build \
-    && export PATH="{$PATH}:/StratosphereLinuxIPS/p2p4slips/";
+    && npm install \
+    && cd ../../p2p4slips \
+    && go build
 
+ENV PATH="$PATH:/StratosphereLinuxIPS/p2p4slips/"
 
 
 WORKDIR ${SLIPS_DIR}

docker/light/excluded_libs.txt

@@ -1,5 +1,4 @@
 tensorflow
-recommonmark
 Keras
 scipy
 pytest