docker: enforce volume use for /var/lib/crowdsec/data/ (#3757)

Author: blotus

.github/workflows/docker-tests.yml

@@ -6,13 +6,15 @@ on:
       - master
       - releases/**
     paths-ignore:
-      - 'README.md'
+      - "README.md"
   pull_request:
     branches:
       - master
       - releases/**
     paths-ignore:
-      - 'README.md'
+      - "README.md"
+env:
+  CROWDSEC_BYPASS_DB_VOLUME_CHECK: "TRUE" # to avoid the db volume check in the docker_start.sh script
 
 jobs:
   test_flavor:
@@ -26,7 +28,6 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
-
       - name: Check out the repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:

docker/README.md

@@ -62,6 +62,11 @@ $ docker build -f Dockerfile.debian --build-arg=BUILD_ENV=slim .
 
 ## Required configuration
 
+### Volumes
+
+Since CrowdSec 1.7.0, `/var/lib/crowdsec/data/` is required to be mounted in a volume.
+If this directory is not mounted, the container will refuse to start.
+
 ### Journalctl (only for debian image)
 
 To use journalctl as a log stream, with or without the `DSN` environment variable, you need to mount the journal log from the host to the container itself.
@@ -350,6 +355,7 @@ config.yaml) each time the container is run.
 | __Developer options__       |                                    |                                                                                                                                                                                                                           |
 | `CI_TESTING`                | false                              | Used during functional tests                                                                                                                                                                                              |
 | `DEBUG`                     | false                              | Trace the entrypoint                                                                                                                                                                                                      |
+| `CROWDSEC_BYPASS_DB_VOLUME_CHECK` | false | Bypass volume check for `/var/lib/crowdsec/data/` |
 
 ## File Locations
 

docker/docker_start.sh

@@ -203,6 +203,10 @@ difference() {
 
 #-----------------------------------#
 
+# Tell crowdsec we are running in docker
+# The user agent will be updated so we can better detect broken installations running in docker
+export CROWDSEC_CONTAINER_ENV="docker"
+
 if [ -n "$CERT_FILE" ] || [ -n "$KEY_FILE" ] ; then
     printf '%b' '\033[0;33m'
     echo "Warning: the variables CERT_FILE and KEY_FILE have been deprecated." >&2
@@ -246,6 +250,21 @@ elif [ -n "$USE_WAL" ] && isfalse "$USE_WAL"; then
     conf_set '.db_config.use_wal = false'
 fi
 
+# Bail out if:
+# - `/var/lib/crowdsec/data` is not a volume
+# - CROWDSEC_BYPASS_DB_VOLUME_CHECK is not set
+# This check is performed regardless of the database type and if we are a LAPI or not:
+# - This directory is also used to store datafiles used by the LP, and some of them are really big and costly to download (MMDB files)
+# Do *not* implement this check in the k8s docker_start.sh
+if ! is_mounted "/var/lib/crowdsec/data" && [ -z "$CROWDSEC_BYPASS_DB_VOLUME_CHECK" ]; then
+        echo "No volume mounted for /var/lib/crowdsec/data"
+        echo "This directory is used to store the crowdsec local database (if using sqlite) and datafiles used by the parsers and scenarios."
+        echo "It is mandatory to mount a volume to this directory to persist the database and any datafiles downloaded from the hub."
+        echo "If you are doing a log replay or using a remote database (mysql, postgresql) on a LAPI-only container, you can set the environment variable CROWDSEC_BYPASS_DB_VOLUME_CHECK to skip this check."
+        echo "Exiting..."
+        exit 0 # No error to avoid a restart loop
+fi
+
 lapi_credentials_path=$(conf_get '.api.client.credentials_path')
 
 if isfalse "$DISABLE_LOCAL_API"; then

docker/test/pyproject.toml

@@ -6,7 +6,7 @@ readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
     "pytest>=8.3.4",
-    "pytest-cs>=0.7.22",
+    "pytest-cs>=0.7.24",
     "pytest-dotenv>=0.5.2",
     "pytest-xdist>=3.6.1",
 ]