Merge branch 'main' into feat/bump-druid-opa-authorizer-pre-25.7.0

Author: NickLarsenNZ

CHANGELOG.md

@@ -69,13 +69,15 @@ All notable changes to this project will be documented in this file.
 - containerdebug updated to 0.2.0 ([#1128])
 - Build Hadoop as `stackable` and configure the Stackable Nexus build-repo for the `root` user ([#1133])
 - patchable: The base branch is now configured as the git upstream branch ([#1131]).
+- airflow: Updates the entrypoint script and removes the check for GID == 0 ([#1138])
 - druid: Bump druiod-opa-authorizer to `0.7.0` ([#1139]).
 
 ### Fixed
 
 - airflow: Pin Cython version ([#1116]).
 - druid: reduce docker image size by removing the recursive chown/chmods in the final image ([#1039]).
 - hadoop: reduce docker image size by removing the recursive chown/chmods in the final image ([#1029]).
+- hadoop: adapt the JMX exporter configuration to also export boolean metrics ([#1140]).
 - hbase: reduce docker image size by removing the recursive chown/chmods in the final image ([#1028]).
 - hive: reduce docker image size by removing the recursive chown/chmods in the final image ([#1040]).
 - kafka: reduce docker image size by removing the recursive chown/chmods in the final image ([#1041]).
@@ -159,6 +161,7 @@ All notable changes to this project will be documented in this file.
 [#1131]: https://github.com/stackabletech/docker-images/pull/1131
 [#1133]: https://github.com/stackabletech/docker-images/pull/1133
 [#1137]: https://github.com/stackabletech/docker-images/pull/1137
+[#1138]: https://github.com/stackabletech/docker-images/pull/1138
 [#1139]: https://github.com/stackabletech/docker-images/pull/1139
 
 ## [25.3.0] - 2025-03-21

airflow/stackable/utils/entrypoint.sh

@@ -19,9 +19,9 @@
 
 # Stackable notes:
 # Source of this file is the upstream Apache Airflow project
-# https://github.com/apache/airflow/blob/main/scripts/docker/entrypoint_prod.sh
-# It was last synced from the upstream repo on 2023-07-31 and is up-to-date as of commit 86193f5
-
+# https://github.com/apache/airflow/blob/dc271d0c604ca1836ee4f943726b2d436547700f/scripts/docker/entrypoint_prod.sh
+# It was last synced from the upstream repo on 2025-06-01 and is up-to-date as of commit dc271d0c604ca1836ee4f943726b2d436547700f
+# Changes we made are denoted with a comment "STACKABLE PATCH BEGIN" and # STACKABLE PATCH END
 
 AIRFLOW_COMMAND="${1:-}"
 
@@ -34,9 +34,12 @@ set -euo pipefail
 # The side effect of this is slightly (in the range of 100s of milliseconds) slower load for any
 # binary started and a little memory used for Heap allocated by initialization of libstdc++
 # This overhead is not happening for binaries that already link dynamically libstdc++
+
+# STACKABLE PATCH BEGIN
+# The path to this file is different on UBI
 # LD_PRELOAD="/usr/lib/$(uname -m)-linux-gnu/libstdc++.so.6"
-# Stackable: The path to this file is different on UBI
 LD_PRELOAD=/usr/lib64/libstdc++.so.6
+# STACKABLE PATCH END
 export LD_PRELOAD
 
 function run_check_with_retries {
@@ -161,13 +164,17 @@ function create_www_user() {
         exit 1
     fi
 
-    airflow users create \
-       --username "${_AIRFLOW_WWW_USER_USERNAME="admin"}" \
-       --firstname "${_AIRFLOW_WWW_USER_FIRSTNAME="Airflow"}" \
-       --lastname "${_AIRFLOW_WWW_USER_LASTNAME="Admin"}" \
-       --email "${_AIRFLOW_WWW_USER_EMAIL="[email protected]"}" \
-       --role "${_AIRFLOW_WWW_USER_ROLE="Admin"}" \
-       --password "${local_password}" || true
+    if airflow config get-value core auth_manager | grep -q "FabAuthManager"; then
+        airflow users create \
+           --username "${_AIRFLOW_WWW_USER_USERNAME="admin"}" \
+           --firstname "${_AIRFLOW_WWW_USER_FIRSTNAME="Airflow"}" \
+           --lastname "${_AIRFLOW_WWW_USER_LASTNAME="Admin"}" \
+           --email "${_AIRFLOW_WWW_USER_EMAIL="[email protected]"}" \
+           --role "${_AIRFLOW_WWW_USER_ROLE="Admin"}" \
+           --password "${local_password}" || true
+    else
+        echo "Skipping user creation as auth manager different from Fab is used"
+    fi
 }
 
 function create_system_user_if_missing() {
@@ -193,7 +200,7 @@ function set_pythonpath_for_root_user() {
     # Now also adds applications installed as local user "airflow".
     if [[ $UID == "0" ]]; then
         local python_major_minor
-        python_major_minor="$(python --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
+        python_major_minor=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
         export PYTHONPATH="${AIRFLOW_USER_HOME_DIR}/.local/lib/python${python_major_minor}/site-packages:${PYTHONPATH:-}"
         >&2 echo "The container is run as root user. For security, consider using a regular user account."
     fi
@@ -204,9 +211,9 @@ function wait_for_airflow_db() {
     run_check_with_retries "airflow db check"
 }
 
-function upgrade_db() {
-    # Runs airflow db upgrade
-    airflow db upgrade || true
+function migrate_db() {
+    # Runs airflow db migrate
+    airflow db migrate || true
 }
 
 function wait_for_celery_broker() {
@@ -272,7 +279,10 @@ function check_uid_gid() {
 # not set when PIP is run by Airflow later on
 unset PIP_USER
 
-check_uid_gid
+# STACKABLE PATCH BEGIN
+# Disable check for uid & gid (https://github.com/stackabletech/issues/issues/645)
+# check_uid_gid
+# STACKABLE PATCH END
 
 # Set umask to 0002 to make all the directories created by the current user group-writeable
 # This allows the same directories to be writeable for any arbitrary user the image will be
@@ -292,8 +302,12 @@ if [[ "${CONNECTION_CHECK_MAX_COUNT}" -gt "0" ]]; then
     wait_for_airflow_db
 fi
 
+if [[ -n "${_AIRFLOW_DB_UPGRADE=}" ]] || [[ -n "${_AIRFLOW_DB_MIGRATE=}" ]] ; then
+    migrate_db
+fi
+
 if [[ -n "${_AIRFLOW_DB_UPGRADE=}" ]] ; then
-    upgrade_db
+    >&2 echo "WARNING: Environment variable '_AIRFLOW_DB_UPGRADE' is deprecated please use '_AIRFLOW_DB_MIGRATE' instead"
 fi
 
 if [[ -n "${_AIRFLOW_WWW_USER_CREATE=}" ]] ; then
@@ -310,10 +324,13 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then
     >&2 echo "         https://airflow.apache.org/docs/docker-stack/build.html"
     >&2 echo
     >&2 echo "         Adding requirements at container startup is fragile and is done every time"
-    >&2 echo "         the container starts, so it is onlny useful for testing and trying out"
+    >&2 echo "         the container starts, so it is only useful for testing and trying out"
     >&2 echo "         of adding dependencies."
     >&2 echo
-    pip install --root-user-action ignore --no-cache-dir "${_PIP_ADDITIONAL_REQUIREMENTS}"
+    # STACKABLE PATCH BEGIN
+    # Add double quotes to silence Shellcheck warning SC2086
+    pip install --root-user-action ignore "${_PIP_ADDITIONAL_REQUIREMENTS}"
+    # STACKABLE PATCH END
 fi
 
 

hadoop/stackable/jmx/datanode.yaml

@@ -12,10 +12,9 @@ blacklistObjectNames:
   - 'Hadoop:service=DataNode,name=UgiMetrics'
 rules:
   # MetricsSystem
-  - pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
@@ -24,21 +23,19 @@ rules:
     type: GAUGE
   # FSDatasetState with _total suffix (also extracts the FSDataset ID),
   # e.g. Hadoop:name=FSDatasetState,attribute=EstimatedCapacityLostTotal
-  - pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*_total): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*_total):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
       fsdatasetid: $2
       kind: 'FSDatasetState'
     type: COUNTER
   # FSDatasetState (also extracts the FSDataset ID)
-  - pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
@@ -47,21 +44,19 @@ rules:
     type: GAUGE
   # DataNodeActivity with _info suffix (also extracts hostname and port),
   # e.g. Hadoop:name=DataNodeActivity-hdfs-datanode-default-0-9866,attribute=BlocksGetLocalPathInfo
-  - pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*_info): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*_info):'
     attrNameSnakeCase: true
     name: hadoop_$1_$4_
-    value: $5
     labels:
       service: HDFS
       role: $1
       host: $2
       port: $3
       kind: 'DataNodeActivity'
     type: GAUGE
-  - pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$4
-    value: $5
     labels:
       service: HDFS
       role: $1
@@ -70,31 +65,28 @@ rules:
       kind: 'DataNodeActivity'
     type: GAUGE
   # Generic counter, e.g. Hadoop:name=FSDatasetState,attribute=EstimatedCapacityLostTotal
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: COUNTER
   # Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
   # The suffix _info is reserved for static information, therefore an underscore is appended.
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3_
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: GAUGE
   # All other Hadoop metrics
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1

hadoop/stackable/jmx/journalnode.yaml

@@ -13,10 +13,9 @@ blacklistObjectNames:
   - 'Hadoop:service=JournalNode,name=UgiMetrics'
 rules:
   # MetricsSystem
-  - pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
@@ -25,20 +24,18 @@ rules:
     type: GAUGE
   # Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
   # The suffix _info is reserved for static information, therefore an underscore is appended.
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3_
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: GAUGE
   # All other Hadoop metrics
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1

hadoop/stackable/jmx/namenode.yaml

@@ -13,63 +13,57 @@ blacklistObjectNames:
   - 'Hadoop:service=NameNode,name=UgiMetrics'
 rules:
   # MetricsSystem
-  - pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=MetricsSystem, sub=(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: 'MetricsSystem'
       sub: $2
     type: GAUGE
   # Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(total): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(total):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: COUNTER
   # Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total): (\d+)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_total):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: COUNTER
   # Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
   # The suffix _created is reserved for timestamps, therefore an underscore is appended.
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_created): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_created):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3_
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: GAUGE
   # Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
   # The suffix _info is reserved for static information, therefore an underscore is appended.
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*_info):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3_
-    value: $4
     labels:
       service: HDFS
       role: $1
       kind: $2
     type: GAUGE
   # All other Hadoop metrics
-  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*): (.*)'
+  - pattern: 'Hadoop<service=(.*), name=(.*)><>(.*):'
     attrNameSnakeCase: true
     name: hadoop_$1_$3
-    value: $4
     labels:
       service: HDFS
       role: $1