From 4796d8945a7f947f2359cd91aa1ffa2df6610ede Mon Sep 17 00:00:00 2001 From: smartbit Date: Tue, 20 May 2025 17:30:43 +0200 Subject: [PATCH] Add persistent kernel parameter tuning for CloudNativePG Kind cluster Signed-off-by: smartbit --- scripts/setup.sh | 34 ++++++++++ scripts/tune-sysctl.sh | 140 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100755 scripts/tune-sysctl.sh diff --git a/scripts/setup.sh b/scripts/setup.sh index 7b0fbb3..31ba09b 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -31,6 +31,40 @@ set -eu +# Minimal thresholds to check before calling tuning script +declare -A thresholds=( + [fs.inotify.max_user_watches]=20000 + [fs.inotify.max_user_instances]=1000 + [kernel.keys.maxkeys]=1000 + [kernel.keys.maxbytes]=250000 +) + +needs_update=0 + +# Check current sysctl values +for key in "${!thresholds[@]}"; do + path="/proc/sys/$(echo "$key" | tr '.' '/')" + if [[ -f "$path" ]]; then + current=$(cat "$path") + if (( current < thresholds[$key] )); then + echo "Current $key ($current) is below threshold (${thresholds[$key]})" + needs_update=1 + fi + else + echo "Warning: sysctl key $key not found at $path" + fi +done + +# Run the tuning script if needed +if (( needs_update )); then + echo "Running tuning script to update sysctl settings..." + "${BASH_SOURCE%/*}/tune-sysctl.sh" + ret=$? + if (( ret != 0 )); then + echo "$(basename "$0"): Tuning script exited without applying changes." + fi +fi + # MinIO settings and credentials MINIO_IMAGE="${MINIO_IMAGE:-quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z}" MINIO_EU_ROOT_USER="${MINIO_EU_ROOT_USER:-cnpg-eu}" diff --git a/scripts/tune-sysctl.sh b/scripts/tune-sysctl.sh new file mode 100755 index 0000000..e0d4cf5 --- /dev/null +++ b/scripts/tune-sysctl.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +# Desired settings and thresholds +declare -A limits=( + [fs.inotify.max_user_watches]="655360:20000" + [fs.inotify.max_user_instances]="1280:1000" + [kernel.keys.maxkeys]="20000:1000" + [kernel.keys.maxbytes]="500000:250000" +) + +SYSCTL_CONF_FILE="/etc/sysctl.d/99-custom-tuning.conf" +TMP_FILE="$(mktemp)" +CHANGES_MADE=0 +CHANGES_OUTPUT=() +KEY_ENTRIES=() +INOTIFY_ENTRIES=() +SYSCTL_APPLY_COMMAND=() + +# Check current values and prepare entries +for key in "${!limits[@]}"; do + current=$(cat /proc/sys/$(echo "$key" | tr '.' '/')) + desired="${limits[$key]%%:*}" + threshold="${limits[$key]##*:}" + + if (( current < threshold )); then + CHANGES_MADE=1 + CHANGES_OUTPUT+=("$key: $current -> $desired") + line="$key=$desired" + + # Prepare for temporary sysctl -w + SYSCTL_APPLY_COMMAND+=("$line") + + if [[ $key == kernel.keys.* ]]; then + KEY_ENTRIES+=("$line") + else + INOTIFY_ENTRIES+=("$line") + fi + fi +done + +if (( CHANGES_MADE == 0 )); then + echo "No changes required. All values meet or exceed thresholds." + rm -f "$TMP_FILE" + exit 0 +fi + +# Show proposed changes +echo "The following sysctl parameter changes are proposed:" +for change in "${CHANGES_OUTPUT[@]}"; do + echo " - $change" +done + +# Non-interactive: apply with sysctl -w and exit +if ! [ -t 0 ]; then + echo "Non-interactive session detected. Applying changes temporarily (not persisted)..." + sudo sysctl -w "${SYSCTL_APPLY_COMMAND[@]}" + rm -f "$TMP_FILE" + exit 0 +fi + +# Prompt user +echo -e "\nChoose how to apply these changes:" +echo " [1] Persist and apply (recommended)" +echo " [2] Apply temporarily only (until reboot)" +echo " [3] Do not apply" + +read -rp "Your choice (1/2/3): " choice +choice=${choice:-1} + +if [[ "$choice" == "1" ]]; then + # Clean low-value entries from sysctl.conf and other files (but not from our custom file) + for key in "${!limits[@]}"; do + key_escaped=$(echo "$key" | sed 's/\./\\./g') + desired="${limits[$key]%%:*}" + threshold="${limits[$key]##*:}" + + # Remove from /etc/sysctl.conf + if grep -qE "^\s*${key_escaped}\s*=" /etc/sysctl.conf 2>/dev/null; then + current_val=$(grep -E "^\s*${key_escaped}\s*=" /etc/sysctl.conf | head -n1 | cut -d= -f2 | tr -d ' ') + if [[ "$current_val" =~ ^[0-9]+$ ]] && (( current_val < threshold )); then + sudo sed -i "/^\s*${key_escaped}\s*=/d" /etc/sysctl.conf + fi + fi + + # Remove from other sysctl.d files + find /etc/sysctl.d /usr/lib/sysctl.d -type f ! -name '99-custom-tuning.conf' 2>/dev/null | while read -r file; do + if grep -qE "^\s*${key_escaped}\s*=" "$file"; then + current_val=$(grep -E "^\s*${key_escaped}\s*=" "$file" | head -n1 | cut -d= -f2 | tr -d ' ') + if [[ "$current_val" =~ ^[0-9]+$ ]] && (( current_val < threshold )); then + sudo sed -i "/^\s*${key_escaped}\s*=/d" "$file" + fi + fi + done + done + + # Build new config + { + echo "# Custom sysctl tuning" + for line in "${INOTIFY_ENTRIES[@]}"; do + echo "$line" + done + if (( ${#KEY_ENTRIES[@]} > 0 )); then + echo -e "\n# See https://github.com/moby/moby/issues/22865" + for line in "${KEY_ENTRIES[@]}"; do + echo "$line" + done + fi + } > "$TMP_FILE" + + sudo mv "$TMP_FILE" "$SYSCTL_CONF_FILE" + sudo chmod 644 "$SYSCTL_CONF_FILE" + + echo -e "\nApplying and persisting changes..." + sudo sysctl --system >/dev/null + for key in "${!limits[@]}"; do + sysctl "$key" + done + + echo "Changes persisted in $SYSCTL_CONF_FILE" + exit 0 + +elif [[ "$choice" == "2" ]]; then + echo "Applying changes temporarily with sysctl -w..." + sudo sysctl -w "${SYSCTL_APPLY_COMMAND[@]}" + exit 0 + +else + echo -e "\n\033[1;31mWARNING:\033[0m" + echo -e "\033[1;31mYou chose not to apply the recommended kernel parameter updates.\033[0m" + echo -e "\033[1;31mThis may cause failures in Kind or CloudNativePG clusters due to insufficient kernel limits.\033[0m" + read -rp "Are you sure you want to continue without applying these settings? (y/N): " confirm + confirm=${confirm:-N} + if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + echo "Re-running script to apply settings..." + exec "$0" + else + echo "No changes applied." + exit 1 + fi +fi