fix several static machine issues - pyenv, thread limit, resource requirements (#418)

# Summary
This pull request introduces several improvements to the development and
test environment setup scripts, focusing on container runtime
configuration, resource limits, and performance tuning. The main changes
include adding a dedicated script for IBM container runtime setup,
updating resource requests and limits for Honeycomb performance testing,
and enhancing system resource limits for Minikube clusters.

**Container Runtime Setup and Configuration:**
* Added a new script `setup_ibm_container_runtime.sh` to automate
installation, upgrade, and configuration of `crun` as the container
runtime, including cleanup of conflicting configs and minimal
cross-version settings.

**Resource Limits and Performance Tuning:**
* set a resource requests and limits for Honeycomb DaemonSet and
Deployment in their respective values files to improve performance and
stability during testing (especially limiting it).
* Enhanced system resource limits in `setup_minikube.sh` by increasing
open files, process, kernel thread, PID, and memory map counts for
limits on podman.

**Minikube Cluster Improvements:**
* Updated Minikube cluster startup parameters to allocate more CPUs and
memory for improved cluster performance.

**Python Environment Setup:**
* Ensured `pyenv` is always updated before checking for required Python
versions, preventing issues with stale installations on static hosts.

# Proof of Work
- green ci
- green multi arch ibm ci:
https://spruce.mongodb.com/version/68bb1549eab6910007249140 +
https://spruce.mongodb.com/version/68be930977ae8a00073fbbf1

## Checklist

- [x] Have you linked a jira ticket and/or is the ticket in the title?
- [x] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: nammn

scripts/dev/recreate_python_venv.sh

@@ -76,6 +76,13 @@ ensure_required_python() {
         return 1
     fi
 
+    # Always update pyenv to ensure we have the latest Python versions available
+    # On static hosts we might have a stale pyenv installation.
+    echo "Updating pyenv to get latest Python versions..." >&2
+    if [[ -d "${HOME}/.pyenv/.git" ]]; then
+        cd "${HOME}/.pyenv" && git pull && cd - > /dev/null || echo "Warning: Failed to update pyenv via git" >&2
+    fi
+
     # Check if the required version is already installed
     if pyenv versions --bare | grep -q "^${required_version}$"; then
         echo "Python ${required_version} already installed via pyenv" >&2

scripts/dev/setup_ibm_container_runtime.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+echo "Installing/upgrading crun..."
+sudo dnf upgrade -y crun || sudo dnf install -y crun || sudo yum upgrade -y crun || sudo yum install -y crun
+
+if ! crun --version &>/dev/null; then
+  echo "❌ crun installation failed"
+  exit 1
+fi
+
+current_version=$(crun --version | head -n1)
+echo "✅ Using crun: ${current_version}"
+
+# Clean up any existing conflicting configurations
+echo "Cleaning up existing container configurations..."
+rm -f ~/.config/containers/containers.conf 2>/dev/null || true
+sudo rm -f /root/.config/containers/containers.conf 2>/dev/null || true
+sudo rm -f /etc/containers/containers.conf 2>/dev/null || true
+
+crun_path=$(which crun)
+echo "Using crun path: ${crun_path}"
+
+config="[containers]
+cgroup_manager = \"cgroupfs\"
+
+[engine]
+runtime = \"crun\""
+
+mkdir -p ~/.config/containers
+echo "${config}" > ~/.config/containers/containers.conf
+
+sudo mkdir -p /root/.config/containers
+echo "${config}" | sudo tee /root/.config/containers/containers.conf >/dev/null
+
+echo "✅ Configured crun"

scripts/evergreen/e2e/performance/honeycomb/values-daemonset.yaml

@@ -14,6 +14,14 @@ clusterRole:
       verbs:
         - get
 
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 100m
+    memory: 128Mi
+
 extraEnvs:
   - name: ENDPOINT
     valueFrom:

scripts/evergreen/e2e/performance/honeycomb/values-deployment.yaml

@@ -38,6 +38,14 @@ extraEnvs:
 # We only want one of these collectors - any more and we'd produce duplicate data
 replicaCount: 1
 
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 200m
+    memory: 256Mi
+
 presets:
   # enables the k8sclusterreceiver and adds it to the metrics pipelines
   clusterMetrics:

scripts/evergreen/setup_minikube_host.sh

@@ -49,6 +49,8 @@ run_setup_step "kubectl and helm Setup" "scripts/evergreen/setup_kubectl.sh"
 
 run_setup_step "jq Setup" "scripts/evergreen/setup_jq.sh"
 
+run_setup_step "IBM Container Runtime Setup" "scripts/dev/setup_ibm_container_runtime.sh"
+
 if [[ "${SKIP_MINIKUBE_SETUP:-}" != "true" ]]; then
   run_setup_step "Minikube Host Setup with Container Runtime Detection" "scripts/minikube/setup_minikube.sh"
 else
@@ -63,6 +65,7 @@ echo "✅ host setup completed successfully!"
 echo "=========================================="
 echo ""
 echo "Installed tools summary:"
+echo "Container Runtime: podman"
 echo "- Python: $(python --version 2>/dev/null || python3 --version 2>/dev/null || echo 'Not found')"
 echo "- AWS CLI: $(aws --version 2>/dev/null || echo 'Not found')"
 echo "- kubectl: $(kubectl version --client 2>/dev/null || echo 'Not found')"

scripts/minikube/setup_minikube.sh

@@ -13,9 +13,9 @@ set_limits() {
   echo "Increasing fs.inotify.max_user_watches"
   sudo sysctl -w fs.inotify.max_user_watches=10485760
 
-  echo "Increasing the number of open files"
+  echo "Increasing the number of open files and processes"
   nofile_max=$(cat /proc/sys/fs/nr_open)
-  nproc_max=$(ulimit -u)
+  nproc_max=65536
   sudo tee -a /etc/security/limits.conf <<EOF
   root hard nofile ${nofile_max}
   root hard nproc ${nproc_max}
@@ -26,6 +26,11 @@ set_limits() {
   * soft nofile ${nofile_max}
   * soft nproc ${nproc_max}
 EOF
+
+  echo "Setting kernel thread limits"
+  sudo sysctl -w kernel.threads-max=131072
+  sudo sysctl -w kernel.pid_max=131072
+  sudo sysctl -w vm.max_map_count=262144
 }
 
 # retrieve arch variable off the shell command line
@@ -149,6 +154,7 @@ start_minikube_cluster() {
   "${PROJECT_DIR:-.}/bin/minikube" delete 2>/dev/null || true
 
   local start_args=("--driver=podman")
+  start_args+=("--cpus=4" "--memory=8g")
 
   if [[ "${ARCH}" == "ppc64le" ]]; then
     echo "Using custom kicbase image for ppc64le with crictl..."
@@ -171,24 +177,6 @@ start_minikube_cluster() {
   fi
 }
 
-setup_podman() {
-  echo "Setting up podman for ${ARCH}..."
-
-  # Configure podman to use cgroupfs instead of systemd in CI
-  mkdir -p ~/.config/containers
-  cat > ~/.config/containers/containers.conf << EOF
-[containers]
-cgroup_manager = "cgroupfs"
-events_logger = "file"
-
-[engine]
-cgroup_manager = "cgroupfs"
-EOF
-
-}
-
-# Setup podman and container runtime
-setup_podman
 set_limits
 download_minikube
 
@@ -231,7 +219,6 @@ echo "=========================================="
 echo "✅ Setup Summary"
 echo "=========================================="
 echo "Architecture: ${ARCH}"
-echo "Container Runtime: podman"
 echo "Minikube Driver: podman"
 echo "Minikube: Default cluster"
 echo "Minikube: ${minikube_version}"