Add validation for pool names to prevent InvalidStatsNameException (#59938)

kalluripradeep · web-flow · commit b7fe72676fad · 2026-01-03T18:35:42.000+01:00
* Add Validation for pool names to prevent InvalidStatsNameException Pool names can now only contain ASCII alphabets, numbers , underscores,dots,and dashes to ensure compatibility with stats naming requirments. fixes #59935 * Add unit Tests and news fragment for pool name validation * Fix test file with proper database session handling * Fix test assertions to use pytest.raises match parameter * Add newline at the end of pool.py file * Fix trailing whitespace and quote style in pool.py * Fix syntax error: use consistent double quotes in regex * Add missing newline and __init__.py file * Add missing newline and __init__.py with license * Fix __init__.py with proper license and single newline * Implement pool name normalization for stats reporting Following @potiuk's feedback, changed approach from preventing invalid pool names to normalizing them for stats reporting. Changes: - Replaced validate_pool_name() with normalize_pool_name_for_stats() - Pool names with invalid characters are normalized (replaced with _) when emitting metrics - Logs warning when normalization occurs, suggesting pool rename - Removed validation from create_or_update_pool() - Updated scheduler_job_runner.py to use normalized names for stats - Removed validation tests - Updated news fragment This prevents InvalidStatsNameException without breaking existing pools with invalid names. Fixes #59935 * Implement pool name normalization for stats reporting Following @potiuk's feedback, changed approach from preventing invalid pool names to normalizing them for stats reporting. Changes: - Added normalize_pool_name_for_stats() function in pool.py - Pool names with invalid characters are normalized (replaced with _) when emitting metrics in scheduler_job_runner.py - Logs warning when normalization occurs - Removed validation tests - Removed accidental fix_quote.py file - Updated news fragment This prevents InvalidStatsNameException without breaking existing pools. Fixes #59935 * Add missing 're' module import * Remove duplicate import * Apply CI formatting fixes - import order and blank lines * Fix formatting - add proper blank lines per ruff format requirements
diff --git a/airflow-core/src/airflow/jobs/scheduler_job_runner.py b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
@@ -74,6 +74,7 @@
 from airflow.models.dagbundle import DagBundleModel
 from airflow.models.dagrun import DagRun
 from airflow.models.dagwarning import DagWarning, DagWarningType
+from airflow.models.pool import normalize_pool_name_for_stats
 from airflow.models.serialized_dag import SerializedDagModel
 from airflow.models.taskinstance import TaskInstance
 from airflow.models.team import Team
@@ -2534,11 +2535,12 @@ def _emit_pool_metrics(self, session: Session = NEW_SESSION) -> None:
         with DebugTrace.start_span(span_name="emit_pool_metrics", component="SchedulerJobRunner") as span:
             pools = Pool.slots_stats(session=session)
             for pool_name, slot_stats in pools.items():
-                Stats.gauge(f"pool.open_slots.{pool_name}", slot_stats["open"])
-                Stats.gauge(f"pool.queued_slots.{pool_name}", slot_stats["queued"])
-                Stats.gauge(f"pool.running_slots.{pool_name}", slot_stats["running"])
-                Stats.gauge(f"pool.deferred_slots.{pool_name}", slot_stats["deferred"])
-                Stats.gauge(f"pool.scheduled_slots.{pool_name}", slot_stats["scheduled"])
+                normalized_pool_name = normalize_pool_name_for_stats(pool_name)
+                Stats.gauge(f"pool.open_slots.{normalized_pool_name}", slot_stats["open"])
+                Stats.gauge(f"pool.queued_slots.{normalized_pool_name}", slot_stats["queued"])
+                Stats.gauge(f"pool.running_slots.{normalized_pool_name}", slot_stats["running"])
+                Stats.gauge(f"pool.deferred_slots.{normalized_pool_name}", slot_stats["deferred"])
+                Stats.gauge(f"pool.scheduled_slots.{normalized_pool_name}", slot_stats["scheduled"])
 
                 # Same metrics with tagging
                 Stats.gauge("pool.open_slots", slot_stats["open"], tags={"pool_name": pool_name})
@@ -2550,11 +2552,11 @@ def _emit_pool_metrics(self, session: Session = NEW_SESSION) -> None:
                 span.set_attributes(
                     {
                         "category": "scheduler",
-                        f"pool.open_slots.{pool_name}": slot_stats["open"],
-                        f"pool.queued_slots.{pool_name}": slot_stats["queued"],
-                        f"pool.running_slots.{pool_name}": slot_stats["running"],
-                        f"pool.deferred_slots.{pool_name}": slot_stats["deferred"],
-                        f"pool.scheduled_slots.{pool_name}": slot_stats["scheduled"],
+                        f"pool.open_slots.{normalized_pool_name}": slot_stats["open"],
+                        f"pool.queued_slots.{normalized_pool_name}": slot_stats["queued"],
+                        f"pool.running_slots.{normalized_pool_name}": slot_stats["running"],
+                        f"pool.deferred_slots.{normalized_pool_name}": slot_stats["deferred"],
+                        f"pool.scheduled_slots.{normalized_pool_name}": slot_stats["scheduled"],
                     }
                 )
 
diff --git a/airflow-core/src/airflow/models/pool.py b/airflow-core/src/airflow/models/pool.py
@@ -17,6 +17,8 @@
 # under the License.
 from __future__ import annotations
 
+import logging
+import re
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, TypedDict
 
@@ -35,6 +37,37 @@
     from sqlalchemy.orm.session import Session
     from sqlalchemy.sql import Select
 
+logger = logging.getLogger(__name__)
+
+
+def normalize_pool_name_for_stats(name: str) -> str:
+    """
+    Normalize pool name for stats reporting by replacing invalid characters.
+
+    Stats names must only contain ASCII alphabets, numbers, underscores, dots, and dashes.
+    Invalid characters are replaced with underscores.
+
+    :param name: The pool name to normalize
+    :return: Normalized pool name safe for stats reporting
+    """
+    # Check if normalization is needed
+    if re.match(r"^[a-zA-Z0-9_.-]+$", name):
+        return name
+
+    # Replace invalid characters with underscores
+    normalized = re.sub(r"[^a-zA-Z0-9_.-]", "_", name)
+
+    # Log warning
+    logger.warning(
+        "Pool name '%s' contains invalid characters for stats reporting. "
+        "Reporting stats with normalized name '%s'. "
+        "Consider renaming the pool to avoid this warning.",
+        name,
+        normalized,
+    )
+
+    return normalized
+
 
 class PoolStats(TypedDict):
     """Dictionary containing Pool Stats."""
diff --git a/newsfragments/59938.bugfix.rst b/newsfragments/59938.bugfix.rst
@@ -0,0 +1 @@
+Pool names with invalid characters for stats reporting are now automatically normalized (invalid characters replaced with underscores) when emitting metrics, preventing ``InvalidStatsNameException``. A warning is logged when normalization occurs, suggesting the pool be renamed.
diff --git a/tests/models/__init__.py b/tests/models/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/models/test_pool.py b/tests/models/test_pool.py
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Pool names with invalid characters for stats reporting are now automatically normalized (invalid characters replaced with underscores) when emitting metrics, preventing ``InvalidStatsNameException``. A warning is logged when normalization occurs, suggesting the pool be renamed.