Source conf from airflow.sdk.configuration.conf for apache providers (#59966)

sunank200 · web-flow · commit c8aa74a19e39 · 2025-12-31T21:08:01.000+05:30
* Migrate apache provider to use airflow.sdk.configuration.conf

* Add 'use next version' comment for common-compat dependency
diff --git a/providers/apache/beam/pyproject.toml b/providers/apache/beam/pyproject.toml
@@ -58,7 +58,7 @@ requires-python = ">=3.10,!=3.13"
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.11.0",
-    "apache-airflow-providers-common-compat>=1.10.1",
+    "apache-airflow-providers-common-compat>=1.10.1",  # use next version
     'apache-beam>=2.60.0; python_version < "3.13"',
     "pyarrow>=16.1.0; python_version < '3.13'",
     "numpy>=1.22.4; python_version<'3.11'",
diff --git a/providers/apache/beam/src/airflow/providers/apache/beam/operators/beam.py b/providers/apache/beam/src/airflow/providers/apache/beam/operators/beam.py
@@ -30,11 +30,10 @@
 from functools import partial
 from typing import TYPE_CHECKING, Any
 
-from airflow.configuration import conf
 from airflow.exceptions import AirflowOptionalProviderFeatureException
 from airflow.providers.apache.beam.hooks.beam import BeamHook, BeamRunnerType
 from airflow.providers.apache.beam.triggers.beam import BeamJavaPipelineTrigger, BeamPythonPipelineTrigger
-from airflow.providers.common.compat.sdk import AirflowException, BaseOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, conf
 from airflow.providers_manager import ProvidersManager
 from airflow.utils.helpers import convert_camel_to_snake, exactly_one
 from airflow.version import version
diff --git a/providers/apache/hdfs/pyproject.toml b/providers/apache/hdfs/pyproject.toml
@@ -59,7 +59,7 @@ requires-python = ">=3.10"
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.11.0",
-    "apache-airflow-providers-common-compat>=1.10.1",
+    "apache-airflow-providers-common-compat>=1.10.1",  # use next version
     'hdfs[avro,dataframe,kerberos]>=2.5.4;python_version<"3.12"',
     'hdfs[avro,dataframe,kerberos]>=2.7.3;python_version>="3.12"',
     'pandas>=2.1.2; python_version <"3.13"',
diff --git a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/hooks/webhdfs.py b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/hooks/webhdfs.py
@@ -24,8 +24,7 @@
 import requests
 from hdfs import HdfsError, InsecureClient
 
-from airflow.configuration import conf
-from airflow.providers.common.compat.sdk import AirflowException, BaseHook
+from airflow.providers.common.compat.sdk import AirflowException, BaseHook, conf
 
 log = logging.getLogger(__name__)
 
diff --git a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/log/hdfs_task_handler.py
@@ -27,8 +27,8 @@
 
 import attrs
 
-from airflow.configuration import conf
 from airflow.providers.apache.hdfs.hooks.webhdfs import WebHDFSHook
+from airflow.providers.common.compat.sdk import conf
 from airflow.utils.log.file_task_handler import FileTaskHandler
 from airflow.utils.log.logging_mixin import LoggingMixin
 
diff --git a/providers/apache/hive/pyproject.toml b/providers/apache/hive/pyproject.toml
@@ -59,7 +59,7 @@ requires-python = ">=3.10"
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.11.0",
-    "apache-airflow-providers-common-compat>=1.10.1",
+    "apache-airflow-providers-common-compat>=1.10.1",  # use next version
     "apache-airflow-providers-common-sql>=1.26.0",
     "hmsclient>=0.1.0",
     'pandas>=2.1.2; python_version <"3.13"',
diff --git a/providers/apache/hive/src/airflow/providers/apache/hive/hooks/hive.py b/providers/apache/hive/src/airflow/providers/apache/hive/hooks/hive.py
@@ -32,12 +32,12 @@
 from sqlalchemy.engine import URL
 from typing_extensions import overload
 
-from airflow.configuration import conf
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.common.compat.sdk import (
     AIRFLOW_VAR_NAME_FORMAT_MAPPING,
     AirflowException,
     BaseHook,
+    conf,
 )
 from airflow.providers.common.sql.hooks.sql import DbApiHook
 from airflow.security import utils
diff --git a/providers/apache/hive/src/airflow/providers/apache/hive/operators/hive.py b/providers/apache/hive/src/airflow/providers/apache/hive/operators/hive.py
@@ -23,11 +23,11 @@
 from functools import cached_property
 from typing import TYPE_CHECKING, Any
 
-from airflow.configuration import conf
 from airflow.providers.apache.hive.hooks.hive import HiveCliHook
 from airflow.providers.common.compat.sdk import (
     AIRFLOW_VAR_NAME_FORMAT_MAPPING,
     BaseOperator,
+    conf,
     context_to_airflow_vars,
 )
 
diff --git a/providers/apache/hive/tests/unit/apache/hive/operators/test_hive.py b/providers/apache/hive/tests/unit/apache/hive/operators/test_hive.py
@@ -22,9 +22,9 @@
 
 import pytest
 
-from airflow.configuration import conf
 from airflow.models import DagRun, TaskInstance
 from airflow.providers.apache.hive.operators.hive import HiveOperator
+from airflow.providers.common.compat.sdk import conf
 from airflow.utils import timezone
 
 from unit.apache.hive import DEFAULT_DATE, MockSubProcess, TestHiveEnvironment
diff --git a/providers/apache/livy/pyproject.toml b/providers/apache/livy/pyproject.toml
@@ -60,7 +60,7 @@ requires-python = ">=3.10"
 dependencies = [
     "apache-airflow>=2.11.0",
     "apache-airflow-providers-http>=5.1.0",
-    "apache-airflow-providers-common-compat>=1.10.1",
+    "apache-airflow-providers-common-compat>=1.10.1",  # use next version
     "aiohttp>=3.9.2",
     "asgiref>=2.3.0",
 ]
diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/operators/livy.py b/providers/apache/livy/src/airflow/providers/apache/livy/operators/livy.py
@@ -21,14 +21,13 @@
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, cast
 
-from airflow.configuration import conf
 from airflow.providers.apache.livy.hooks.livy import BatchState, LivyHook
 from airflow.providers.apache.livy.triggers.livy import LivyTrigger
 from airflow.providers.common.compat.openlineage.utils.spark import (
     inject_parent_job_information_into_spark_properties,
     inject_transport_information_into_spark_properties,
 )
-from airflow.providers.common.compat.sdk import AirflowException, BaseOperator
+from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, conf
 
 if TYPE_CHECKING:
     from airflow.providers.common.compat.sdk import Context
diff --git a/providers/apache/spark/pyproject.toml b/providers/apache/spark/pyproject.toml
@@ -59,7 +59,7 @@ requires-python = ">=3.10"
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.11.0",
-    "apache-airflow-providers-common-compat>=1.10.1",
+    "apache-airflow-providers-common-compat>=1.10.1",  # use next version
     "pyspark>=3.5.2",
     "grpcio-status>=1.59.0",
 ]
diff --git a/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py b/providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py
@@ -30,8 +30,7 @@
 from pathlib import Path
 from typing import Any
 
-from airflow.configuration import conf as airflow_conf
-from airflow.providers.common.compat.sdk import AirflowException, BaseHook
+from airflow.providers.common.compat.sdk import AirflowException, BaseHook, conf as airflow_conf
 from airflow.security.kerberos import renew_from_kt
 from airflow.utils.log.logging_mixin import LoggingMixin
 
diff --git a/providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_submit.py b/providers/apache/spark/src/airflow/providers/apache/spark/operators/spark_submit.py
@@ -20,13 +20,12 @@
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any
 
-from airflow.configuration import conf
 from airflow.providers.apache.spark.hooks.spark_submit import SparkSubmitHook
 from airflow.providers.common.compat.openlineage.utils.spark import (
     inject_parent_job_information_into_spark_properties,
     inject_transport_information_into_spark_properties,
 )
-from airflow.providers.common.compat.sdk import BaseOperator
+from airflow.providers.common.compat.sdk import BaseOperator, conf
 
 if TYPE_CHECKING:
     from airflow.providers.common.compat.sdk import Context
diff --git a/providers/common/compat/src/airflow/providers/common/compat/sdk.py b/providers/common/compat/src/airflow/providers/common/compat/sdk.py
@@ -76,6 +76,7 @@
         task_decorator_factory as task_decorator_factory,
     )
     from airflow.sdk.bases.sensor import poke_mode_only as poke_mode_only
+    from airflow.sdk.configuration import conf as conf
     from airflow.sdk.definitions.context import context_merge as context_merge
     from airflow.sdk.definitions.mappedoperator import MappedOperator as MappedOperator
     from airflow.sdk.definitions.template import literal as literal
@@ -249,6 +250,10 @@
         "airflow.sdk.execution_time.secrets_masker",
         "airflow.utils.log.secrets_masker",
     ),
+    # ============================================================================
+    # Configuration
+    # ============================================================================
+    "conf": ("airflow.sdk.configuration", "airflow.configuration"),
 }
 
 # Airflow 3-only exceptions (not available in Airflow 2)

Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ requires-python = ">=3.10"`
`60`	`60`	`dependencies = [`
`61`	`61`	`"apache-airflow>=2.11.0",`
`62`	`62`	`"apache-airflow-providers-http>=5.1.0",`
`63`		`- "apache-airflow-providers-common-compat>=1.10.1",`
	`63`	`+ "apache-airflow-providers-common-compat>=1.10.1", # use next version`
`64`	`64`	`"aiohttp>=3.9.2",`
`65`	`65`	`"asgiref>=2.3.0",`
`66`	`66`	`]`