Skip to content

Commit 3cd78b4

Browse files
potiukpotiuk
authored
Synchronize common compat to 1.2.1 in v2-11-test branch (#61602)
When we release 2.11.1 - we will also have to release fab provider 1.5.4 and it has to have common-compat 1.2.1 as dependency because it uses some compatibility code from that version. While we are not planning to release common.compat together with the FAB provider and airflow, we need to import things from this provider from sources of airflow and fab provider in v2-11-test branch, and bringing the sources of common compat is needed to work in breeze and checked out sources and run tests. This is the same version of code as in common.compat 1.2.1 however it is adjusted back to the provider's structure that we had in Airflow 2 when providers shared the source tree with airflow.
1 parent 415166f commit 3cd78b4

File tree

13 files changed

+364
-9
lines changed

13 files changed

+364
-9
lines changed

‎airflow/providers/common/compat/CHANGELOG.rst‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,34 @@
2525
Changelog
2626
---------
2727

28+
1.2.1
29+
.....
30+
31+
Misc
32+
~~~~
33+
34+
* ``Rename dataset related python variable names to asset (#41348)``
35+
36+
37+
.. Below changes are excluded from the changelog. Move them to
38+
appropriate section above if needed. Do not delete the lines(!):
39+
40+
1.2.0
41+
.....
42+
43+
.. note::
44+
This release of provider is only available for Airflow 2.8+ as explained in the
45+
`Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
46+
47+
Misc
48+
~~~~
49+
50+
* ``Bump minimum Airflow version in providers to Airflow 2.8.0 (#41396)``
51+
52+
53+
.. Below changes are excluded from the changelog. Move them to
54+
appropriate section above if needed. Do not delete the lines(!):
55+
2856
1.1.0
2957
.....
3058

‎airflow/providers/common/compat/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@
2929

3030
__all__ = ["__version__"]
3131

32-
__version__ = "1.1.0"
32+
__version__ = "1.2.1"
3333

3434
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35-
"2.7.0"
35+
"2.8.0"
3636
):
3737
raise RuntimeError(
38-
f"The package `apache-airflow-providers-common-compat:{__version__}` needs Apache Airflow 2.7.0+"
38+
f"The package `apache-airflow-providers-common-compat:{__version__}` needs Apache Airflow 2.8.0+"
3939
)

‎airflow/providers/common/compat/assets/__init__.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
from __future__ import annotations
19+
20+
from typing import TYPE_CHECKING
21+
22+
from airflow import __version__ as AIRFLOW_VERSION
23+
24+
if TYPE_CHECKING:
25+
from airflow.assets import (
26+
Asset,
27+
AssetAlias,
28+
AssetAliasEvent,
29+
AssetAll,
30+
AssetAny,
31+
expand_alias_to_assets,
32+
)
33+
from airflow.auth.managers.models.resource_details import AssetDetails
34+
else:
35+
try:
36+
from airflow.assets import (
37+
Asset,
38+
AssetAlias,
39+
AssetAliasEvent,
40+
AssetAll,
41+
AssetAny,
42+
expand_alias_to_assets,
43+
)
44+
from airflow.auth.managers.models.resource_details import AssetDetails
45+
except ModuleNotFoundError:
46+
from packaging.version import Version
47+
48+
_IS_AIRFLOW_2_10_OR_HIGHER = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.10.0")
49+
_IS_AIRFLOW_2_9_OR_HIGHER = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.9.0")
50+
51+
# dataset is renamed to asset since Airflow 3.0
52+
from airflow.auth.managers.models.resource_details import DatasetDetails as AssetDetails
53+
from airflow.datasets import Dataset as Asset
54+
55+
if _IS_AIRFLOW_2_9_OR_HIGHER:
56+
from airflow.datasets import (
57+
DatasetAll as AssetAll,
58+
DatasetAny as AssetAny,
59+
)
60+
61+
if _IS_AIRFLOW_2_10_OR_HIGHER:
62+
from airflow.datasets import (
63+
DatasetAlias as AssetAlias,
64+
DatasetAliasEvent as AssetAliasEvent,
65+
expand_alias_to_datasets as expand_alias_to_assets,
66+
)
67+
68+
69+
__all__ = [
70+
"Asset",
71+
"AssetAlias",
72+
"AssetAliasEvent",
73+
"AssetAll",
74+
"AssetAny",
75+
"AssetDetails",
76+
"expand_alias_to_assets",
77+
]

‎airflow/providers/common/compat/lineage/hook.py‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,78 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19+
from importlib.util import find_spec
20+
21+
22+
def _get_asset_compat_hook_lineage_collector():
23+
from airflow.lineage.hook import get_hook_lineage_collector
24+
25+
collector = get_hook_lineage_collector()
26+
27+
if all(
28+
getattr(collector, asset_method_name, None)
29+
for asset_method_name in ("add_input_asset", "add_output_asset", "collected_assets")
30+
):
31+
return collector
32+
33+
# dataset is renamed as asset in Airflow 3.0
34+
35+
from functools import wraps
36+
37+
from airflow.lineage.hook import DatasetLineageInfo, HookLineage
38+
39+
DatasetLineageInfo.asset = DatasetLineageInfo.dataset
40+
41+
def rename_dataset_kwargs_as_assets_kwargs(function):
42+
@wraps(function)
43+
def wrapper(*args, **kwargs):
44+
if "asset_kwargs" in kwargs:
45+
kwargs["dataset_kwargs"] = kwargs.pop("asset_kwargs")
46+
47+
if "asset_extra" in kwargs:
48+
kwargs["dataset_extra"] = kwargs.pop("asset_extra")
49+
50+
return function(*args, **kwargs)
51+
52+
return wrapper
53+
54+
collector.create_asset = rename_dataset_kwargs_as_assets_kwargs(collector.create_dataset)
55+
collector.add_input_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_input_dataset)
56+
collector.add_output_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_output_dataset)
57+
58+
def collected_assets_compat(collector) -> HookLineage:
59+
"""Get the collected hook lineage information."""
60+
lineage = collector.collected_datasets
61+
return HookLineage(
62+
[
63+
DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
64+
for item in lineage.inputs
65+
],
66+
[
67+
DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
68+
for item in lineage.outputs
69+
],
70+
)
71+
72+
setattr(
73+
collector.__class__,
74+
"collected_assets",
75+
property(lambda collector: collected_assets_compat(collector)),
76+
)
77+
78+
return collector
79+
1980

2081
def get_hook_lineage_collector():
2182
# HookLineageCollector added in 2.10
2283
try:
23-
from airflow.lineage.hook import get_hook_lineage_collector
84+
if find_spec("airflow.assets"):
85+
# Dataset has been renamed as Asset in 3.0
86+
from airflow.lineage.hook import get_hook_lineage_collector
87+
88+
return get_hook_lineage_collector()
2489

25-
return get_hook_lineage_collector()
90+
return _get_asset_compat_hook_lineage_collector()
2691
except ImportError:
2792

2893
class NoOpCollector:
@@ -32,10 +97,10 @@ class NoOpCollector:
3297
It is used when you want to disable lineage collection.
3398
"""
3499

35-
def add_input_dataset(self, *_, **__):
100+
def add_input_asset(self, *_, **__):
36101
pass
37102

38-
def add_output_dataset(self, *_, **__):
103+
def add_output_asset(self, *_, **__):
39104
pass
40105

41106
return NoOpCollector()

‎airflow/providers/common/compat/openlineage/utils/__init__.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.

‎airflow/providers/common/compat/openlineage/utils/utils.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
from __future__ import annotations
19+
20+
from functools import wraps
21+
from typing import TYPE_CHECKING
22+
23+
if TYPE_CHECKING:
24+
from airflow.providers.openlineage.utils.utils import translate_airflow_asset
25+
else:
26+
try:
27+
from airflow.providers.openlineage.utils.utils import translate_airflow_asset
28+
except ImportError:
29+
from airflow.providers.openlineage.utils.utils import translate_airflow_dataset
30+
31+
def rename_asset_as_dataset(function):
32+
@wraps(function)
33+
def wrapper(*args, **kwargs):
34+
if "asset" in kwargs:
35+
kwargs["dataset"] = kwargs.pop("asset")
36+
return function(*args, **kwargs)
37+
38+
return wrapper
39+
40+
translate_airflow_asset = rename_asset_as_dataset(translate_airflow_dataset)
41+
42+
43+
__all__ = ["translate_airflow_asset"]

‎airflow/providers/common/compat/provider.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,16 @@ description: |
2222
``Common Compatibility Provider - providing compatibility code for previous Airflow versions.``
2323
2424
state: ready
25-
source-date-epoch: 1716287191
25+
source-date-epoch: 1728484960
2626
# note that those versions are maintained by release manager - do not update them manually
2727
versions:
28+
- 1.2.1
29+
- 1.2.0
2830
- 1.1.0
2931
- 1.0.0
3032

3133
dependencies:
32-
- apache-airflow>=2.7.0
34+
- apache-airflow>=2.8.0
3335

3436
integrations:
3537
- integration-name: Common Compat

‎airflow/providers/common/compat/security/__init__.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.

‎airflow/providers/common/compat/security/permissions.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
from typing import TYPE_CHECKING
20+
21+
if TYPE_CHECKING:
22+
from airflow.security.permissions import RESOURCE_ASSET
23+
else:
24+
try:
25+
from airflow.security.permissions import RESOURCE_ASSET
26+
except ImportError:
27+
from airflow.security.permissions import RESOURCE_DATASET as RESOURCE_ASSET
28+
29+
30+
__all__ = ["RESOURCE_ASSET"]

‎tests/providers/common/compat/openlineage/utils/__init__.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.

0 commit comments

Comments
 (0)