diff --git a/CHANGELOG.md b/CHANGELOG.md index a8280048b..e085b8bab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file. - hbase: Add `2.6.3` ([#1296]). - airflow,superset: Build from source ([#1304]). - kafka: Add updated JMX config `server.yaml` for KRaft ([#1308]). +- airflow: Add apache-kafka provider plus dependencies to `3.0.6` image ([#1313]). ### Changed @@ -109,6 +110,7 @@ All notable changes to this project will be documented in this file. [#1308]: https://github.com/stackabletech/docker-images/pull/1308 [#1309]: https://github.com/stackabletech/docker-images/pull/1309 [#1311]: https://github.com/stackabletech/docker-images/pull/1311 +[#1313]: https://github.com/stackabletech/docker-images/pull/1313 [#1314]: https://github.com/stackabletech/docker-images/pull/1314 [#1315]: https://github.com/stackabletech/docker-images/pull/1315 [#1316]: https://github.com/stackabletech/docker-images/pull/1316 diff --git a/airflow/boil-config.toml b/airflow/boil-config.toml index a787c09ac..54d5f62c2 100644 --- a/airflow/boil-config.toml +++ b/airflow/boil-config.toml @@ -58,6 +58,6 @@ s3fs-version = "2024.9.0" cyclonedx-bom-version = "6.0.0" tini-version = "0.19.0" uv-version = "0.7.8" -airflow-extras = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino" +airflow-extras = "amazon,apache-kafka,async,celery,cncf-kubernetes,common-messaging,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino" opa-auth-manager = "airflow-3" nodejs-version = "20" diff --git a/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt b/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt index 904a717b6..4e025f60c 100644 --- a/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt +++ b/airflow/stackable/constraints/3.0.6/constraints-python3.12.txt @@ -109,7 +109,10 @@ apache-airflow-providers-apache-hdfs==4.10.2 apache-airflow-providers-apache-hive==9.1.2 apache-airflow-providers-apache-iceberg==1.3.2 apache-airflow-providers-apache-impala==1.7.2 -apache-airflow-providers-apache-kafka==1.10.2 +# Stackable patch: +# The original version 1.10.2 has sync/async issues when using event scheduling +# with the triggerer. The fix could not be traced to a specific change/commit. +apache-airflow-providers-apache-kafka==1.10.4 apache-airflow-providers-apache-kylin==3.9.2 apache-airflow-providers-apache-livy==4.4.2 apache-airflow-providers-apache-pig==4.7.2 @@ -125,7 +128,12 @@ apache-airflow-providers-cncf-kubernetes==10.7.0 apache-airflow-providers-cohere==1.5.3 apache-airflow-providers-common-compat==1.7.3 apache-airflow-providers-common-io==1.6.2 -apache-airflow-providers-common-messaging==1.0.5 +# Stackable patch: +# Update from previous version 1.0.5 due to refactoring and compatability with +# apache-airflow-providers-apache-kafka==1.10.4 (see above). +# See https://github.com/apache/airflow/commit/1790ec41eeaff6e33f78b39ad50b2bcd98107e67 +# See https://github.com/apache/airflow/commit/1790ec41eeaff6e33f78b39ad50b2bcd98107e67 +apache-airflow-providers-common-messaging==2.0.0 apache-airflow-providers-common-sql==1.27.5 apache-airflow-providers-databricks==7.7.1 apache-airflow-providers-datadog==3.9.2