Skip to content

Commit 6ee520a

Browse files
tarun-googletarun-google
authored
Merge branch 'apache:master' into remote_rt
2 parents 1756bef + ef0a03c commit 6ee520a

File tree

123 files changed

+20083
-218
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

123 files changed

+20083
-218
lines changed

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ github:
5151

5252
protected_branches:
5353
master: {}
54+
release-2.71: {}
5455
release-2.70.0-postrelease: {}
5556
release-2.70: {}
5657
release-2.69.0-postrelease: {}

‎.github/workflows/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,7 @@ PostCommit Jobs run in a schedule against master branch and generally do not get
344344
| [ PostCommit Java Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml) | N/A |`beam_PostCommit_Java_Nexmark_Direct.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml?query=event%3Aschedule) |
345345
| [ PostCommit Java Nexmark Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml) | N/A |`beam_PostCommit_Java_Nexmark_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml?query=event%3Aschedule) |
346346
| [ PostCommit Java Nexmark Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml) | N/A |`beam_PostCommit_Java_Nexmark_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml?query=event%3Aschedule) |
347+
| [ PostCommit Java PVR Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml) | N/A |`beam_PostCommit_Java_PVR_Flink_Batch.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml?query=event%3Aschedule) |
347348
| [ PostCommit Java PVR Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml) | N/A |`beam_PostCommit_Java_PVR_Flink_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml?query=event%3Aschedule) |
348349
| [ PostCommit Java PVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml) | N/A |`beam_PostCommit_Java_PVR_Samza.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml?query=event%3Aschedule) |
349350
| [ PostCommit Java SingleStoreIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml) | N/A |`beam_PostCommit_Java_SingleStoreIO_IT.json`| [![.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml?query=event%3Aschedule) |

‎.github/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PostCommit Java PVR Flink Batch
19+
20+
on:
21+
push:
22+
tags: ['v*']
23+
branches: ['master', 'release-*']
24+
paths:
25+
- 'runners/flink/**'
26+
- 'runners/java-fn-execution/**'
27+
- 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
28+
- '.github/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml'
29+
pull_request_target:
30+
branches: ['master', 'release-*']
31+
paths:
32+
- 'release/trigger_all_tests.json'
33+
- '.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Batch.json'
34+
schedule:
35+
- cron: '15 2/6 * * *'
36+
workflow_dispatch:
37+
38+
# This allows a subsequently queued workflow run to interrupt previous runs
39+
concurrency:
40+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
41+
cancel-in-progress: true
42+
43+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
44+
permissions:
45+
actions: write
46+
pull-requests: write
47+
checks: write
48+
contents: read
49+
deployments: read
50+
id-token: none
51+
issues: write
52+
discussions: read
53+
packages: read
54+
pages: read
55+
repository-projects: read
56+
security-events: read
57+
statuses: read
58+
59+
env:
60+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
61+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
62+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
63+
64+
jobs:
65+
beam_PostCommit_Java_PVR_Flink_Batch:
66+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
67+
strategy:
68+
matrix:
69+
job_name: ["beam_PostCommit_Java_PVR_Flink_Batch"]
70+
job_phrase: ["Run Java_PVR_Flink_Batch PostCommit"]
71+
timeout-minutes: 240
72+
runs-on: [self-hosted, ubuntu-20.04, highmem]
73+
if: |
74+
github.event_name == 'push' ||
75+
github.event_name == 'pull_request_target' ||
76+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
77+
github.event_name == 'workflow_dispatch' ||
78+
github.event.comment.body == 'Run Java_PVR_Flink_Batch PostCommit'
79+
steps:
80+
- uses: actions/checkout@v4
81+
- name: Setup repository
82+
uses: ./.github/actions/setup-action
83+
with:
84+
comment_phrase: ${{ matrix.job_phrase }}
85+
github_token: ${{ secrets.GITHUB_TOKEN }}
86+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
87+
- name: Setup environment
88+
uses: ./.github/actions/setup-environment-action
89+
- name: run validatesPortableRunnerBatch script
90+
uses: ./.github/actions/gradle-command-self-hosted-action
91+
with:
92+
gradle-command: :runners:flink:1.20:job-server:validatesPortableRunnerBatchDataSet
93+
env:
94+
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }}
95+
- name: Archive JUnit Test Results
96+
uses: actions/upload-artifact@v4
97+
if: ${{ !success() }}
98+
with:
99+
name: JUnit Test Results
100+
path: "**/build/reports/tests/"
101+
- name: Upload test report
102+
uses: actions/upload-artifact@v4
103+
with:
104+
name: java-code-coverage-report
105+
path: "**/build/test-results/**/*.xml"
106+
# TODO: Investigate 'Max retries exceeded' issue with EnricoMi/publish-unit-test-result-action@v2.

‎CHANGES.md‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,6 @@
2020
<!--
2121
# [2.XX.X] - Unreleased
2222
23-
## Beam 3.0.0 Development Highlights
24-
25-
* New highly anticipated feature ([X](https://github.com/apache/beam/issues/X)) to address Milestone Y ([#Y](https://github.com/apache/beam/issues/Y)).
26-
2723
## Highlights
2824
2925
* New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)).
@@ -59,7 +55,8 @@
5955
* ([#X](https://github.com/apache/beam/issues/X)).
6056
-->
6157

62-
# [2.71.0] - Unreleased
58+
59+
# [2.72.0] - Unreleased
6360

6461
## Highlights
6562

@@ -68,12 +65,11 @@
6865

6966
## I/Os
7067

71-
* (Java) Elasticsearch 9 Support ([#36491](https://github.com/apache/beam/issues/36491)).
68+
* Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
7269

7370
## New Features / Improvements
7471

75-
* Support configuring Firestore database on ReadFn transforms (Java) ([#36904](https://github.com/apache/beam/issues/36904)).
76-
* (Python) Inference args are now allowed in most model handlers, except where they are explicitly/intentionally disallowed ([#37093](https://github.com/apache/beam/issues/37093)).
72+
* X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
7773

7874
## Breaking Changes
7975

@@ -85,13 +81,36 @@
8581

8682
## Bugfixes
8783

88-
* Fixed FirestoreV1 Beam connectors allow configuring inconsistent project/database IDs between RPC requests and routing headers #36895 (Java) ([#36895](https://github.com/apache/beam/issues/36895)).
89-
Logical type and coder registry are saved for pipelines in the case of default pickler. This fixes a side effect of switching to cloudpickle as default pickler in Beam 2.65.0 (Python) ([#35738](https://github.com/apache/beam/issues/35738)).
84+
* Fixed X (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
85+
86+
## Security Fixes
87+
88+
* Fixed [CVE-YYYY-NNNN](https://www.cve.org/CVERecord?id=CVE-YYYY-NNNN) (Java/Python/Go) ([#X](https://github.com/apache/beam/issues/X)).
9089

9190
## Known Issues
9291

92+
[comment]: # ( When updating known issues after release, make sure also update website blog in website/www/site/content/blog.)
9393
* ([#X](https://github.com/apache/beam/issues/X)).
9494

95+
# [2.71.0] - Unreleased
96+
97+
## I/Os
98+
99+
* (Java) Elasticsearch 9 Support ([#36491](https://github.com/apache/beam/issues/36491)).
100+
* (Java) Upgraded HCatalogIO to Hive 4.0.1 ([#32189](https://github.com/apache/beam/issues/32189)).
101+
102+
## New Features / Improvements
103+
104+
* Support configuring Firestore database on ReadFn transforms (Java) ([#36904](https://github.com/apache/beam/issues/36904)).
105+
* (Python) Inference args are now allowed in most model handlers, except where they are explicitly/intentionally disallowed ([#37093](https://github.com/apache/beam/issues/37093)).
106+
107+
## Bugfixes
108+
109+
* Fixed FirestoreV1 Beam connectors allow configuring inconsistent project/database IDs between RPC requests and routing headers #36895 (Java) ([#36895](https://github.com/apache/beam/issues/36895)).
110+
* Logical type and coder registry are saved for pipelines in the case of default pickler. This fixes a side effect of switching to cloudpickle as default pickler in Beam 2.65.0 (Python) ([#35738](https://github.com/apache/beam/issues/35738)).
111+
112+
## Known Issues
113+
95114
# [2.70.0] - 2025-12-16
96115

97116
## Highlights

‎CONTRIBUTING.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,12 @@ These steps and instructions on getting started are outlined below as well.
4444
- Latest [Go](https://golang.org) 1.x installed.
4545
- [Docker](https://www.docker.com/) installed for some tasks including building worker containers and testing changes to this website locally.
4646
- For SDK Development:
47-
- For manual testing install any version of supported Python version (found [here](gradle.properties)) suffices.
47+
- For manual testing, install any supported Python version (found [here](gradle.properties)) suffices.
4848
- For running test suites, however, you will need Python interpreters for all Python versions supported by Beam.
4949
Interpreters should be installed and available in shell via `python3.x` commands.
5050
For more information, see:
5151
Python installation tips in [Developer Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters).
52-
- For large contributions, a signed [Individual Contributor License.
52+
- For large contributions, a signed [Individual Contributor License
5353
Agreement](https://www.apache.org/licenses/icla.pdf) (ICLA) to the Apache
5454
Software Foundation (ASF).
5555

@@ -112,7 +112,7 @@ To install these in a Debian-based distribution:
112112
113113
###### Automated script for Linux and macOS
114114
115-
You can install these in a Debian-based distribution for Linux or macOs using the [local-env-setup.sh](https://github.com/apache/beam/blob/master/local-env-setup.sh) script, which is part of the Beam repo. It contains:
115+
You can install these in a Debian-based distribution for Linux or macOS using the [local-env-setup.sh](https://github.com/apache/beam/blob/master/local-env-setup.sh) script, which is part of the Beam repo. It contains:
116116
117117
* pip3 packages
118118
* go packages
@@ -285,7 +285,6 @@ If you run into any issues, check out the [contribution FAQ](https://cwiki.apach
285285
If you didn't find the information you were looking for in this guide, please
286286
[reach out to the Beam community](https://beam.apache.org/community/contact-us/).
287287
288-
</div>
289288
290289
## Find Efforts to Contribute to
291290
A great way to contribute is to join an existing effort. If you want to get involved but don’t have a project in mind, check our [list of open starter tasks](https://s.apache.org/beam-starter-tasks).

‎buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,8 @@ class BeamModulePlugin implements Plugin<Project> {
552552
project.ext.currentJavaVersion = getSupportedJavaVersion()
553553

554554
project.ext.allFlinkVersions = project.flink_versions.split(',')
555-
project.ext.latestFlinkVersion = project.ext.allFlinkVersions.last()
555+
// TODO(https://github.com/apache/beam/issues/36947): Move to use project.ext.allFlinkVersions.last() when Flink 2 support completed
556+
project.ext.latestFlinkVersion = '1.20'
556557

557558
project.ext.nativeArchitecture = {
558559
// Best guess as to this system's normalized native architecture name.
@@ -630,7 +631,7 @@ class BeamModulePlugin implements Plugin<Project> {
630631
def jsr305_version = "3.0.2"
631632
def everit_json_version = "1.14.2"
632633
def kafka_version = "2.4.1"
633-
def log4j2_version = "2.20.0"
634+
def log4j2_version = "2.25.3"
634635
def nemo_version = "0.1"
635636
// [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
636637
def netty_version = "4.1.124.Final"

‎gradle.properties‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@ signing.gnupg.useLegacyGpg=true
3030
# buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy.
3131
# To build a custom Beam version make sure you change it in both places, see
3232
# https://github.com/apache/beam/issues/21302.
33-
version=2.71.0-SNAPSHOT
34-
sdk_version=2.71.0.dev
33+
version=2.72.0-SNAPSHOT
34+
sdk_version=2.72.0.dev
3535

3636
javaVersion=1.8
3737

3838
docker_image_default_repo_root=apache
3939
docker_image_default_repo_prefix=beam_
4040

4141
# supported flink versions
42-
flink_versions=1.17,1.18,1.19,1.20
42+
flink_versions=1.17,1.18,1.19,1.20,2.0
4343
# supported python versions
4444
python_versions=3.10,3.11,3.12,3.13

‎runners/core-java/src/main/java/org/apache/beam/runners/core/StateNamespaces.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ public W getWindow() {
102102
return window;
103103
}
104104

105+
public Coder<W> getWindowCoder() {
106+
return windowCoder;
107+
}
108+
105109
@Override
106110
public String stringKey() {
107111
try {
@@ -170,6 +174,10 @@ public W getWindow() {
170174
return window;
171175
}
172176

177+
public Coder<W> getWindowCoder() {
178+
return windowCoder;
179+
}
180+
173181
public int getTriggerIndex() {
174182
return triggerIndex;
175183
}

‎runners/core-java/src/main/java/org/apache/beam/runners/core/StateTags.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,8 @@ private StateTags() {}
144144

145145
private interface SystemStateTag<StateT extends State> {
146146
StateTag<StateT> asKind(StateKind kind);
147+
148+
StateKind getKind();
147149
}
148150

149151
/** Create a state tag for the given id and spec. */
@@ -243,6 +245,16 @@ public static <StateT extends State> StateTag<StateT> makeSystemTagInternal(
243245
return typedTag.asKind(StateKind.SYSTEM);
244246
}
245247

248+
/*
249+
* Returns true if the tag is a system internal tag.
250+
*/
251+
public static <StateT extends State> boolean isSystemTagInternal(StateTag<StateT> tag) {
252+
if (!(tag instanceof SystemStateTag)) {
253+
return false;
254+
}
255+
return StateKind.SYSTEM.equals(((SystemStateTag<?>) tag).getKind());
256+
}
257+
246258
public static <InputT, AccumT, OutputT> StateTag<BagState<AccumT>> convertToBagTagInternal(
247259
StateTag<CombiningState<InputT, AccumT, OutputT>> combiningTag) {
248260
return new SimpleStateTag<>(
@@ -358,6 +370,11 @@ public StateTag<StateT> asKind(StateKind kind) {
358370
return new SimpleStateTag<>(id.asKind(kind), spec);
359371
}
360372

373+
@Override
374+
public StateKind getKind() {
375+
return id.kind;
376+
}
377+
361378
@Override
362379
public boolean equals(@Nullable Object other) {
363380
if (!(other instanceof SimpleStateTag)) {

‎runners/flink/2.0/build.gradle‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
project.ext {
20+
flink_major = '2.0'
21+
flink_version = '2.0.1'
22+
excluded_files = [
23+
'main': [
24+
// Used by DataSet API only
25+
"org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapter.java",
26+
"org/apache/beam/runners/flink/FlinkBatchPipelineTranslator.java",
27+
"org/apache/beam/runners/flink/FlinkBatchPortablePipelineTranslator.java",
28+
"org/apache/beam/runners/flink/FlinkBatchTransformTranslators.java",
29+
"org/apache/beam/runners/flink/translation/functions/FlinkNonMergingReduceFunction.java",
30+
// Moved to org.apache.flink.runtime.state.StateBackendFactory
31+
"org/apache/beam/runners/flink/FlinkStateBackendFactory.java",
32+
],
33+
'test': [
34+
// Used by DataSet API only
35+
"org/apache/beam/runners/flink/adapter/BeamFlinkDataSetAdapterTest.java",
36+
"org/apache/beam/runners/flink/batch/NonMergingGroupByKeyTest.java",
37+
"org/apache/beam/runners/flink/batch/ReshuffleTest.java",
38+
]
39+
]
40+
}
41+
42+
// Load the main build script which contains all build logic.
43+
apply from: "../flink_runner.gradle"

0 commit comments

Comments
 (0)