Skip to content

Commit 63e5f2c

Browse files
AbacnAbacn
committed
Create a PostCommit run validate runner tests on legacy DataSet
1 parent 25a2844 commit 63e5f2c

File tree

4 files changed

+175
-36
lines changed

4 files changed

+175
-36
lines changed

‎.github/workflows/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,7 @@ PostCommit Jobs run in a schedule against master branch and generally do not get
344344
| [ PostCommit Java Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml) | N/A |`beam_PostCommit_Java_Nexmark_Direct.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml?query=event%3Aschedule) |
345345
| [ PostCommit Java Nexmark Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml) | N/A |`beam_PostCommit_Java_Nexmark_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml?query=event%3Aschedule) |
346346
| [ PostCommit Java Nexmark Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml) | N/A |`beam_PostCommit_Java_Nexmark_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml?query=event%3Aschedule) |
347+
| [ PostCommit Java PVR Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml) | N/A |`beam_PostCommit_Java_PVR_Flink_Batch.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml?query=event%3Aschedule) |
347348
| [ PostCommit Java PVR Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml) | N/A |`beam_PostCommit_Java_PVR_Flink_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml?query=event%3Aschedule) |
348349
| [ PostCommit Java PVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml) | N/A |`beam_PostCommit_Java_PVR_Samza.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml?query=event%3Aschedule) |
349350
| [ PostCommit Java SingleStoreIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml) | N/A |`beam_PostCommit_Java_SingleStoreIO_IT.json`| [![.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml?query=event%3Aschedule) |

‎.github/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PostCommit Java PVR Flink Batch
19+
20+
on:
21+
push:
22+
tags: ['v*']
23+
branches: ['master', 'release-*']
24+
paths:
25+
- 'runners/flink/**'
26+
- 'runners/java-fn-execution/**'
27+
- 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
28+
- '.github/workflows/beam_PostCommit_Java_PVR_Flink_Batch.yml'
29+
pull_request_target:
30+
branches: ['master', 'release-*']
31+
paths:
32+
- 'release/trigger_all_tests.json'
33+
- '.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Batch.json'
34+
schedule:
35+
- cron: '15 2/6 * * *'
36+
workflow_dispatch:
37+
38+
# This allows a subsequently queued workflow run to interrupt previous runs
39+
concurrency:
40+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
41+
cancel-in-progress: true
42+
43+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
44+
permissions:
45+
actions: write
46+
pull-requests: write
47+
checks: write
48+
contents: read
49+
deployments: read
50+
id-token: none
51+
issues: write
52+
discussions: read
53+
packages: read
54+
pages: read
55+
repository-projects: read
56+
security-events: read
57+
statuses: read
58+
59+
env:
60+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
61+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
62+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
63+
64+
jobs:
65+
beam_PostCommit_Java_PVR_Flink_Batch:
66+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
67+
strategy:
68+
matrix:
69+
job_name: ["beam_PostCommit_Java_PVR_Flink_Batch"]
70+
job_phrase: ["Run Java_PVR_Flink_Batch PostCommit"]
71+
timeout-minutes: 240
72+
runs-on: [self-hosted, ubuntu-20.04, highmem]
73+
if: |
74+
github.event_name == 'push' ||
75+
github.event_name == 'pull_request_target' ||
76+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
77+
github.event_name == 'workflow_dispatch' ||
78+
github.event.comment.body == 'Run Java_PVR_Flink_Batch PostCommit'
79+
steps:
80+
- uses: actions/checkout@v4
81+
- name: Setup repository
82+
uses: ./.github/actions/setup-action
83+
with:
84+
comment_phrase: ${{ matrix.job_phrase }}
85+
github_token: ${{ secrets.GITHUB_TOKEN }}
86+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
87+
- name: Setup environment
88+
uses: ./.github/actions/setup-environment-action
89+
- name: run validatesPortableRunnerBatch script
90+
uses: ./.github/actions/gradle-command-self-hosted-action
91+
with:
92+
gradle-command: :runners:flink:1.20:job-server:validatesPortableRunnerBatchDataSet
93+
env:
94+
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }}
95+
- name: Archive JUnit Test Results
96+
uses: actions/upload-artifact@v4
97+
if: ${{ !success() }}
98+
with:
99+
name: JUnit Test Results
100+
path: "**/build/reports/tests/"
101+
- name: Upload test report
102+
uses: actions/upload-artifact@v4
103+
with:
104+
name: java-code-coverage-report
105+
path: "**/build/test-results/**/*.xml"
106+
# TODO: Investigate 'Max retries exceeded' issue with EnricoMi/publish-unit-test-result-action@v2.

‎runners/flink/flink_runner.gradle‎

Lines changed: 65 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,16 @@ def copyResourcesOverrides = tasks.register('copyResourcesOverrides', Copy) {
8888
it.duplicatesStrategy DuplicatesStrategy.INCLUDE
8989
}
9090

91-
def copyTestSourceOverrides = tasks.register('copyTestSourceOverrides', Copy) {
92-
it.from test_source_overrides
93-
it.into "${sourceOverridesBase}/test/java"
94-
it.duplicatesStrategy DuplicatesStrategy.INCLUDE
91+
def copyTestSourceOverrides = tasks.register('copyTestSourceOverrides', Copy) { copyTask ->
92+
copyTask.from test_source_overrides
93+
copyTask.into "${sourceOverridesBase}/test/java"
94+
copyTask.duplicatesStrategy DuplicatesStrategy.INCLUDE
95+
96+
if (project.ext.has('excluded_files') && project.ext.excluded_files.containsKey('test')) {
97+
project.ext.excluded_files.test.each { file ->
98+
copyTask.exclude "**/${file}"
99+
}
100+
}
95101
}
96102

97103
def copyTestResourcesOverrides = tasks.register('copyTestResourcesOverrides', Copy) {
@@ -100,45 +106,69 @@ def copyTestResourcesOverrides = tasks.register('copyTestResourcesOverrides', Co
100106
it.duplicatesStrategy DuplicatesStrategy.INCLUDE
101107
}
102108

103-
// add dependency to gradle Java plugin defined tasks
104-
compileJava.dependsOn copySourceOverrides
105-
processResources.dependsOn copyResourcesOverrides
106-
compileTestJava.dependsOn copyTestSourceOverrides
107-
processTestResources.dependsOn copyTestResourcesOverrides
108-
109-
// add dependency BeamModulePlugin defined custom tasks
110-
// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc)
111-
def sourcesJar = project.tasks.findByName('sourcesJar')
112-
if (sourcesJar != null) {
113-
sourcesJar.dependsOn copySourceOverrides
114-
sourcesJar.dependsOn copyResourcesOverrides
115-
}
116-
def testSourcesJar = project.tasks.findByName('testSourcesJar')
117-
if (testSourcesJar != null) {
118-
testSourcesJar.dependsOn copyTestSourceOverrides
119-
testSourcesJar.dependsOn copyTestResourcesOverrides
120-
}
109+
def use_override = (flink_major != all_versions.first())
110+
def sourceBase = "${project.projectDir}/../src"
121111

122-
/*
112+
if (use_override) {
113+
// Copy original+version specific sources to a tmp dir and use it as sourceSet
114+
// add dependency to gradle Java plugin defined tasks
115+
compileJava.dependsOn copySourceOverrides
116+
processResources.dependsOn copyResourcesOverrides
117+
compileTestJava.dependsOn copyTestSourceOverrides
118+
processTestResources.dependsOn copyTestResourcesOverrides
119+
120+
// add dependency BeamModulePlugin defined custom tasks
121+
// they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc)
122+
def sourcesJar = project.tasks.findByName('sourcesJar')
123+
if (sourcesJar != null) {
124+
sourcesJar.dependsOn copySourceOverrides
125+
sourcesJar.dependsOn copyResourcesOverrides
126+
}
127+
def testSourcesJar = project.tasks.findByName('testSourcesJar')
128+
if (testSourcesJar != null) {
129+
testSourcesJar.dependsOn copyTestSourceOverrides
130+
testSourcesJar.dependsOn copyTestResourcesOverrides
131+
}
132+
/*
123133
* We have to explicitly set all directories here to make sure each
124134
* version of Flink has the correct overrides set.
125135
*/
126-
def sourceBase = "${project.projectDir}/../src"
127-
sourceSets {
128-
main {
129-
java {
130-
srcDirs = ["${sourceOverridesBase}/main/java"]
136+
sourceSets {
137+
main {
138+
java {
139+
srcDirs = ["${sourceOverridesBase}/main/java"]
140+
}
141+
resources {
142+
srcDirs = ["${sourceBase}/main/resources", "${sourceOverridesBase}/main/resources"]
143+
}
131144
}
132-
resources {
133-
srcDirs = ["${sourceBase}/main/resources", "${sourceOverridesBase}/main/resources"]
145+
test {
146+
java {
147+
srcDirs = ["${sourceOverridesBase}/test/java"]
148+
}
149+
resources {
150+
srcDirs = ["${sourceBase}/test/resources", "${sourceOverridesBase}/test/resources"]
151+
}
134152
}
135153
}
136-
test {
137-
java {
138-
srcDirs = ["${sourceOverridesBase}/test/java"]
154+
} else {
155+
// Use the original sources directly for the lowest supported Flink version.
156+
sourceSets {
157+
main {
158+
java {
159+
srcDirs = ["${sourceBase}/main/java"]
160+
}
161+
resources {
162+
srcDirs = ["${sourceBase}/main/resources"]
163+
}
139164
}
140-
resources {
141-
srcDirs = ["${sourceBase}/test/resources", "${sourceOverridesBase}/test/resources"]
165+
test {
166+
java {
167+
srcDirs = ["${sourceBase}/test/java"]
168+
}
169+
resources {
170+
srcDirs = ["${sourceBase}/test/resources"]
171+
}
142172
}
143173
}
144174
}

‎runners/flink/job-server/flink_job_server.gradle‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,9 @@ def portableValidatesRunnerTask(String name, String mode, boolean checkpointing,
193193
excludeCategories 'org.apache.beam.sdk.testing.UsesTriggeredSideInputs'
194194
return
195195
}
196-
196+
if (mode == "batch") {
197+
excludeCategories 'org.apache.beam.sdk.testing.UsesTriggeredSideInputs'
198+
}
197199
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
198200
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
199201
excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'

0 commit comments

Comments
 (0)