Skip to content

Commit 12f0a9a

Browse files
emilymyeemilymye
authored
[release-2.26.0][BEAM-11409] Revert java-versioning of SDK container #12505 
1 parent 1f73b3f commit 12f0a9a

File tree

13 files changed

+86
-104
lines changed

13 files changed

+86
-104
lines changed

‎.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ builder.build {
4646
gradle {
4747
rootBuildScriptDir(properties.checkoutDir)
4848
tasks 'javaExamplesDataflowPreCommit'
49-
switches '-PimageJavaVersion=11'
49+
switches '-Pdockerfile=Dockerfile-java11'
5050
switches '-PdisableSpotlessCheck=true'
5151
switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11
5252
switches '-PcompileAndRunTestsWithJava11'

‎buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1613,7 +1613,7 @@ class BeamModulePlugin implements Plugin<Project> {
16131613
// development image at docker.io (see sdks/CONTAINERS.md):
16141614
//
16151615
// format: apache/beam_$NAME_sdk:latest
1616-
// ie: apache/beam_python3.7_sdk:latest apache/beam_java8_sdk:latest apache/beam_go_sdk:latest
1616+
// ie: apache/beam_python3.7_sdk:latest apache/beam_java_sdk:latest apache/beam_go_sdk:latest
16171617
//
16181618
// Both the root and tag can be defined using properties or explicitly provided.
16191619
project.ext.containerImageName = {

‎release/src/main/scripts/publish_docker_images.sh‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ set -e
2727
DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache
2828
DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_
2929

30-
JAVA_VER=("java8")
3130
PYTHON_VER=("python3.6" "python3.7" "python3.8")
3231
FLINK_VER=("1.8" "1.9" "1.10")
3332

@@ -49,7 +48,7 @@ if [[ $confirmation = "y" ]]; then
4948

5049
echo '-------------------Tagging and Pushing Python images-----------------'
5150
for ver in "${PYTHON_VER[@]}"; do
52-
# Pull verified RC from dockerhub.
51+
# Pull varified RC from dockerhub.
5352
docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
5453

5554
# Tag with ${RELEASE} and push to dockerhub.
@@ -67,23 +66,21 @@ if [[ $confirmation = "y" ]]; then
6766
done
6867

6968
echo '-------------------Tagging and Pushing Java images-----------------'
70-
for ver in "${JAVA_VER[@]}"; do
71-
# Pull verified RC from dockerhub.
72-
docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
69+
# Pull varified RC from dockerhub.
70+
docker pull ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION}
7371

74-
# Tag with ${RELEASE} and push to dockerhub.
75-
docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
76-
docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
72+
# Tag with ${RELEASE} and push to dockerhub.
73+
docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}
74+
docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}
7775

78-
# Tag with latest and push to dockerhub.
79-
docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
80-
docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
76+
# Tag with latest and push to dockerhub.
77+
docker tag ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION} ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:latest
78+
docker push ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:latest
8179

82-
# Cleanup images from local
83-
docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
84-
docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
85-
docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
86-
end
80+
# Cleanup images from local
81+
docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_${RC_VERSION}
82+
docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}
83+
docker rmi -f ${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:latest
8784

8885
echo '-------------Tagging and Pushing Flink job server images-------------'
8986
echo "Publishing images for the following Flink versions:" "${FLINK_VER[@]}"

‎runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java‎

Lines changed: 5 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -75,48 +75,19 @@ public class Environments {
7575
public static final String ENVIRONMENT_EMBEDDED = "EMBEDDED"; // Non Public urn for testing
7676
public static final String ENVIRONMENT_LOOPBACK = "LOOPBACK"; // Non Public urn for testing
7777

78-
public enum JavaVersion {
79-
v8("java8", "1.8"),
80-
v11("java11", "11");
81-
82-
private final String name;
83-
private final String specification;
84-
85-
JavaVersion(final String name, final String specification) {
86-
this.name = name;
87-
this.specification = specification;
88-
}
89-
90-
@Override
91-
public String toString() {
92-
return this.name;
93-
}
94-
95-
public String specification() {
96-
return this.specification;
97-
}
98-
99-
public static JavaVersion forSpecification(String specification) {
100-
for (JavaVersion ver : JavaVersion.values()) {
101-
if (ver.specification.equals(specification)) {
102-
return ver;
103-
}
104-
}
105-
throw new UnsupportedOperationException(
106-
String.format("unsupported Java version: %s", specification));
107-
}
108-
}
109-
11078
/* For development, use the container build by the current user to ensure that the SDK harness and
11179
* the SDK agree on how they should interact. This should be changed to a version-specific
11280
* container during a release.
11381
*
11482
* See https://beam.apache.org/contribute/docker-images/ for more information on how to build a
11583
* container.
11684
*/
117-
11885
private static final String JAVA_SDK_HARNESS_CONTAINER_URL =
119-
getDefaultJavaSdkHarnessContainerUrl();
86+
ReleaseInfo.getReleaseInfo().getDefaultDockerRepoRoot()
87+
+ "/"
88+
+ ReleaseInfo.getReleaseInfo().getDefaultDockerRepoPrefix()
89+
+ "java_sdk:"
90+
+ ReleaseInfo.getReleaseInfo().getSdkVersion();
12091
public static final Environment JAVA_SDK_HARNESS_ENVIRONMENT =
12192
createDockerEnvironment(JAVA_SDK_HARNESS_CONTAINER_URL);
12293

@@ -351,10 +322,6 @@ public static Set<String> getJavaCapabilities() {
351322
return capabilities.build();
352323
}
353324

354-
public static JavaVersion getJavaVersion() {
355-
return JavaVersion.forSpecification(System.getProperty("java.specification.version"));
356-
}
357-
358325
public static String createStagingFileName(File path, HashCode hash) {
359326
String encodedHash = Base64Variants.MODIFIED_FOR_URL.encode(hash.asBytes());
360327
String fileName = Files.getNameWithoutExtension(path.getAbsolutePath());
@@ -393,13 +360,4 @@ private static class ProcessPayloadReferenceJSON {
393360
return env;
394361
}
395362
}
396-
397-
private static String getDefaultJavaSdkHarnessContainerUrl() {
398-
return String.format(
399-
"%s/%s%s_sdk:%s",
400-
ReleaseInfo.getReleaseInfo().getDefaultDockerRepoRoot(),
401-
ReleaseInfo.getReleaseInfo().getDefaultDockerRepoPrefix(),
402-
getJavaVersion().toString(),
403-
ReleaseInfo.getReleaseInfo().getSdkVersion());
404-
}
405363
}

‎runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import static org.hamcrest.Matchers.equalTo;
2121
import static org.hamcrest.Matchers.hasItem;
2222
import static org.hamcrest.Matchers.is;
23-
import static org.junit.Assert.assertEquals;
2423
import static org.junit.Assert.assertThat;
2524

2625
import java.io.IOException;
@@ -34,7 +33,6 @@
3433
import org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload;
3534
import org.apache.beam.model.pipeline.v1.RunnerApi.ProcessPayload;
3635
import org.apache.beam.model.pipeline.v1.RunnerApi.StandardEnvironments;
37-
import org.apache.beam.runners.core.construction.Environments.JavaVersion;
3836
import org.apache.beam.sdk.Pipeline;
3937
import org.apache.beam.sdk.coders.StringUtf8Coder;
4038
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -166,17 +164,4 @@ public void process(ProcessContext ctxt) {}
166164
env1,
167165
equalTo(components.toComponents().getEnvironmentsOrThrow(ptransform.getEnvironmentId())));
168166
}
169-
170-
@Test
171-
public void testJavaVersion() {
172-
assertEquals(JavaVersion.v8, JavaVersion.forSpecification("1.8"));
173-
assertEquals("java8", JavaVersion.v8.toString());
174-
assertEquals(JavaVersion.v11, JavaVersion.forSpecification("11"));
175-
assertEquals("java11", JavaVersion.v11.toString());
176-
}
177-
178-
@Test(expected = UnsupportedOperationException.class)
179-
public void testJavaVersionInvalid() {
180-
assertEquals(JavaVersion.v8, JavaVersion.forSpecification("invalid"));
181-
}
182167
}

‎runners/google-cloud-dataflow-java/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ task buildAndPushDockerContainer() {
243243
dependsOn ":sdks:java:container:docker"
244244
finalizedBy 'cleanUpDockerImages'
245245
def defaultDockerImageName = containerImageName(
246-
name: project.docker_image_default_repo_prefix + "java8_sdk",
246+
name: project.docker_image_default_repo_prefix + "java_sdk",
247247
root: "apache",
248248
tag: project.sdk_version)
249249
doLast {

‎runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -382,16 +382,16 @@ && isServiceEndpoint(dataflowOptions.getDataflowEndpoint())) {
382382
}
383383

384384
// Adding the Java version to the SDK name for user's and support convenience.
385-
String agentJavaVer =
386-
(Environments.getJavaVersion() == Environments.JavaVersion.v8)
387-
? "(JRE 8 environment)"
388-
: "(JDK 11 environment)";
385+
String javaVersion =
386+
Float.parseFloat(System.getProperty("java.specification.version")) >= 9
387+
? "(JDK 11 environment)"
388+
: "(JRE 8 environment)";
389389

390390
DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo();
391391
String userAgent =
392392
String.format(
393393
"%s/%s%s",
394-
dataflowRunnerInfo.getName(), dataflowRunnerInfo.getVersion(), agentJavaVer)
394+
dataflowRunnerInfo.getName(), dataflowRunnerInfo.getVersion(), javaVersion)
395395
.replace(" ", "_");
396396
dataflowOptions.setUserAgent(userAgent);
397397

@@ -2145,9 +2145,8 @@ public Map<PCollection<?>, ReplacementOutput> mapOutputs(
21452145
static String getContainerImageForJob(DataflowPipelineOptions options) {
21462146
String workerHarnessContainerImage = options.getWorkerHarnessContainerImage();
21472147

2148-
Environments.JavaVersion javaVersion = Environments.getJavaVersion();
21492148
String javaVersionId =
2150-
(javaVersion == Environments.JavaVersion.v8) ? "java" : javaVersion.toString();
2149+
Float.parseFloat(System.getProperty("java.specification.version")) >= 9 ? "java11" : "java";
21512150
if (!workerHarnessContainerImage.contains("IMAGE")) {
21522151
return workerHarnessContainerImage;
21532152
} else if (hasExperiment(options, "beam_fn_api")) {

‎sdks/java/container/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717
###############################################################################
18-
ARG java_version
19-
FROM openjdk:${java_version}
18+
19+
FROM openjdk:8
2020
MAINTAINER "Apache Beam <dev@beam.apache.org>"
2121

2222
ARG pull_licenses
@@ -43,7 +43,7 @@ ADD target/third_party_licenses /opt/apache/beam/third_party_licenses/
4343
# copy an extra LICENSE file then remove it.
4444
COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
4545
RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
46-
RUN if [ "${pull_licenses}" = "false" ] ; then \
46+
RUN if [ "$pull_licenses" = "false" ] ; then \
4747
# Remove above license dir if pull licenses false
4848
rm -rf /opt/apache/beam/third_party_licenses ; \
4949
fi

‎sdks/java/container/Dockerfile-java11‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
###############################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
###############################################################################
18+
19+
FROM openjdk:11
20+
MAINTAINER "Apache Beam <dev@beam.apache.org>"
21+
22+
ARG pull_licenses
23+
24+
ADD target/slf4j-api.jar /opt/apache/beam/jars/
25+
ADD target/slf4j-jdk14.jar /opt/apache/beam/jars/
26+
ADD target/beam-sdks-java-harness.jar /opt/apache/beam/jars/
27+
28+
# Required to run cross-language pipelines with KafkaIO
29+
# TODO May be removed once custom environments are supported
30+
ADD target/beam-sdks-java-io-kafka.jar /opt/apache/beam/jars/
31+
ADD target/kafka-clients.jar /opt/apache/beam/jars/
32+
33+
ADD target/linux_amd64/boot /opt/apache/beam/
34+
35+
# Add golang licenses. Because the go-license directory may be empty if
36+
# pull_licenses is false, and COPY fails if there are no files,
37+
# copy an extra LICENSE file then remove it.
38+
COPY target/LICENSE target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/
39+
RUN rm /opt/apache/beam/third_party_licenses/golang/LICENSE
40+
RUN if [ "$pull_licenses" = "false" ] ; then \
41+
# Remove above license dir if pull licenses false
42+
rm -rf /opt/apache/beam/third_party_licenses ; \
43+
fi
44+
45+
ENTRYPOINT ["/opt/apache/beam/boot"]

‎sdks/java/container/build.gradle‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ dependencies {
5454
dockerDependency library.java.kafka_clients
5555
}
5656

57+
def dockerfileName = project.findProperty('dockerfile') ?: 'Dockerfile'
58+
5759
task copyDockerfileDependencies(type: Copy) {
5860
from configurations.dockerDependency
5961
rename "slf4j-api.*", "slf4j-api.jar"
@@ -86,22 +88,18 @@ licenseReport {
8688
renderers = [new JsonReportRenderer()]
8789
}
8890

89-
def imageJavaVersion = project.hasProperty('imageJavaVersion') ? project.findProperty('imageJavaVersion') : '8'
9091
docker {
9192
name containerImageName(
92-
name: "${project.docker_image_default_repo_prefix}java${imageJavaVersion}_sdk",
93+
name: project.docker_image_default_repo_prefix + "java_sdk",
9394
root: project.rootProject.hasProperty(["docker-repository-root"]) ?
9495
project.rootProject["docker-repository-root"] :
9596
project.docker_image_default_repo_root,
9697
tag: project.rootProject.hasProperty(["docker-tag"]) ?
9798
project.rootProject["docker-tag"] : project.sdk_version)
98-
dockerfile project.file("./Dockerfile")
99+
dockerfile project.file("./${dockerfileName}")
99100
files "./build/"
100-
buildArgs([
101-
'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) ||
102-
project.rootProject.hasProperty(["isRelease"]),
103-
'java_version': imageJavaVersion,
104-
])
101+
buildArgs(['pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) ||
102+
project.rootProject.hasProperty(["isRelease"])])
105103
}
106104

107105
task pullLicenses(type: Exec) {

0 commit comments

Comments
 (0)