Skip to content

Commit 17030be

Browse files
csviricsviri
committed
crd checker and changed e2e test
Signed-off-by: Attila Mészáros <[email protected]>
1 parent 447ea75 commit 17030be

File tree

7 files changed

+123
-78
lines changed

7 files changed

+123
-78
lines changed

‎e2e-tests/data/autoscaler.yaml‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,18 @@ spec:
3131
nginx.ingress.kubernetes.io/rewrite-target: "/$2"
3232
flinkConfiguration:
3333
taskmanager.numberOfTaskSlots: "2"
34-
high-availability.type: kubernetes
35-
high-availability.storageDir: file:///opt/flink/volume/flink-ha
36-
state.checkpoints.dir: file:///opt/flink/volume/flink-cp
37-
state.savepoints.dir: file:///opt/flink/volume/flink-sp
38-
39-
job.autoscaler.enabled: "true"
40-
job.autoscaler.scaling.enabled: "true"
41-
job.autoscaler.stabilization.interval: "5s"
42-
job.autoscaler.metrics.window: "1m"
43-
job.autoscaler.scale-down.interval: "0m"
34+
high-availability:
35+
type: kubernetes
36+
storageDir: file:///opt/flink/volume/flink-ha
37+
state:
38+
checkpoints.dir: file:///opt/flink/volume/flink-cp
39+
savepoints.dir: file:///opt/flink/volume/flink-sp
40+
job.autoscaler:
41+
enabled: "true"
42+
scaling.enabled: "true"
43+
stabilization.interval: "5s"
44+
metrics.window: "1m"
45+
scale-down.interval: "0m"
4446

4547
# Invalid Validations for testing autoscaler configurations
4648
# kubernetes.operator.job.autoscaler.scale-down.max-factor: "-0.6"

‎e2e-tests/data/flinkdep-cr.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ spec:
3131
nginx.ingress.kubernetes.io/rewrite-target: "/$2"
3232
flinkConfiguration:
3333
taskmanager.numberOfTaskSlots: "2"
34-
high-availability.type: kubernetes
35-
high-availability.storageDir: file:///opt/flink/volume/flink-ha
36-
state.checkpoints.dir: file:///opt/flink/volume/flink-cp
37-
state.savepoints.dir: file:///opt/flink/volume/flink-sp
34+
high-availability:
35+
type: kubernetes
36+
storageDir: file:///opt/flink/volume/flink-ha
37+
state:
38+
checkpoints.dir: file:///opt/flink/volume/flink-cp
39+
savepoints.dir: file:///opt/flink/volume/flink-sp
3840
kubernetes.operator.snapshot.resource.enabled: "false"
3941
serviceAccount: flink
4042
podTemplate:

‎e2e-tests/data/multi-sessionjob.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ spec:
3131
nginx.ingress.kubernetes.io/rewrite-target: "/$2"
3232
flinkConfiguration:
3333
taskmanager.numberOfTaskSlots: "2"
34-
high-availability.type: kubernetes
35-
high-availability.storageDir: file:///opt/flink/volume/flink-ha
36-
state.checkpoints.dir: file:///opt/flink/volume/flink-cp
37-
state.savepoints.dir: file:///opt/flink/volume/flink-sp
34+
high-availability:
35+
type: kubernetes
36+
storageDir: file:///opt/flink/volume/flink-ha
37+
state:
38+
checkpoints.dir: file:///opt/flink/volume/flink-cp
39+
savepoints.dir: file:///opt/flink/volume/flink-sp
3840
serviceAccount: flink
3941
podTemplate:
4042
spec:

‎e2e-tests/data/sessionjob-cr.yaml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@ spec:
3131
nginx.ingress.kubernetes.io/rewrite-target: "/$2"
3232
flinkConfiguration:
3333
taskmanager.numberOfTaskSlots: "2"
34-
high-availability.type: kubernetes
35-
high-availability.storageDir: file:///opt/flink/volume/flink-ha
36-
state.checkpoints.dir: file:///opt/flink/volume/flink-cp
37-
state.savepoints.dir: file:///opt/flink/volume/flink-sp
34+
high-availability:
35+
type: kubernetes
36+
storageDir: file:///opt/flink/volume/flink-ha
37+
state:
38+
checkpoints.dir: file:///opt/flink/volume/flink-cp
39+
savepoints.dir: file:///opt/flink/volume/flink-sp
3840
kubernetes.operator.snapshot.resource.enabled: "false"
3941
serviceAccount: flink
4042
podTemplate:

‎flink-kubernetes-operator-api/pom.xml‎

Lines changed: 56 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -254,62 +254,62 @@ under the License.
254254
</execution>
255255
</executions>
256256
</plugin>
257-
<!-- <plugin>-->
258-
<!-- <artifactId>maven-antrun-plugin</artifactId>-->
259-
<!-- <executions>-->
260-
<!-- <execution>-->
261-
<!-- <id>deployment-crd-compatibility-check</id>-->
262-
<!-- <phase>package</phase>-->
263-
<!-- <goals>-->
264-
<!-- <goal>run</goal>-->
265-
<!-- </goals>-->
266-
<!-- <configuration>-->
267-
<!-- <target>-->
268-
<!-- <java classname="org.apache.flink.kubernetes.operator.api.validation.CrdCompatibilityChecker"-->
269-
<!-- fork="true" failonerror="true">-->
270-
<!-- <classpath refid="maven.compile.classpath"/>-->
271-
<!-- <arg value="file://${rootDir}/helm/flink-kubernetes-operator/crds/flinkdeployments.flink.apache.org-v1.yml"/>-->
272-
<!-- <arg value="https://raw.githubusercontent.com/apache/flink-kubernetes-operator/release-1.9.0/helm/flink-kubernetes-operator/crds/flinkdeployments.flink.apache.org-v1.yml"/>-->
273-
<!-- </java>-->
274-
<!-- </target>-->
275-
<!-- </configuration>-->
276-
<!-- </execution>-->
277-
<!-- <execution>-->
278-
<!-- <id>sessionjob-crd-compatibility-check</id>-->
279-
<!-- <phase>package</phase>-->
280-
<!-- <goals>-->
281-
<!-- <goal>run</goal>-->
282-
<!-- </goals>-->
283-
<!-- <configuration>-->
284-
<!-- <target>-->
285-
<!-- <java classname="org.apache.flink.kubernetes.operator.api.validation.CrdCompatibilityChecker"-->
286-
<!-- fork="true" failonerror="true">-->
287-
<!-- <classpath refid="maven.compile.classpath"/>-->
288-
<!-- <arg value="file://${rootDir}/helm/flink-kubernetes-operator/crds/flinksessionjobs.flink.apache.org-v1.yml"/>-->
289-
<!-- <arg value="https://raw.githubusercontent.com/apache/flink-kubernetes-operator/release-1.9.0/helm/flink-kubernetes-operator/crds/flinksessionjobs.flink.apache.org-v1.yml"/>-->
290-
<!-- </java>-->
291-
<!-- </target>-->
292-
<!-- </configuration>-->
293-
<!-- </execution>-->
294-
<!-- <execution>-->
295-
<!-- <id>statesnapshot-crd-compatibility-check</id>-->
296-
<!-- <phase>package</phase>-->
297-
<!-- <goals>-->
298-
<!-- <goal>run</goal>-->
299-
<!-- </goals>-->
300-
<!-- <configuration>-->
301-
<!-- <target>-->
302-
<!-- <java classname="org.apache.flink.kubernetes.operator.api.validation.CrdCompatibilityChecker"-->
303-
<!-- fork="true" failonerror="true">-->
304-
<!-- <classpath refid="maven.compile.classpath"/>-->
305-
<!-- <arg value="file://${rootDir}/helm/flink-kubernetes-operator/crds/flinkstatesnapshots.flink.apache.org-v1.yml"/>-->
306-
<!-- <arg value="https://raw.githubusercontent.com/apache/flink-kubernetes-operator/release-1.10.0/helm/flink-kubernetes-operator/crds/flinkstatesnapshots.flink.apache.org-v1.yml"/>-->
307-
<!-- </java>-->
308-
<!-- </target>-->
309-
<!-- </configuration>-->
310-
<!-- </execution>-->
311-
<!-- </executions>-->
312-
<!-- </plugin>-->
257+
<plugin>
258+
<artifactId>maven-antrun-plugin</artifactId>
259+
<executions>
260+
<execution>
261+
<id>deployment-crd-compatibility-check</id>
262+
<phase>package</phase>
263+
<goals>
264+
<goal>run</goal>
265+
</goals>
266+
<configuration>
267+
<target>
268+
<java classname="org.apache.flink.kubernetes.operator.api.validation.CrdCompatibilityChecker"
269+
fork="true" failonerror="true">
270+
<classpath refid="maven.compile.classpath"/>
271+
<arg value="file://${rootDir}/helm/flink-kubernetes-operator/crds/flinkdeployments.flink.apache.org-v1.yml"/>
272+
<arg value="https://raw.githubusercontent.com/apache/flink-kubernetes-operator/release-1.9.0/helm/flink-kubernetes-operator/crds/flinkdeployments.flink.apache.org-v1.yml"/>
273+
</java>
274+
</target>
275+
</configuration>
276+
</execution>
277+
<execution>
278+
<id>sessionjob-crd-compatibility-check</id>
279+
<phase>package</phase>
280+
<goals>
281+
<goal>run</goal>
282+
</goals>
283+
<configuration>
284+
<target>
285+
<java classname="org.apache.flink.kubernetes.operator.api.validation.CrdCompatibilityChecker"
286+
fork="true" failonerror="true">
287+
<classpath refid="maven.compile.classpath"/>
288+
<arg value="file://${rootDir}/helm/flink-kubernetes-operator/crds/flinksessionjobs.flink.apache.org-v1.yml"/>
289+
<arg value="https://raw.githubusercontent.com/apache/flink-kubernetes-operator/release-1.9.0/helm/flink-kubernetes-operator/crds/flinksessionjobs.flink.apache.org-v1.yml"/>
290+
</java>
291+
</target>
292+
</configuration>
293+
</execution>
294+
<execution>
295+
<id>statesnapshot-crd-compatibility-check</id>
296+
<phase>package</phase>
297+
<goals>
298+
<goal>run</goal>
299+
</goals>
300+
<configuration>
301+
<target>
302+
<java classname="org.apache.flink.kubernetes.operator.api.validation.CrdCompatibilityChecker"
303+
fork="true" failonerror="true">
304+
<classpath refid="maven.compile.classpath"/>
305+
<arg value="file://${rootDir}/helm/flink-kubernetes-operator/crds/flinkstatesnapshots.flink.apache.org-v1.yml"/>
306+
<arg value="https://raw.githubusercontent.com/apache/flink-kubernetes-operator/release-1.10.0/helm/flink-kubernetes-operator/crds/flinkstatesnapshots.flink.apache.org-v1.yml"/>
307+
</java>
308+
</target>
309+
</configuration>
310+
</execution>
311+
</executions>
312+
</plugin>
313313
<plugin>
314314
<groupId>org.apache.maven.plugins</groupId>
315315
<artifactId>maven-jar-plugin</artifactId>

‎flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/validation/CrdCompatibilityChecker.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,16 @@ protected static void checkTypeCompatibility(String path, JsonNode oldNode, Json
125125
}
126126
}
127127

128+
if (oldNode.get("type") == null && newNode.get("type") != null) {
129+
err("Type mismatch for " + path + ". Old node type is null, while new node is not");
130+
}
131+
if (oldNode.get("type") != null && newNode.get("type") == null) {
132+
if (isGeneralizingAdditionalPropertiesForYaml(oldNode, newNode)) {
133+
return;
134+
}
135+
err("Type mismatch for " + path + ". Old node type is not null, while new node null");
136+
}
137+
128138
String oldType = oldNode.get("type").asText();
129139

130140
if (!oldType.equals(newNode.get("type").asText())) {
@@ -149,6 +159,16 @@ protected static void checkTypeCompatibility(String path, JsonNode oldNode, Json
149159
}
150160
}
151161

162+
private static boolean isGeneralizingAdditionalPropertiesForYaml(
163+
JsonNode oldNode, JsonNode newNode) {
164+
var oldAdditionalProperties = oldNode.get("additionalProperties");
165+
166+
return oldAdditionalProperties != null
167+
&& "object".equals(oldNode.get("type").asText())
168+
&& "string".equals(oldAdditionalProperties.get("type").asText())
169+
&& "true".equals(newNode.get("x-kubernetes-preserve-unknown-fields").asText());
170+
}
171+
152172
protected static void verifyOtherPropsMatch(String path, JsonNode oldNode, JsonNode newNode) {
153173
var oldCopy = (ObjectNode) oldNode.deepCopy();
154174
var newCopy = (ObjectNode) newNode.deepCopy();

‎flink-kubernetes-operator-api/src/test/java/org/apache/flink/kubernetes/operator/api/validation/CrdCompatibilityCheckerTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,23 @@ public void testOtherPropertyMismatch() throws Exception {
247247
+ " type: object");
248248
}
249249

250+
@Test
251+
void generalizingPropertyMapToAnyType() throws Exception {
252+
expectSuccess(
253+
"openAPIV3Schema:\n"
254+
+ " properties:\n"
255+
+ " flinkConfiguration:\n"
256+
+ " additionalProperties:\n"
257+
+ " type: string\n"
258+
+ " type: object\n"
259+
+ " type: object",
260+
"openAPIV3Schema:\n"
261+
+ " properties:\n"
262+
+ " flinkConfiguration:\n"
263+
+ " x-kubernetes-preserve-unknown-fields: true\n"
264+
+ " type: object");
265+
}
266+
250267
@Test
251268
public void testCreateFlinkSessionJobIgnoreUnknownFields() throws IOException {
252269
FlinkSessionJob flinkSessionJobWithUnknownFields =

0 commit comments

Comments
 (0)