Skip to content

Commit 9a3e976

Browse files
kerwin-zkkerwin-zk
committed
[spark] Support AlwaysTrue and AlwaysFalse predicates in SparkV2FilterConverter
1 parent 9e6887a commit 9a3e976

File tree

10 files changed

+541
-6
lines changed

10 files changed

+541
-6
lines changed

‎paimon-common/src/main/java/org/apache/paimon/predicate/FalseFunction.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.predicate;
20+
21+
import org.apache.paimon.types.DataType;
22+
23+
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
24+
25+
import java.util.List;
26+
import java.util.Optional;
27+
28+
/** A {@link LeafFunction} that always returns {@code false}. Used for AlwaysFalse predicates. */
29+
public class FalseFunction extends LeafFunction {
30+
31+
private static final long serialVersionUID = 1L;
32+
33+
public static final String NAME = "FALSE";
34+
35+
public static final FalseFunction INSTANCE = new FalseFunction();
36+
37+
@JsonCreator
38+
private FalseFunction() {}
39+
40+
@Override
41+
public boolean test(DataType type, Object field, List<Object> literals) {
42+
return false;
43+
}
44+
45+
@Override
46+
public boolean test(
47+
DataType type,
48+
long rowCount,
49+
Object min,
50+
Object max,
51+
Long nullCount,
52+
List<Object> literals) {
53+
return false;
54+
}
55+
56+
@Override
57+
public Optional<LeafFunction> negate() {
58+
return Optional.of(TrueFunction.INSTANCE);
59+
}
60+
61+
@Override
62+
public <T> T visit(FunctionVisitor<T> visitor, FieldRef fieldRef, List<Object> literals) {
63+
throw new UnsupportedOperationException(
64+
"FalseFunction does not support field-based visitation.");
65+
}
66+
67+
@Override
68+
public String toJson() {
69+
return NAME;
70+
}
71+
}

‎paimon-common/src/main/java/org/apache/paimon/predicate/LeafFunction.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ private static Map<String, LeafFunction> createRegistry() {
6868
registry.put(NotIn.NAME, NotIn.INSTANCE);
6969
registry.put(Between.NAME, Between.INSTANCE);
7070
registry.put(NotBetween.NAME, NotBetween.INSTANCE);
71+
registry.put(TrueFunction.NAME, TrueFunction.INSTANCE);
72+
registry.put(FalseFunction.NAME, FalseFunction.INSTANCE);
7173
return Collections.unmodifiableMap(registry);
7274
}
7375
}

‎paimon-common/src/main/java/org/apache/paimon/predicate/LeafPredicate.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,9 @@ public boolean test(
143143
long rowCount, InternalRow minValues, InternalRow maxValues, InternalArray nullCounts) {
144144
Optional<FieldRef> fieldRefOptional = fieldRefOptional();
145145
if (!fieldRefOptional.isPresent()) {
146+
if (transform instanceof TrueTransform) {
147+
return function.test(transform.outputType(), 0, null, null, null, literals);
148+
}
146149
return true;
147150
}
148151
FieldRef fieldRef = fieldRefOptional.get();
@@ -165,6 +168,9 @@ public boolean test(
165168

166169
@Override
167170
public Optional<Predicate> negate() {
171+
if (transform instanceof TrueTransform) {
172+
return function.negate().map(neg -> new LeafPredicate(transform, neg, literals));
173+
}
168174
Optional<FieldRef> fieldRefOptional = fieldRefOptional();
169175
if (!fieldRefOptional.isPresent()) {
170176
return Optional.empty();

‎paimon-common/src/main/java/org/apache/paimon/predicate/Transform.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@
3838
@JsonSubTypes.Type(value = ConcatTransform.class, name = ConcatTransform.NAME),
3939
@JsonSubTypes.Type(value = ConcatWsTransform.class, name = ConcatWsTransform.NAME),
4040
@JsonSubTypes.Type(value = UpperTransform.class, name = UpperTransform.NAME),
41-
@JsonSubTypes.Type(value = LowerTransform.class, name = LowerTransform.NAME)
41+
@JsonSubTypes.Type(value = LowerTransform.class, name = LowerTransform.NAME),
42+
@JsonSubTypes.Type(value = TrueTransform.class, name = TrueTransform.NAME)
4243
})
4344
public interface Transform extends Serializable {
4445

‎paimon-common/src/main/java/org/apache/paimon/predicate/TrueFunction.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.predicate;
20+
21+
import org.apache.paimon.types.DataType;
22+
23+
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
24+
25+
import java.util.List;
26+
import java.util.Optional;
27+
28+
/** A {@link LeafFunction} that always returns {@code true}. Used for AlwaysTrue predicates. */
29+
public class TrueFunction extends LeafFunction {
30+
31+
private static final long serialVersionUID = 1L;
32+
33+
public static final String NAME = "TRUE";
34+
35+
public static final TrueFunction INSTANCE = new TrueFunction();
36+
37+
@JsonCreator
38+
private TrueFunction() {}
39+
40+
@Override
41+
public boolean test(DataType type, Object field, List<Object> literals) {
42+
return true;
43+
}
44+
45+
@Override
46+
public boolean test(
47+
DataType type,
48+
long rowCount,
49+
Object min,
50+
Object max,
51+
Long nullCount,
52+
List<Object> literals) {
53+
return true;
54+
}
55+
56+
@Override
57+
public Optional<LeafFunction> negate() {
58+
return Optional.of(FalseFunction.INSTANCE);
59+
}
60+
61+
@Override
62+
public <T> T visit(FunctionVisitor<T> visitor, FieldRef fieldRef, List<Object> literals) {
63+
throw new UnsupportedOperationException(
64+
"TrueFunction does not support field-based visitation.");
65+
}
66+
67+
@Override
68+
public String toJson() {
69+
return NAME;
70+
}
71+
}

‎paimon-common/src/main/java/org/apache/paimon/predicate/TrueTransform.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.predicate;
20+
21+
import org.apache.paimon.data.InternalRow;
22+
import org.apache.paimon.types.DataType;
23+
import org.apache.paimon.types.DataTypes;
24+
25+
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
26+
27+
import java.util.Collections;
28+
import java.util.List;
29+
30+
/** A {@link Transform} that always returns {@code true}. Used for constant predicates. */
31+
public class TrueTransform implements Transform {
32+
33+
private static final long serialVersionUID = 1L;
34+
35+
public static final String NAME = "TRUE";
36+
37+
public static final TrueTransform INSTANCE = new TrueTransform();
38+
39+
@JsonCreator
40+
private TrueTransform() {}
41+
42+
@Override
43+
public String name() {
44+
return NAME;
45+
}
46+
47+
@Override
48+
public List<Object> inputs() {
49+
return Collections.emptyList();
50+
}
51+
52+
@Override
53+
public DataType outputType() {
54+
return DataTypes.BOOLEAN();
55+
}
56+
57+
@Override
58+
public Object transform(InternalRow row) {
59+
return true;
60+
}
61+
62+
@Override
63+
public Transform copyWithNewInputs(List<Object> inputs) {
64+
return INSTANCE;
65+
}
66+
67+
@Override
68+
public boolean equals(Object o) {
69+
if (this == o) {
70+
return true;
71+
}
72+
return o != null && getClass() == o.getClass();
73+
}
74+
75+
@Override
76+
public int hashCode() {
77+
return NAME.hashCode();
78+
}
79+
80+
@Override
81+
public String toString() {
82+
return NAME;
83+
}
84+
}

‎paimon-common/src/test/java/org/apache/paimon/predicate/LeafPredicateTest.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.paimon.predicate;
2020

2121
import org.apache.paimon.data.BinaryString;
22+
import org.apache.paimon.data.GenericArray;
2223
import org.apache.paimon.data.GenericRow;
2324
import org.apache.paimon.types.DataTypes;
2425
import org.apache.paimon.utils.InstantiationUtil;
@@ -27,6 +28,7 @@
2728

2829
import java.io.IOException;
2930
import java.util.ArrayList;
31+
import java.util.Collections;
3032
import java.util.List;
3133

3234
import static org.assertj.core.api.Assertions.assertThat;
@@ -78,4 +80,98 @@ private LeafPredicate create() {
7880
literals.add(BinaryString.fromString("ha-he"));
7981
return LeafPredicate.of(transform, Equal.INSTANCE, literals);
8082
}
83+
84+
@Test
85+
public void testAlwaysTrueRow() {
86+
LeafPredicate predicate =
87+
LeafPredicate.of(
88+
TrueTransform.INSTANCE, TrueFunction.INSTANCE, Collections.emptyList());
89+
assertThat(predicate.test(GenericRow.of(1))).isTrue();
90+
assertThat(predicate.test(GenericRow.of((Object) null))).isTrue();
91+
}
92+
93+
@Test
94+
public void testAlwaysFalseRow() {
95+
LeafPredicate predicate =
96+
LeafPredicate.of(
97+
TrueTransform.INSTANCE, FalseFunction.INSTANCE, Collections.emptyList());
98+
assertThat(predicate.test(GenericRow.of(1))).isFalse();
99+
assertThat(predicate.test(GenericRow.of((Object) null))).isFalse();
100+
}
101+
102+
@Test
103+
public void testAlwaysTrueMinMax() {
104+
LeafPredicate predicate =
105+
LeafPredicate.of(
106+
TrueTransform.INSTANCE, TrueFunction.INSTANCE, Collections.emptyList());
107+
assertThat(
108+
predicate.test(
109+
10,
110+
GenericRow.of(1),
111+
GenericRow.of(10),
112+
new GenericArray(new long[] {0})))
113+
.isTrue();
114+
assertThat(predicate.test(1, null, null, null)).isTrue();
115+
}
116+
117+
@Test
118+
public void testAlwaysFalseMinMax() {
119+
LeafPredicate predicate =
120+
LeafPredicate.of(
121+
TrueTransform.INSTANCE, FalseFunction.INSTANCE, Collections.emptyList());
122+
assertThat(
123+
predicate.test(
124+
10,
125+
GenericRow.of(1),
126+
GenericRow.of(10),
127+
new GenericArray(new long[] {0})))
128+
.isFalse();
129+
assertThat(predicate.test(1, null, null, null)).isFalse();
130+
}
131+
132+
@Test
133+
public void testAlwaysTrueNegate() {
134+
LeafPredicate predicate =
135+
LeafPredicate.of(
136+
TrueTransform.INSTANCE, TrueFunction.INSTANCE, Collections.emptyList());
137+
Predicate negated = predicate.negate().get();
138+
assertThat(negated).isInstanceOf(LeafPredicate.class);
139+
LeafPredicate negatedLeaf = (LeafPredicate) negated;
140+
assertThat(negatedLeaf.function()).isEqualTo(FalseFunction.INSTANCE);
141+
assertThat(negatedLeaf.transform()).isInstanceOf(TrueTransform.class);
142+
assertThat(negatedLeaf.test(GenericRow.of(1))).isFalse();
143+
}
144+
145+
@Test
146+
public void testAlwaysFalseNegate() {
147+
LeafPredicate predicate =
148+
LeafPredicate.of(
149+
TrueTransform.INSTANCE, FalseFunction.INSTANCE, Collections.emptyList());
150+
Predicate negated = predicate.negate().get();
151+
assertThat(negated).isInstanceOf(LeafPredicate.class);
152+
LeafPredicate negatedLeaf = (LeafPredicate) negated;
153+
assertThat(negatedLeaf.function()).isEqualTo(TrueFunction.INSTANCE);
154+
assertThat(negatedLeaf.transform()).isInstanceOf(TrueTransform.class);
155+
assertThat(negatedLeaf.test(GenericRow.of(1))).isTrue();
156+
}
157+
158+
@Test
159+
public void testAlwaysTrueSerialization() throws IOException, ClassNotFoundException {
160+
LeafPredicate predicate =
161+
LeafPredicate.of(
162+
TrueTransform.INSTANCE, TrueFunction.INSTANCE, Collections.emptyList());
163+
LeafPredicate clone = InstantiationUtil.clone(predicate);
164+
assertThat(clone).isEqualTo(predicate);
165+
assertThat(clone.hashCode()).isEqualTo(predicate.hashCode());
166+
}
167+
168+
@Test
169+
public void testAlwaysFalseSerialization() throws IOException, ClassNotFoundException {
170+
LeafPredicate predicate =
171+
LeafPredicate.of(
172+
TrueTransform.INSTANCE, FalseFunction.INSTANCE, Collections.emptyList());
173+
LeafPredicate clone = InstantiationUtil.clone(predicate);
174+
assertThat(clone).isEqualTo(predicate);
175+
assertThat(clone.hashCode()).isEqualTo(predicate.hashCode());
176+
}
81177
}

0 commit comments

Comments
 (0)