Skip to content

Commit c1b1d7d

Browse files
dwdwqfwedwdwqfwe
committed
hash
1 parent 9b601b5 commit c1b1d7d

File tree

1 file changed

+236
-4
lines changed
  • fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar

1 file changed

+236
-4
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32Internal.java‎

Lines changed: 236 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,30 +18,43 @@
1818
package org.apache.doris.nereids.trees.expressions.functions.scalar;
1919

2020
import org.apache.doris.catalog.FunctionSignature;
21+
import org.apache.doris.nereids.exceptions.AnalysisException;
2122
import org.apache.doris.nereids.trees.expressions.Expression;
2223
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
2324
import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision;
25+
import org.apache.doris.nereids.trees.expressions.functions.ComputeSignatureHelper;
2426
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
2527
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
2628
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
29+
import org.apache.doris.nereids.types.ArrayType;
2730
import org.apache.doris.nereids.types.BigIntType;
31+
import org.apache.doris.nereids.types.DataType;
32+
import org.apache.doris.nereids.types.MapType;
33+
import org.apache.doris.nereids.types.NullType;
2834
import org.apache.doris.nereids.types.coercion.AnyDataType;
35+
import org.apache.doris.nereids.types.coercion.FollowToAnyDataType;
2936
import org.apache.doris.nereids.util.ExpressionUtils;
3037

3138
import com.google.common.base.Preconditions;
3239
import com.google.common.collect.ImmutableList;
40+
import com.google.common.collect.Lists;
41+
import com.google.common.collect.Maps;
42+
import com.google.common.collect.Sets;
3343

3444
import java.util.List;
45+
import java.util.Map;
46+
import java.util.Optional;
47+
import java.util.Set;
3548

3649
/**
37-
* for debug only, compute crc32 hash value as the same way in `VOlapTablePartitionParam::find_tablets()`
50+
* for debug only, compute crc32 hash value as the same way in
51+
* `VOlapTablePartitionParam::find_tablets()`
3852
*/
3953
public class Crc32Internal extends ScalarFunction
4054
implements UnaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, ComputePrecision {
4155

4256
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
43-
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(AnyDataType.INSTANCE_WITHOUT_INDEX)
44-
);
57+
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(AnyDataType.INSTANCE_WITHOUT_INDEX));
4558

4659
/**
4760
* constructor with 1 or more arguments.
@@ -78,4 +91,223 @@ public FunctionSignature computePrecision(FunctionSignature signature) {
7891
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
7992
return visitor.visitCrc32Internal(this, context);
8093
}
81-
}
94+
95+
/**
96+
* Override computeSignature to skip legacy date type conversion.
97+
* This function needs to preserve the original DateTime/Date types without
98+
* converting to V2 types.
99+
*/
100+
@Override
101+
public FunctionSignature computeSignature(FunctionSignature signature) {
102+
// Manually chain the signature computation steps, skipping date type conversion
103+
signature = implementAnyDataTypeWithOutIndexSkipDateConversion(signature, getArguments());
104+
signature = implementAnyDataTypeWithIndexSkipDateConversion(signature, getArguments());
105+
signature = ComputeSignatureHelper.computePrecision(this, signature, getArguments());
106+
signature = ComputeSignatureHelper.implementFollowToArgumentReturnType(signature, getArguments());
107+
signature = ComputeSignatureHelper.normalizeDecimalV2(signature, getArguments());
108+
signature = ComputeSignatureHelper.ensureNestedNullableOfArray(signature, getArguments());
109+
signature = ComputeSignatureHelper.dynamicComputeVariantArgs(signature, getArguments());
110+
return signature;
111+
}
112+
113+
/**
114+
* Custom implementation of implementAnyDataTypeWithOutIndex that skips date
115+
* type conversion.
116+
*/
117+
private FunctionSignature implementAnyDataTypeWithOutIndexSkipDateConversion(
118+
FunctionSignature signature, List<Expression> arguments) {
119+
List<DataType> newArgTypes = Lists.newArrayListWithCapacity(arguments.size());
120+
for (int i = 0; i < arguments.size(); i++) {
121+
DataType sigType;
122+
if (i >= signature.argumentsTypes.size()) {
123+
sigType = signature.getVarArgType().orElseThrow(
124+
() -> new AnalysisException("function arity not match with signature"));
125+
} else {
126+
sigType = signature.argumentsTypes.get(i);
127+
}
128+
DataType expressionType = arguments.get(i).getDataType();
129+
// Skip date type conversion - directly use replaceAnyDataTypeWithOutIndex
130+
newArgTypes.add(replaceAnyDataTypeWithOutIndex(sigType, expressionType));
131+
}
132+
return signature.withArgumentTypes(signature.hasVarArgs, newArgTypes);
133+
}
134+
135+
/**
136+
* Custom implementation of implementAnyDataTypeWithIndex that skips date type
137+
* conversion.
138+
*/
139+
private FunctionSignature implementAnyDataTypeWithIndexSkipDateConversion(
140+
FunctionSignature signature, List<Expression> arguments) {
141+
// Collect all any data type with index
142+
Map<Integer, List<DataType>> indexToArgumentTypes = Maps.newHashMap();
143+
Map<Integer, Optional<DataType>> indexToCommonTypes = Maps.newHashMap();
144+
145+
for (int i = 0; i < arguments.size(); i++) {
146+
DataType sigType;
147+
if (i >= signature.argumentsTypes.size()) {
148+
sigType = signature.getVarArgType().orElseThrow(
149+
() -> new AnalysisException("function arity not match with signature"));
150+
} else {
151+
sigType = signature.argumentsTypes.get(i);
152+
}
153+
DataType expressionType = arguments.get(i).getDataType();
154+
collectAnyDataType(sigType, expressionType, indexToArgumentTypes);
155+
}
156+
157+
// Handle all null type index
158+
Set<Integer> allNullTypeIndex = Sets.newHashSetWithExpectedSize(indexToArgumentTypes.size());
159+
for (Map.Entry<Integer, List<DataType>> entry : indexToArgumentTypes.entrySet()) {
160+
boolean allIsNullType = entry.getValue().stream().allMatch(dt -> dt instanceof NullType);
161+
if (allIsNullType) {
162+
allNullTypeIndex.add(entry.getKey());
163+
}
164+
}
165+
166+
if (!allNullTypeIndex.isEmpty()) {
167+
for (int i = 0; i < arguments.size(); i++) {
168+
DataType sigType;
169+
if (i >= signature.argumentsTypes.size()) {
170+
sigType = signature.getVarArgType().orElseThrow(
171+
() -> new IllegalStateException("function arity not match with signature"));
172+
} else {
173+
sigType = signature.argumentsTypes.get(i);
174+
}
175+
DataType expressionType = arguments.get(i).getDataType();
176+
collectFollowToAnyDataType(sigType, expressionType, indexToArgumentTypes, allNullTypeIndex);
177+
}
178+
}
179+
180+
// Get common types - reuse ComputeSignatureHelper logic
181+
for (Map.Entry<Integer, List<DataType>> dataTypes : indexToArgumentTypes.entrySet()) {
182+
Optional<DataType> dataType = org.apache.doris.nereids.util.TypeCoercionUtils
183+
.findWiderCommonType(dataTypes.getValue(), false, true);
184+
indexToCommonTypes.put(dataTypes.getKey(), dataType);
185+
}
186+
187+
// Replace any data type - skip date conversion
188+
List<DataType> newArgTypes = Lists.newArrayListWithCapacity(signature.argumentsTypes.size());
189+
for (DataType sigType : signature.argumentsTypes) {
190+
newArgTypes.add(replaceAnyDataType(sigType, indexToCommonTypes));
191+
}
192+
signature = signature.withArgumentTypes(signature.hasVarArgs, newArgTypes);
193+
DataType returnType = replaceAnyDataType(signature.returnType, indexToCommonTypes);
194+
return signature.withReturnType(returnType);
195+
}
196+
197+
// below are copied from ComputeSignatureHelper,the reason to implement here is
198+
// they are static function in original
199+
// Helper methods copied from ComputeSignatureHelper
200+
private static DataType replaceAnyDataTypeWithOutIndex(DataType sigType, DataType expressionType) {
201+
if (expressionType instanceof NullType) {
202+
if (sigType instanceof ArrayType) {
203+
return ArrayType.of(replaceAnyDataTypeWithOutIndex(
204+
((ArrayType) sigType).getItemType(), NullType.INSTANCE));
205+
} else if (sigType instanceof MapType) {
206+
return MapType.of(
207+
replaceAnyDataTypeWithOutIndex(((MapType) sigType).getKeyType(), NullType.INSTANCE),
208+
replaceAnyDataTypeWithOutIndex(((MapType) sigType).getValueType(), NullType.INSTANCE));
209+
} else if (sigType instanceof AnyDataType
210+
&& ((AnyDataType) sigType).getIndex() == AnyDataType.INDEX_OF_INSTANCE_WITHOUT_INDEX) {
211+
return expressionType;
212+
}
213+
return sigType;
214+
} else if (sigType instanceof ArrayType && expressionType instanceof ArrayType) {
215+
return ArrayType.of(replaceAnyDataTypeWithOutIndex(
216+
((ArrayType) sigType).getItemType(), ((ArrayType) expressionType).getItemType()));
217+
} else if (sigType instanceof MapType && expressionType instanceof MapType) {
218+
return MapType.of(
219+
replaceAnyDataTypeWithOutIndex(
220+
((MapType) sigType).getKeyType(), ((MapType) expressionType).getKeyType()),
221+
replaceAnyDataTypeWithOutIndex(
222+
((MapType) sigType).getValueType(), ((MapType) expressionType).getValueType()));
223+
} else if (sigType instanceof AnyDataType
224+
&& ((AnyDataType) sigType).getIndex() == AnyDataType.INDEX_OF_INSTANCE_WITHOUT_INDEX) {
225+
return expressionType;
226+
}
227+
return sigType;
228+
}
229+
230+
private static void collectAnyDataType(DataType sigType, DataType expressionType,
231+
Map<Integer, List<DataType>> indexToArgumentTypes) {
232+
if (expressionType instanceof NullType) {
233+
if (sigType instanceof ArrayType) {
234+
collectAnyDataType(((ArrayType) sigType).getItemType(), NullType.INSTANCE, indexToArgumentTypes);
235+
} else if (sigType instanceof MapType) {
236+
collectAnyDataType(((MapType) sigType).getKeyType(), NullType.INSTANCE, indexToArgumentTypes);
237+
collectAnyDataType(((MapType) sigType).getValueType(), NullType.INSTANCE, indexToArgumentTypes);
238+
} else if (sigType instanceof AnyDataType && ((AnyDataType) sigType).getIndex() >= 0) {
239+
List<DataType> dataTypes = indexToArgumentTypes.computeIfAbsent(
240+
((AnyDataType) sigType).getIndex(), i -> Lists.newArrayList());
241+
dataTypes.add(expressionType);
242+
}
243+
} else if (sigType instanceof ArrayType && expressionType instanceof ArrayType) {
244+
collectAnyDataType(((ArrayType) sigType).getItemType(),
245+
((ArrayType) expressionType).getItemType(), indexToArgumentTypes);
246+
} else if (sigType instanceof MapType && expressionType instanceof MapType) {
247+
collectAnyDataType(((MapType) sigType).getKeyType(),
248+
((MapType) expressionType).getKeyType(), indexToArgumentTypes);
249+
collectAnyDataType(((MapType) sigType).getValueType(),
250+
((MapType) expressionType).getValueType(), indexToArgumentTypes);
251+
} else if (sigType instanceof AnyDataType && ((AnyDataType) sigType).getIndex() >= 0) {
252+
List<DataType> dataTypes = indexToArgumentTypes.computeIfAbsent(
253+
((AnyDataType) sigType).getIndex(), i -> Lists.newArrayList());
254+
dataTypes.add(expressionType);
255+
}
256+
}
257+
258+
private static void collectFollowToAnyDataType(DataType sigType, DataType expressionType,
259+
Map<Integer, List<DataType>> indexToArgumentTypes, Set<Integer> allNullTypeIndex) {
260+
if (expressionType instanceof NullType) {
261+
if (sigType instanceof ArrayType) {
262+
collectFollowToAnyDataType(((ArrayType) sigType).getItemType(),
263+
NullType.INSTANCE, indexToArgumentTypes, allNullTypeIndex);
264+
} else if (sigType instanceof MapType) {
265+
collectFollowToAnyDataType(((MapType) sigType).getKeyType(),
266+
NullType.INSTANCE, indexToArgumentTypes, allNullTypeIndex);
267+
collectFollowToAnyDataType(((MapType) sigType).getValueType(),
268+
NullType.INSTANCE, indexToArgumentTypes, allNullTypeIndex);
269+
} else if (sigType instanceof FollowToAnyDataType
270+
&& allNullTypeIndex.contains(((FollowToAnyDataType) sigType).getIndex())) {
271+
List<DataType> dataTypes = indexToArgumentTypes.computeIfAbsent(
272+
((FollowToAnyDataType) sigType).getIndex(), i -> Lists.newArrayList());
273+
dataTypes.add(expressionType);
274+
}
275+
} else if (sigType instanceof ArrayType && expressionType instanceof ArrayType) {
276+
collectFollowToAnyDataType(((ArrayType) sigType).getItemType(),
277+
((ArrayType) expressionType).getItemType(), indexToArgumentTypes, allNullTypeIndex);
278+
} else if (sigType instanceof MapType && expressionType instanceof MapType) {
279+
collectFollowToAnyDataType(((MapType) sigType).getKeyType(),
280+
((MapType) expressionType).getKeyType(), indexToArgumentTypes, allNullTypeIndex);
281+
collectFollowToAnyDataType(((MapType) sigType).getValueType(),
282+
((MapType) expressionType).getValueType(), indexToArgumentTypes, allNullTypeIndex);
283+
} else if (sigType instanceof FollowToAnyDataType
284+
&& allNullTypeIndex.contains(((FollowToAnyDataType) sigType).getIndex())) {
285+
List<DataType> dataTypes = indexToArgumentTypes.computeIfAbsent(
286+
((FollowToAnyDataType) sigType).getIndex(), i -> Lists.newArrayList());
287+
dataTypes.add(expressionType);
288+
}
289+
}
290+
291+
private static DataType replaceAnyDataType(DataType dataType,
292+
Map<Integer, Optional<DataType>> indexToCommonTypes) {
293+
if (dataType instanceof ArrayType) {
294+
return ArrayType.of(replaceAnyDataType(((ArrayType) dataType).getItemType(), indexToCommonTypes));
295+
} else if (dataType instanceof MapType) {
296+
return MapType.of(
297+
replaceAnyDataType(((MapType) dataType).getKeyType(), indexToCommonTypes),
298+
replaceAnyDataType(((MapType) dataType).getValueType(), indexToCommonTypes));
299+
} else if (dataType instanceof AnyDataType && ((AnyDataType) dataType).getIndex() >= 0) {
300+
Optional<DataType> optionalDataType = indexToCommonTypes.get(((AnyDataType) dataType).getIndex());
301+
if (optionalDataType != null && optionalDataType.isPresent()) {
302+
return optionalDataType.get();
303+
}
304+
} else if (dataType instanceof FollowToAnyDataType) {
305+
Optional<DataType> optionalDataType = indexToCommonTypes.get(
306+
((FollowToAnyDataType) dataType).getIndex());
307+
if (optionalDataType != null && optionalDataType.isPresent()) {
308+
return optionalDataType.get();
309+
}
310+
}
311+
return dataType;
312+
}
313+
}

0 commit comments

Comments
 (0)