Skip to content

Commit 99318f4

Browse files
alex-spiesalex-spies
authored
ESQL: Qualifiers syntax (#132925)
Add initial syntax for qualifiers. This will enable referring to columns from LOOKUP JOIN via qualified names (in a follow-up PR): ... | LOOKUP JOIN lookup_idx AS lookup ON key | WHERE [lookup].[field] > 10 AND 2*([lookup].[field] - [lookup].[otherfield]) < 1 This only makes grammar changes (SNAPSHOT only) and adds qualifiers back into the serialization of attributes.
1 parent 98a73ce commit 99318f4

File tree

56 files changed

+4512
-2995
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+4512
-2995
lines changed

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,7 @@ static TransportVersion def(int id) {
369369
public static final TransportVersion SCRIPT_RESCORER = def(9_143_0_00);
370370
public static final TransportVersion ESQL_LOOKUP_OPERATOR_EMITTED_ROWS = def(9_144_0_00);
371371
public static final TransportVersion ALLOCATION_DECISION_NOT_PREFERRED = def(9_145_0_00);
372+
public static final TransportVersion ESQL_QUALIFIERS_IN_ATTRIBUTES = def(9_146_0_00);
372373

373374
/*
374375
* STOP! READ THIS FIRST! No, really,

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ public DataType dataType() {
125125
public Attribute toAttribute() {
126126
if (lazyAttribute == null) {
127127
lazyAttribute = resolved()
128-
? new ReferenceAttribute(source(), name(), dataType(), nullable(), id(), synthetic())
128+
? new ReferenceAttribute(source(), null, name(), dataType(), nullable(), id(), synthetic())
129129
: new UnresolvedAttribute(source(), name());
130130
}
131131
return lazyAttribute;

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java‎

Lines changed: 78 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,19 @@
66
*/
77
package org.elasticsearch.xpack.esql.core.expression;
88

9+
import org.elasticsearch.TransportVersion;
910
import org.elasticsearch.core.Nullable;
1011
import org.elasticsearch.xpack.esql.core.tree.Source;
1112
import org.elasticsearch.xpack.esql.core.type.DataType;
13+
import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
14+
import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
1215

16+
import java.io.IOException;
1317
import java.util.List;
1418
import java.util.Objects;
1519

1620
import static java.util.Collections.emptyList;
21+
import static org.elasticsearch.TransportVersions.ESQL_QUALIFIERS_IN_ATTRIBUTES;
1722

1823
/**
1924
* {@link Expression}s that can be materialized and describe properties of the derived table.
@@ -33,19 +38,40 @@ public abstract class Attribute extends NamedExpression {
3338
*/
3439
protected static final String SYNTHETIC_ATTRIBUTE_NAME_PREFIX = "$$";
3540

36-
// can the attr be null - typically used in JOINs
41+
// can the attr be null
3742
private final Nullability nullability;
43+
private final String qualifier;
3844

3945
public Attribute(Source source, String name, @Nullable NameId id) {
4046
this(source, name, Nullability.TRUE, id);
4147
}
4248

49+
public Attribute(Source source, @Nullable String qualifier, String name, @Nullable NameId id) {
50+
this(source, qualifier, name, Nullability.TRUE, id);
51+
}
52+
4353
public Attribute(Source source, String name, Nullability nullability, @Nullable NameId id) {
44-
this(source, name, nullability, id, false);
54+
this(source, null, name, nullability, id);
55+
}
56+
57+
public Attribute(Source source, @Nullable String qualifier, String name, Nullability nullability, @Nullable NameId id) {
58+
this(source, qualifier, name, nullability, id, false);
4559
}
4660

4761
public Attribute(Source source, String name, Nullability nullability, @Nullable NameId id, boolean synthetic) {
62+
this(source, null, name, nullability, id, synthetic);
63+
}
64+
65+
public Attribute(
66+
Source source,
67+
@Nullable String qualifier,
68+
String name,
69+
Nullability nullability,
70+
@Nullable NameId id,
71+
boolean synthetic
72+
) {
4873
super(source, name, emptyList(), id, synthetic);
74+
this.qualifier = qualifier;
4975
this.nullability = nullability;
5076
}
5177

@@ -59,6 +85,14 @@ public final Expression replaceChildren(List<Expression> newChildren) {
5985
throw new UnsupportedOperationException("this type of node doesn't have any children to replace");
6086
}
6187

88+
public String qualifier() {
89+
return qualifier;
90+
}
91+
92+
public String qualifiedName() {
93+
return qualifier != null ? "[" + qualifier + "].[" + name() + "]" : name();
94+
}
95+
6296
@Override
6397
public Nullability nullable() {
6498
return nullability;
@@ -70,26 +104,40 @@ public AttributeSet references() {
70104
}
71105

72106
public Attribute withLocation(Source source) {
73-
return Objects.equals(source(), source) ? this : clone(source, name(), dataType(), nullable(), id(), synthetic());
107+
return Objects.equals(source(), source) ? this : clone(source, qualifier(), name(), dataType(), nullable(), id(), synthetic());
108+
}
109+
110+
public Attribute withQualifier(String qualifier) {
111+
return Objects.equals(qualifier, qualifier) ? this : clone(source(), qualifier, name(), dataType(), nullable(), id(), synthetic());
74112
}
75113

76114
public Attribute withName(String name) {
77-
return Objects.equals(name(), name) ? this : clone(source(), name, dataType(), nullable(), id(), synthetic());
115+
return Objects.equals(name(), name) ? this : clone(source(), qualifier(), name, dataType(), nullable(), id(), synthetic());
78116
}
79117

80118
public Attribute withNullability(Nullability nullability) {
81-
return Objects.equals(nullable(), nullability) ? this : clone(source(), name(), dataType(), nullability, id(), synthetic());
119+
return Objects.equals(nullable(), nullability)
120+
? this
121+
: clone(source(), qualifier(), name(), dataType(), nullability, id(), synthetic());
82122
}
83123

84124
public Attribute withId(NameId id) {
85-
return clone(source(), name(), dataType(), nullable(), id, synthetic());
125+
return clone(source(), qualifier(), name(), dataType(), nullable(), id, synthetic());
86126
}
87127

88128
public Attribute withDataType(DataType type) {
89-
return Objects.equals(dataType(), type) ? this : clone(source(), name(), type, nullable(), id(), synthetic());
129+
return Objects.equals(dataType(), type) ? this : clone(source(), qualifier(), name(), type, nullable(), id(), synthetic());
90130
}
91131

92-
protected abstract Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic);
132+
protected abstract Attribute clone(
133+
Source source,
134+
String qualifier,
135+
String name,
136+
DataType type,
137+
Nullability nullability,
138+
NameId id,
139+
boolean synthetic
140+
);
93141

94142
@Override
95143
public Attribute toAttribute() {
@@ -108,24 +156,24 @@ public boolean semanticEquals(Expression other) {
108156

109157
@Override
110158
protected Expression canonicalize() {
111-
return clone(Source.EMPTY, name(), dataType(), nullability, id(), synthetic());
159+
return clone(Source.EMPTY, qualifier(), name(), dataType(), nullability, id(), synthetic());
112160
}
113161

114162
@Override
115163
@SuppressWarnings("checkstyle:EqualsHashCode")// equals is implemented in parent. See innerEquals instead
116164
public int hashCode() {
117-
return Objects.hash(super.hashCode(), nullability);
165+
return Objects.hash(super.hashCode(), qualifier, nullability);
118166
}
119167

120168
@Override
121169
protected boolean innerEquals(Object o) {
122170
var other = (Attribute) o;
123-
return super.innerEquals(other) && Objects.equals(nullability, other.nullability);
171+
return super.innerEquals(other) && Objects.equals(qualifier, other.qualifier) && Objects.equals(nullability, other.nullability);
124172
}
125173

126174
@Override
127175
public String toString() {
128-
return name() + "{" + label() + (synthetic() ? "$" : "") + "}" + "#" + id();
176+
return qualifiedName() + "{" + label() + (synthetic() ? "$" : "") + "}" + "#" + id();
129177
}
130178

131179
@Override
@@ -154,4 +202,22 @@ public static boolean dataTypeEquals(List<Attribute> left, List<Attribute> right
154202
* @return true if the attribute represents a TSDB dimension type
155203
*/
156204
public abstract boolean isDimension();
205+
206+
protected void checkAndSerializeQualifier(PlanStreamOutput out, TransportVersion version) throws IOException {
207+
if (version.onOrAfter(ESQL_QUALIFIERS_IN_ATTRIBUTES)) {
208+
out.writeOptionalCachedString(qualifier());
209+
} else if (qualifier() != null) {
210+
// Non-null qualifier means the query specifically defined one. Old nodes don't know what to do with it and just writing
211+
// null would lose information and lead to undefined, likely invalid queries.
212+
// IllegalArgumentException returns a 400 to the user, which is what we want here.
213+
throw new IllegalArgumentException("Trying to serialize an Attribute with a qualifier to an old node");
214+
}
215+
}
216+
217+
protected static String readQualifier(PlanStreamInput in, TransportVersion version) throws IOException {
218+
if (version.onOrAfter(ESQL_QUALIFIERS_IN_ATTRIBUTES)) {
219+
return in.readOptionalCachedString();
220+
}
221+
return null;
222+
}
157223
}

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,15 @@ public String getWriteableName() {
3535
}
3636

3737
@Override
38-
protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
38+
protected Attribute clone(
39+
Source source,
40+
String qualifier,
41+
String name,
42+
DataType type,
43+
Nullability nullability,
44+
NameId id,
45+
boolean synthetic
46+
) {
3947
return this;
4048
}
4149

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,6 @@ public final class Expressions {
2020

2121
private Expressions() {}
2222

23-
public static NamedExpression wrapAsNamed(Expression exp) {
24-
return exp instanceof NamedExpression ne ? ne : new Alias(exp.source(), exp.sourceText(), exp);
25-
}
26-
2723
public static List<Attribute> asAttributes(List<? extends NamedExpression> named) {
2824
if (named.isEmpty()) {
2925
return emptyList();
@@ -109,7 +105,9 @@ public static AttributeSet references(List<? extends Expression> exps) {
109105
}
110106

111107
public static String name(Expression e) {
112-
return e instanceof NamedExpression ne ? ne.name() : e.sourceText();
108+
return e instanceof Attribute attr && attr.qualifier() != null ? attr.qualifiedName()
109+
: e instanceof NamedExpression ne ? ne.name()
110+
: e.sourceText();
113111
}
114112

115113
/**

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -55,28 +55,40 @@ public record FieldName(String string) {};
5555
private final EsField field;
5656
protected FieldName lazyFieldName;
5757

58+
@Deprecated
59+
/**
60+
* For testing only
61+
*/
5862
public FieldAttribute(Source source, String name, EsField field) {
59-
this(source, null, name, field);
63+
this(source, null, null, name, field, Nullability.TRUE, null, false);
6064
}
6165

62-
public FieldAttribute(Source source, @Nullable String parentName, String name, EsField field) {
63-
this(source, parentName, name, field, Nullability.TRUE, null, false);
66+
public FieldAttribute(
67+
Source source,
68+
@Nullable String parentName,
69+
@Nullable String qualifier,
70+
String name,
71+
EsField field,
72+
boolean synthetic
73+
) {
74+
this(source, parentName, qualifier, name, field, Nullability.TRUE, null, synthetic);
6475
}
6576

66-
public FieldAttribute(Source source, @Nullable String parentName, String name, EsField field, boolean synthetic) {
67-
this(source, parentName, name, field, Nullability.TRUE, null, synthetic);
77+
public FieldAttribute(Source source, @Nullable String parentName, @Nullable String qualifier, String name, EsField field) {
78+
this(source, parentName, qualifier, name, field, Nullability.TRUE, null, false);
6879
}
6980

7081
public FieldAttribute(
7182
Source source,
7283
@Nullable String parentName,
84+
@Nullable String qualifier,
7385
String name,
7486
EsField field,
7587
Nullability nullability,
7688
@Nullable NameId id,
7789
boolean synthetic
7890
) {
79-
super(source, name, field.getDataType(), nullability, id, synthetic);
91+
super(source, qualifier, name, field.getDataType(), nullability, id, synthetic);
8092
this.parentName = parentName;
8193
this.field = field;
8294
}
@@ -92,6 +104,7 @@ private static FieldAttribute innerReadFrom(StreamInput in) throws IOException {
92104
*/
93105
Source source = Source.readFrom((StreamInput & PlanStreamInput) in);
94106
String parentName = ((PlanStreamInput) in).readOptionalCachedString();
107+
String qualifier = readQualifier((PlanStreamInput) in, in.getTransportVersion());
95108
String name = readCachedStringWithVersionCheck(in);
96109
if (in.getTransportVersion().before(ESQL_FIELD_ATTRIBUTE_DROP_TYPE)) {
97110
DataType.readFrom(in);
@@ -103,21 +116,22 @@ private static FieldAttribute innerReadFrom(StreamInput in) throws IOException {
103116
Nullability nullability = in.readEnum(Nullability.class);
104117
NameId nameId = NameId.readFrom((StreamInput & PlanStreamInput) in);
105118
boolean synthetic = in.readBoolean();
106-
return new FieldAttribute(source, parentName, name, field, nullability, nameId, synthetic);
119+
return new FieldAttribute(source, parentName, qualifier, name, field, nullability, nameId, synthetic);
107120
}
108121

109122
@Override
110123
public void writeTo(StreamOutput out) throws IOException {
111124
if (((PlanStreamOutput) out).writeAttributeCacheHeader(this)) {
112125
Source.EMPTY.writeTo(out);
113126
((PlanStreamOutput) out).writeOptionalCachedString(parentName);
127+
checkAndSerializeQualifier((PlanStreamOutput) out, out.getTransportVersion());
114128
writeCachedStringWithVersionCheck(out, name());
115129
if (out.getTransportVersion().before(ESQL_FIELD_ATTRIBUTE_DROP_TYPE)) {
116130
dataType().writeTo(out);
117131
}
118132
field.writeTo(out);
119133
if (out.getTransportVersion().before(ESQL_FIELD_ATTRIBUTE_DROP_TYPE)) {
120-
// We used to write the qualifier here. We can still do if needed in the future.
134+
// We used to write the qualifier here, even though it was always null.
121135
out.writeOptionalString(null);
122136
}
123137
out.writeEnum(nullable());
@@ -137,7 +151,7 @@ public String getWriteableName() {
137151

138152
@Override
139153
protected NodeInfo<FieldAttribute> info() {
140-
return NodeInfo.create(this, FieldAttribute::new, parentName, name(), field, nullable(), id(), synthetic());
154+
return NodeInfo.create(this, FieldAttribute::new, parentName, qualifier(), name(), field, nullable(), id(), synthetic());
141155
}
142156

143157
public String parentName() {
@@ -185,13 +199,30 @@ public FieldAttribute exactAttribute() {
185199
}
186200

187201
private FieldAttribute innerField(EsField type) {
188-
return new FieldAttribute(source(), fieldName().string, name() + "." + type.getName(), type, nullable(), id(), synthetic());
202+
return new FieldAttribute(
203+
source(),
204+
fieldName().string,
205+
qualifier(),
206+
name() + "." + type.getName(),
207+
type,
208+
nullable(),
209+
id(),
210+
synthetic()
211+
);
189212
}
190213

191214
@Override
192-
protected Attribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
215+
protected Attribute clone(
216+
Source source,
217+
String qualifier,
218+
String name,
219+
DataType type,
220+
Nullability nullability,
221+
NameId id,
222+
boolean synthetic
223+
) {
193224
// Ignore `type`, this must be the same as the field's type.
194-
return new FieldAttribute(source, parentName, name, field, nullability, id, synthetic);
225+
return new FieldAttribute(source, parentName, qualifier, name, field, nullability, id, synthetic);
195226
}
196227

197228
@Override

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,16 @@ public String getWriteableName() {
111111
}
112112

113113
@Override
114-
protected MetadataAttribute clone(Source source, String name, DataType type, Nullability nullability, NameId id, boolean synthetic) {
114+
protected MetadataAttribute clone(
115+
Source source,
116+
String qualifier,
117+
String name,
118+
DataType type,
119+
Nullability nullability,
120+
NameId id,
121+
boolean synthetic
122+
) {
123+
// Ignores qualifier, as metadata attributes do not have qualifiers.
115124
return new MetadataAttribute(source, name, type, nullability, id, synthetic, searchable);
116125
}
117126

0 commit comments

Comments
 (0)