Skip to content

Commit 4ef4152

Browse files
authored
feat(plugin-memory): Add support for materialized views in memory connector (#26405)
## Description Per title. Also, REFRESH with WHERE clause does not make sense for memory connector (which only has unpartitioned tables), so making it optional with a new session property. ## Motivation and Context Add tests for materialized views ## Impact N/A ## Test Plan Included tests. ## Contributor checklist - [ ] Please make sure your submission complies with our [contributing guide](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md), in particular [code style](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#code-style) and [commit standards](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#commit-standards). - [ ] PR description addresses the issue accurately and concisely. If the change is non-trivial, a GitHub Issue is referenced. - [ ] Documented new properties (with its default value), SQL syntax, functions, or other functionality. - [ ] If release notes are required, they follow the [release notes guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines). - [ ] Adequate tests were added if applicable. - [ ] CI passed. - [ ] If adding new dependencies, verified they have an [OpenSSF Scorecard](https://securityscorecards.dev/#the-checks) score of 5.0 or higher (or obtained explicit TSC approval for lower scores). ## Release Notes ``` == NO RELEASE NOTE == ```
1 parent ee5e263 commit 4ef4152

File tree

10 files changed

+879
-13
lines changed

10 files changed

+879
-13
lines changed

presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,7 @@ public final class SystemSessionProperties
244244
public static final String MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED = "materialized_view_data_consistency_enabled";
245245
public static final String CONSIDER_QUERY_FILTERS_FOR_MATERIALIZED_VIEW_PARTITIONS = "consider-query-filters-for-materialized-view-partitions";
246246
public static final String QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED = "query_optimization_with_materialized_view_enabled";
247+
public static final String LEGACY_MATERIALIZED_VIEWS = "legacy_materialized_views";
247248
public static final String AGGREGATION_IF_TO_FILTER_REWRITE_STRATEGY = "aggregation_if_to_filter_rewrite_strategy";
248249
public static final String JOINS_NOT_NULL_INFERENCE_STRATEGY = "joins_not_null_inference_strategy";
249250
public static final String RESOURCE_AWARE_SCHEDULING_STRATEGY = "resource_aware_scheduling_strategy";
@@ -1353,6 +1354,12 @@ public SystemSessionProperties(
13531354
"Enable query optimization with materialized view",
13541355
featuresConfig.isQueryOptimizationWithMaterializedViewEnabled(),
13551356
true),
1357+
booleanProperty(
1358+
LEGACY_MATERIALIZED_VIEWS,
1359+
"Experimental: Use legacy materialized views. This feature is under active development and may change" +
1360+
"or be removed at any time. Do not disable in production environments.",
1361+
featuresConfig.isLegacyMaterializedViews(),
1362+
true),
13561363
stringProperty(
13571364
DISTRIBUTED_TRACING_MODE,
13581365
"Mode for distributed tracing. NO_TRACE, ALWAYS_TRACE, or SAMPLE_BASED",
@@ -2882,6 +2889,11 @@ public static boolean isQueryOptimizationWithMaterializedViewEnabled(Session ses
28822889
return session.getSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, Boolean.class);
28832890
}
28842891

2892+
public static boolean isLegacyMaterializedViews(Session session)
2893+
{
2894+
return session.getSystemProperty(LEGACY_MATERIALIZED_VIEWS, Boolean.class);
2895+
}
2896+
28852897
public static boolean isVerboseRuntimeStatsEnabled(Session session)
28862898
{
28872899
return session.getSystemProperty(VERBOSE_RUNTIME_STATS_ENABLED, Boolean.class);

presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.facebook.presto.common.predicate.TupleDomain;
2020
import com.facebook.presto.metadata.Metadata;
2121
import com.facebook.presto.metadata.SessionPropertyManager;
22+
import com.facebook.presto.spi.ConnectorId;
2223
import com.facebook.presto.spi.MaterializedViewStatus;
2324
import com.facebook.presto.spi.SchemaTableName;
2425
import com.facebook.presto.spi.relation.DomainTranslator;
@@ -90,7 +91,7 @@ public static Session buildOwnerSession(Session session, Optional<String> owner,
9091
{
9192
Identity identity = getOwnerIdentity(owner, session);
9293

93-
return Session.builder(sessionPropertyManager)
94+
Session.SessionBuilder builder = Session.builder(sessionPropertyManager)
9495
.setQueryId(session.getQueryId())
9596
.setTransactionId(session.getTransactionId().orElse(null))
9697
.setIdentity(identity)
@@ -102,8 +103,20 @@ public static Session buildOwnerSession(Session session, Optional<String> owner,
102103
.setRemoteUserAddress(session.getRemoteUserAddress().orElse(null))
103104
.setUserAgent(session.getUserAgent().orElse(null))
104105
.setClientInfo(session.getClientInfo().orElse(null))
105-
.setStartTime(session.getStartTime())
106-
.build();
106+
.setStartTime(session.getStartTime());
107+
108+
for (Map.Entry<String, String> property : session.getSystemProperties().entrySet()) {
109+
builder.setSystemProperty(property.getKey(), property.getValue());
110+
}
111+
112+
for (Map.Entry<ConnectorId, Map<String, String>> connectorEntry : session.getConnectorProperties().entrySet()) {
113+
String catalogName = connectorEntry.getKey().getCatalogName();
114+
for (Map.Entry<String, String> property : connectorEntry.getValue().entrySet()) {
115+
builder.setCatalogSessionProperty(catalogName, property.getKey(), property.getValue());
116+
}
117+
}
118+
119+
return builder.build();
107120
}
108121

109122
public static Identity getOwnerIdentity(Optional<String> owner, Session session)

presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,7 @@ public class FeaturesConfig
225225
private boolean materializedViewDataConsistencyEnabled = true;
226226
private boolean materializedViewPartitionFilteringEnabled = true;
227227
private boolean queryOptimizationWithMaterializedViewEnabled;
228+
private boolean legacyMaterializedViewRefresh = true;
228229

229230
private AggregationIfToFilterRewriteStrategy aggregationIfToFilterRewriteStrategy = AggregationIfToFilterRewriteStrategy.DISABLED;
230231
private String analyzerType = "BUILTIN";
@@ -2154,6 +2155,20 @@ public FeaturesConfig setQueryOptimizationWithMaterializedViewEnabled(boolean va
21542155
return this;
21552156
}
21562157

2158+
public boolean isLegacyMaterializedViews()
2159+
{
2160+
return legacyMaterializedViewRefresh;
2161+
}
2162+
2163+
@Config("experimental.legacy-materialized-views")
2164+
@ConfigDescription("Experimental: Use legacy materialized views. This feature is under active development and may change" +
2165+
"or be removed at any time. Do not disable in production environments.")
2166+
public FeaturesConfig setLegacyMaterializedViews(boolean value)
2167+
{
2168+
this.legacyMaterializedViewRefresh = value;
2169+
return this;
2170+
}
2171+
21572172
public boolean isVerboseRuntimeStatsEnabled()
21582173
{
21592174
return verboseRuntimeStatsEnabled;

presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@
228228

229229
import static com.facebook.presto.SystemSessionProperties.getMaxGroupingSets;
230230
import static com.facebook.presto.SystemSessionProperties.isAllowWindowOrderByLiterals;
231+
import static com.facebook.presto.SystemSessionProperties.isLegacyMaterializedViews;
231232
import static com.facebook.presto.SystemSessionProperties.isMaterializedViewDataConsistencyEnabled;
232233
import static com.facebook.presto.SystemSessionProperties.isMaterializedViewPartitionFilteringEnabled;
233234
import static com.facebook.presto.common.RuntimeMetricName.SKIP_READING_FROM_MATERIALIZED_VIEW_COUNT;
@@ -864,10 +865,9 @@ protected Scope visitRefreshMaterializedView(RefreshMaterializedView node, Optio
864865
// Use AllowAllAccessControl; otherwise Analyzer will check SELECT permission on the materialized view, which is not necessary.
865866
StatementAnalyzer viewAnalyzer = new StatementAnalyzer(analysis, metadata, sqlParser, new AllowAllAccessControl(), session, warningCollector);
866867
Scope viewScope = viewAnalyzer.analyze(node.getTarget(), scope);
867-
if (!node.getWhere().isPresent()) {
868-
throw new SemanticException(NOT_SUPPORTED, node, "Refresh Materialized View without predicates is not supported.");
869-
}
870-
Map<SchemaTableName, Expression> tablePredicates = extractTablePredicates(viewName, node.getWhere().get(), viewScope, metadata, session);
868+
869+
Map<SchemaTableName, Expression> tablePredicates = getTablePredicatesForMaterializedViewRefresh(
870+
session, node, viewName, viewScope, metadata);
871871

872872
Query viewQuery = parseView(view.getOriginalSql(), viewName, node);
873873
Query refreshQuery = tablePredicates.containsKey(toSchemaTableName(viewName)) ?
@@ -910,10 +910,9 @@ private Optional<RelationType> analyzeBaseTableForRefreshMaterializedView(Table
910910
// Use AllowAllAccessControl; otherwise Analyzer will check SELECT permission on the materialized view, which is not necessary.
911911
StatementAnalyzer viewAnalyzer = new StatementAnalyzer(analysis, metadata, sqlParser, new AllowAllAccessControl(), session, warningCollector);
912912
Scope viewScope = viewAnalyzer.analyze(refreshMaterializedView.getTarget(), scope);
913-
if (!refreshMaterializedView.getWhere().isPresent()) {
914-
throw new SemanticException(NOT_SUPPORTED, "Refresh Materialized View without predicates is not supported.");
915-
}
916-
Map<SchemaTableName, Expression> tablePredicates = extractTablePredicates(viewName, refreshMaterializedView.getWhere().get(), viewScope, metadata, session);
913+
914+
Map<SchemaTableName, Expression> tablePredicates = getTablePredicatesForMaterializedViewRefresh(
915+
session, refreshMaterializedView, viewName, viewScope, metadata);
917916

918917
SchemaTableName baseTableName = toSchemaTableName(createQualifiedObjectName(session, baseTable, baseTable.getName(), metadata));
919918
if (tablePredicates.containsKey(baseTableName)) {
@@ -928,6 +927,28 @@ private Optional<RelationType> analyzeBaseTableForRefreshMaterializedView(Table
928927
return Optional.empty();
929928
}
930929

930+
private Map<SchemaTableName, Expression> getTablePredicatesForMaterializedViewRefresh(
931+
Session session,
932+
RefreshMaterializedView node,
933+
QualifiedObjectName viewName,
934+
Scope viewScope,
935+
Metadata metadata)
936+
{
937+
if (isLegacyMaterializedViews(session)) {
938+
if (!node.getWhere().isPresent()) {
939+
throw new SemanticException(NOT_SUPPORTED, node, "Refresh Materialized View without predicates is not supported.");
940+
}
941+
return extractTablePredicates(viewName, node.getWhere().get(), viewScope, metadata, session);
942+
}
943+
else {
944+
if (node.getWhere().isPresent()) {
945+
throw new SemanticException(NOT_SUPPORTED, node, "WHERE clause in REFRESH MATERIALIZED VIEW is not supported. " +
946+
"Connectors automatically determine which data needs refreshing based on staleness detection.");
947+
}
948+
return ImmutableMap.of();
949+
}
950+
}
951+
931952
private Query buildQueryWithPredicate(Table table, Expression predicate)
932953
{
933954
Query query = simpleQuery(selectList(new AllColumns()), table, predicate);

presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ public void testDefaults()
187187
.setMaterializedViewDataConsistencyEnabled(true)
188188
.setMaterializedViewPartitionFilteringEnabled(true)
189189
.setQueryOptimizationWithMaterializedViewEnabled(false)
190+
.setLegacyMaterializedViews(true)
190191
.setVerboseRuntimeStatsEnabled(false)
191192
.setAggregationIfToFilterRewriteStrategy(AggregationIfToFilterRewriteStrategy.DISABLED)
192193
.setAnalyzerType("BUILTIN")
@@ -406,6 +407,7 @@ public void testExplicitPropertyMappings()
406407
.put("materialized-view-data-consistency-enabled", "false")
407408
.put("consider-query-filters-for-materialized-view-partitions", "false")
408409
.put("query-optimization-with-materialized-view-enabled", "true")
410+
.put("experimental.legacy-materialized-views", "false")
409411
.put("analyzer-type", "CRUX")
410412
.put("pre-process-metadata-calls", "true")
411413
.put("verbose-runtime-stats-enabled", "true")
@@ -622,6 +624,7 @@ public void testExplicitPropertyMappings()
622624
.setMaterializedViewDataConsistencyEnabled(false)
623625
.setMaterializedViewPartitionFilteringEnabled(false)
624626
.setQueryOptimizationWithMaterializedViewEnabled(true)
627+
.setLegacyMaterializedViews(false)
625628
.setVerboseRuntimeStatsEnabled(true)
626629
.setAggregationIfToFilterRewriteStrategy(AggregationIfToFilterRewriteStrategy.FILTER_WITH_IF)
627630
.setAnalyzerType("CRUX")

presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryInsertTableHandle.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,22 @@ public class MemoryInsertTableHandle
2727
{
2828
private final MemoryTableHandle table;
2929
private final Set<Long> activeTableIds;
30+
private final boolean insertOverwrite;
3031

3132
@JsonCreator
3233
public MemoryInsertTableHandle(
3334
@JsonProperty("table") MemoryTableHandle table,
34-
@JsonProperty("activeTableIds") Set<Long> activeTableIds)
35+
@JsonProperty("activeTableIds") Set<Long> activeTableIds,
36+
@JsonProperty("insertOverwrite") boolean insertOverwrite)
3537
{
3638
this.table = requireNonNull(table, "table is null");
3739
this.activeTableIds = requireNonNull(activeTableIds, "activeTableIds is null");
40+
this.insertOverwrite = insertOverwrite;
41+
}
42+
43+
public MemoryInsertTableHandle(MemoryTableHandle table, Set<Long> activeTableIds)
44+
{
45+
this(table, activeTableIds, false);
3846
}
3947

4048
@JsonProperty
@@ -49,6 +57,12 @@ public Set<Long> getActiveTableIds()
4957
return activeTableIds;
5058
}
5159

60+
@JsonProperty
61+
public boolean isInsertOverwrite()
62+
{
63+
return insertOverwrite;
64+
}
65+
5266
@Override
5367
public String toString()
5468
{

0 commit comments

Comments
 (0)