Merge pull request ClickHouse#79085 from ClickHouse/implicit-table

Implicit table in `clickhouse-local`

Author: alexey-milovidov

programs/local/LocalServer.cpp

@@ -418,12 +418,19 @@ void LocalServer::cleanup()
 }
 
 
-std::string LocalServer::getInitialCreateTableQuery()
+std::pair<std::string, std::string> LocalServer::getInitialCreateTableQuery()
 {
-    if (!getClientConfiguration().has("table-structure") && !getClientConfiguration().has("table-file") && !getClientConfiguration().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty()))
+    /// The input data can be specified explicitly with any of the `file`, `structure`, `input-format` command line arguments,
+    /// or it can be implicitly specified in stdin - then the structure and format is autodetected.
+    /// But if queries were not specified in the command line, they might me in stdin, and this means that stdin is not input data.
+
+    if (!getClientConfiguration().has("table-structure")
+        && !getClientConfiguration().has("table-file")
+        && !getClientConfiguration().has("table-data-format")
+        && (queries.empty() || !isFileDescriptorSuitableForInput(stdin_fd))) /// In we know that there is data in stdin, we can auto-detect the format.
         return {};
 
-    auto table_name = backQuoteIfNeed(getClientConfiguration().getString("table-name", "table"));
+    auto table_name = getClientConfiguration().getString("table-name", "table");
     auto table_structure = getClientConfiguration().getString("table-structure", "auto");
 
     String table_file;
@@ -442,15 +449,24 @@ std::string LocalServer::getInitialCreateTableQuery()
         table_file = quoteString(file_name);
     }
 
-    String data_format = backQuoteIfNeed(default_input_format);
+    String data_format;
+
+    if (default_input_format == "auto" && getClientConfiguration().has("table-structure"))
+        data_format = "TabSeparated";   /// Compatibility with older versions when format inference was not available.
+    else
+        data_format = backQuoteIfNeed(default_input_format);
 
     if (table_structure == "auto")
         table_structure = "";
     else
         table_structure = "(" + table_structure + ")";
 
-    return fmt::format("CREATE TEMPORARY TABLE {} {} ENGINE = File({}, {}, {});",
-                       table_name, table_structure, data_format, table_file, compression);
+    return
+    {
+        table_name,
+        fmt::format("CREATE TEMPORARY TABLE {} {} ENGINE = File({}, {}, {});",
+            backQuote(table_name), table_structure, data_format, table_file, compression)
+    };
 }
 
 
@@ -626,10 +642,13 @@ try
         std::cerr << std::endl;
     }
 
+    auto [table_name, initial_query] = getInitialCreateTableQuery();
+    if (!table_name.empty())
+        client_context->setSetting("implicit_table_at_top_level", table_name);
+
     connect();
 
-    String initial_query = getInitialCreateTableQuery();
-    if (!initial_query.empty())
+    if (!table_name.empty())
         processQueryText(initial_query);
 
 #if USE_FUZZING_MODE
@@ -868,7 +887,7 @@ void LocalServer::processConfig()
 #endif
 
     /// NOTE: it is important to apply any overrides before
-    /// setDefaultProfiles() calls since it will copy current context (i.e.
+    /// `setDefaultProfiles` calls since it will copy current context (i.e.
     /// there is separate context for Buffer tables).
     adjustSettings();
     applySettingsOverridesForLocal(global_context);

programs/local/LocalServer.h

@@ -53,9 +53,10 @@ class LocalServer : public ClientApplicationBase, public Loggers
 private:
     /** Composes CREATE subquery based on passed arguments (--structure --file --table and --input-format)
       * This query will be executed first, before queries passed through --query argument
-      * Returns empty string if it cannot compose that query.
+      * Returns a pair of the table name and the corresponding create table statement.
+      * Returns empty strings if it cannot compose that query.
       */
-    std::string getInitialCreateTableQuery();
+    std::pair<std::string, std::string> getInitialCreateTableQuery();
 
     void tryInitPath();
     void setupUsers();

src/Analyzer/QueryTreeBuilder.cpp

@@ -4,7 +4,6 @@
 #include <Common/quoteString.h>
 
 #include <DataTypes/FieldToDataType.h>
-#include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ASTSelectIntersectExceptQuery.h>
 #include <Parsers/ASTExpressionList.h>
@@ -47,12 +46,12 @@
 
 #include <Databases/IDatabase.h>
 
-#include <Interpreters/StorageID.h>
 #include <Interpreters/Context.h>
 
 
 namespace DB
 {
+
 namespace Setting
 {
     extern const SettingsBool allow_experimental_variant_type;
@@ -62,6 +61,7 @@ namespace Setting
     extern const SettingsUInt64 limit;
     extern const SettingsUInt64 offset;
     extern const SettingsBool use_variant_as_common_type;
+    extern const SettingsString implicit_table_at_top_level;
 }
 
 
@@ -123,7 +123,7 @@ class QueryTreeBuilder
 
     QueryTreeNodePtr buildWindow(const ASTPtr & window_definition, const ContextPtr & context) const;
 
-    QueryTreeNodePtr buildJoinTree(const ASTSelectQuery & select_query, const ContextPtr & context) const;
+    QueryTreeNodePtr buildJoinTree(bool is_subquery, const ASTSelectQuery & select_query, const ContextPtr & context) const;
 
     ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, const ContextPtr & context) const;
 
@@ -239,7 +239,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr
     return union_node;
 }
 
-QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_query,
+QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(
+    const ASTPtr & select_query,
     bool is_subquery,
     const std::string & cte_name,
     const ASTPtr & aliases,
@@ -313,7 +314,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
 
     auto current_context = current_query_tree->getContext();
 
-    current_query_tree->getJoinTree() = buildJoinTree(select_query_typed, current_context);
+    current_query_tree->getJoinTree() = buildJoinTree(is_subquery, select_query_typed, current_context);
 
     auto select_with_list = select_query_typed.with();
     if (select_with_list)
@@ -853,16 +854,23 @@ std::shared_ptr<TableFunctionNode> QueryTreeBuilder::buildTableFunction(const AS
     return node;
 }
 
-QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTSelectQuery & select_query, const ContextPtr & context) const
+QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(bool is_subquery, const ASTSelectQuery & select_query, const ContextPtr & context) const
 {
     const auto & tables_in_select_query = select_query.tables();
     if (!tables_in_select_query)
     {
-        /** If no table is specified in SELECT query we substitute system.one table.
-          * SELECT * FROM system.one;
+        /** If no table is specified in SELECT query,
+          * if 'implicit_table_at_top_level' is set, we substitute it as a table,
+          * otherwise, we substitute the system.one table: SELECT * FROM system.one;
           */
-        Identifier storage_identifier("system.one");
-        return std::make_shared<IdentifierNode>(storage_identifier);
+        if (!is_subquery)
+        {
+            String implicit_table = context->getSettingsRef()[Setting::implicit_table_at_top_level];
+            if (!implicit_table.empty())
+                return std::make_shared<IdentifierNode>(Identifier(implicit_table));
+        }
+
+        return std::make_shared<IdentifierNode>(Identifier("system.one"));
     }
 
     auto & tables = tables_in_select_query->as<ASTTablesInSelectQuery &>();

src/Analyzer/Resolve/QueryAnalyzer.cpp

@@ -113,6 +113,7 @@ namespace Setting
     extern const SettingsBool allow_suspicious_types_in_order_by;
     extern const SettingsBool allow_not_comparable_types_in_order_by;
     extern const SettingsBool use_concurrency_control;
+    extern const SettingsString implicit_table_at_top_level;
 }
 
 
@@ -581,6 +582,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
         Settings subquery_settings = context->getSettingsCopy();
         subquery_settings[Setting::max_result_rows] = 1;
         subquery_settings[Setting::extremes] = false;
+        subquery_settings[Setting::implicit_table_at_top_level] = "";
         /// When execute `INSERT INTO t WITH ... SELECT ...`, it may lead to `Unknown columns`
         /// exception with this settings enabled(https://github.com/ClickHouse/ClickHouse/issues/52494).
         subquery_settings[Setting::use_structure_from_insertion_table_in_table_functions] = false;

src/Client/ClientBase.cpp

@@ -877,10 +877,11 @@ void ClientBase::initClientContext()
     client_context->setQueryParameters(query_parameters);
 }
 
-bool ClientBase::isRegularFile(int fd)
+bool ClientBase::isFileDescriptorSuitableForInput(int fd)
 {
     struct stat file_stat;
-    return fstat(fd, &file_stat) == 0 && S_ISREG(file_stat.st_mode);
+    return fstat(fd, &file_stat) == 0
+        && (S_ISREG(file_stat.st_mode) || S_ISLNK(file_stat.st_mode));
 }
 
 void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
@@ -900,7 +901,7 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
         default_output_format = "Vertical";
         is_default_format = false;
     }
-    else if (isRegularFile(stdout_fd))
+    else if (isFileDescriptorSuitableForInput(stdout_fd))
     {
         std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(stdout_fd);
         if (format_from_file_name)
@@ -936,15 +937,15 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration()
         if (format_from_file_name)
             default_input_format = *format_from_file_name;
         else
-            default_input_format = "TSV";
+            default_input_format = "auto";
     }
     else
     {
         std::optional<String> format_from_file_name = FormatFactory::instance().tryGetFormatFromFileDescriptor(stdin_fd);
         if (format_from_file_name)
             default_input_format = *format_from_file_name;
         else
-            default_input_format = "TSV";
+            default_input_format = "auto";
 
         std::optional<String> file_name = tryGetFileNameFromFileDescriptor(stdin_fd);
         if (file_name)

src/Client/ClientBase.h

@@ -269,7 +269,7 @@ class ClientBase
     static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context);
     bool processMultiQueryFromFile(const String & file_name);
 
-    static bool isRegularFile(int fd);
+    static bool isFileDescriptorSuitableForInput(int fd);
 
     /// Adjust some settings after command line options and config had been processed.
     void adjustSettings();

src/Core/Settings.cpp

@@ -6401,6 +6401,19 @@ Enable pushing user roles from originator to other nodes while performing a quer
 )", 0) \
     DECLARE(Bool, shared_merge_tree_sync_parts_on_partition_operations, true, R"(
 Automatically synchronize set of data parts after MOVE|REPLACE|ATTACH partition operations in SMT tables. Cloud only
+)", 0) \
+    DECLARE(String, implicit_table_at_top_level, "", R"(
+If not empty, queries without FROM at the top level will read from this table instead of system.one.
+
+This is used in clickhouse-local for input data processing.
+The setting could be set explicitly by a user but is not intended for this type of usage.
+
+Subqueries are not affected by this setting (neither scalar, FROM, or IN subqueries).
+SELECTs at the top level of UNION, INTERSECT, EXCEPT chains are treated uniformly and affected by this setting, regardless of their grouping in parentheses.
+It is unspecified how this setting affects views and distributed queries.
+
+The setting accepts a table name (then the table is resolved from the current database) or a qualified name in the form of 'database.table'.
+Both database and table names have to be unquoted - only simple identifiers are allowed.
 )", 0) \
     \
     DECLARE(Bool, allow_experimental_variant_type, true, R"(

src/Core/SettingsChangesHistory.cpp

@@ -69,6 +69,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory()
         addSettingsChanges(settings_changes_history, "25.5",
         {
             {"secondary_indices_enable_bulk_filtering", false, true, "A new algorithm for filtering by data skipping indices"},
+            {"implicit_table_at_top_level", "", "", "A new setting, used in clickhouse-local"},
         });
         addSettingsChanges(settings_changes_history, "25.4",
         {

src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp

@@ -5,14 +5,12 @@
 #include <Core/Settings.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeTuple.h>
-#include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
 #include <Interpreters/ProcessorsProfileLog.h>
 #include <Interpreters/addTypeConversionToAST.h>
 #include <Interpreters/misc.h>
-#include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTSelectQuery.h>
@@ -38,6 +36,7 @@ namespace Setting
     extern const SettingsBool extremes;
     extern const SettingsUInt64 max_result_rows;
     extern const SettingsBool use_concurrency_control;
+    extern const SettingsString implicit_table_at_top_level;
 }
 
 namespace ErrorCodes
@@ -96,6 +95,7 @@ static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqu
     Settings subquery_settings = data.getContext()->getSettingsCopy();
     subquery_settings[Setting::max_result_rows] = 1;
     subquery_settings[Setting::extremes] = false;
+    subquery_settings[Setting::implicit_table_at_top_level] = "";
     subquery_context->setSettings(subquery_settings);
 
     if (subquery_context->hasQueryContext())

src/Interpreters/InterpreterSelectQuery.cpp

@@ -193,6 +193,7 @@ namespace Setting
     extern const SettingsUInt64 max_bytes_to_transfer;
     extern const SettingsUInt64 max_rows_to_transfer;
     extern const SettingsOverflowMode transfer_overflow_mode;
+    extern const SettingsString implicit_table_at_top_level;
 }
 
 namespace ServerSetting
@@ -355,6 +356,7 @@ ContextPtr getSubqueryContext(const ContextPtr & context)
     Settings subquery_settings = context->getSettingsCopy();
     subquery_settings[Setting::max_result_rows] = 0;
     subquery_settings[Setting::max_result_bytes] = 0;
+    subquery_settings[Setting::implicit_table_at_top_level] = "";
     /// The calculation of extremes does not make sense and is not necessary (if you do it, then the extremes of the subquery can be taken for whole query).
     subquery_settings[Setting::extremes] = false;
     subquery_context->setSettings(subquery_settings);