Merge pull request ClickHouse#78816 from arthurpassos/allow_empty_value_hive_partitioning

Allow empty value on hive partitioning

Author: yariks5s

src/Storages/VirtualColumnUtils.cpp

@@ -144,9 +144,9 @@ NameSet getVirtualNamesForFileLikeStorage()
     return getCommonVirtualsForFileLikeStorage().getNameSet();
 }
 
-static std::unordered_map<std::string, std::string> parseHivePartitioningKeysAndValues(const String & path)
+std::unordered_map<std::string, std::string> parseHivePartitioningKeysAndValues(const String & path)
 {
-    std::string pattern = "([^/]+)=([^/]+)/";
+    std::string pattern = "([^/]+)=([^/]*)/";
     re2::StringPiece input_piece(path);
 
     std::unordered_map<std::string, std::string> key_values;

src/Storages/VirtualColumnUtils.h

@@ -110,6 +110,9 @@ struct VirtualsForFileLikeStorage
 void addRequestedFileLikeStorageVirtualsToChunk(
     Chunk & chunk, const NamesAndTypesList & requested_virtual_columns,
     VirtualsForFileLikeStorage virtual_values, ContextPtr context);
+
+// exposed in the header for unit testing purposes
+std::unordered_map<std::string, std::string> parseHivePartitioningKeysAndValues(const String & path);
 }
 
 }

src/Storages/tests/gtest_virtual_column_utils.cpp

@@ -0,0 +1,40 @@
+#include <gtest/gtest.h>
+#include <Common/tests/gtest_global_context.h>
+#include <Storages/VirtualColumnUtils.h>
+#include <DataTypes/DataTypeString.h>
+
+using namespace DB;
+
+TEST(VirtualColumnUtils, parseHivePartitioningKeysAndValuesEmptyValue)
+{
+    static std::string empty_value_path = "/output_data/year=2022/country=/data_0.parquet";
+
+    auto map = VirtualColumnUtils::parseHivePartitioningKeysAndValues(empty_value_path);
+
+    ASSERT_TRUE(map.size() == 2);
+
+    ASSERT_TRUE(map["year"] == "2022");
+    ASSERT_TRUE(map["country"].empty());
+}
+
+TEST(VirtualColumnUtils, getVirtualsForFileLikeStorageEmptyValue)
+{
+    static std::string empty_value_path = "/output_data/year=2022/country=/data_0.parquet";
+
+    const auto & context_holder = getContext();
+
+    auto year_column = ColumnDescription("year", std::make_shared<DataTypeString>());
+    auto country_column = ColumnDescription("country", std::make_shared<DataTypeString>());
+    auto non_partition_column = ColumnDescription("non_partition", std::make_shared<DataTypeString>());
+
+    ColumnsDescription columns;
+
+    columns.add(year_column);
+    columns.add(country_column);
+    columns.add(non_partition_column);
+
+    auto res = VirtualColumnUtils::getVirtualsForFileLikeStorage(columns, context_holder.context, empty_value_path);
+
+    ASSERT_TRUE(res.has("year"));
+    ASSERT_TRUE(res.has("country"));
+}

tests/queries/0_stateless/03203_hive_style_partitioning.reference

@@ -91,6 +91,9 @@ Gibson	Elizabeth
 Greer	Elizabeth
 Delgado	Elizabeth
 Cross	Elizabeth
+test/hive_partitioning/column0=Arthur/column1=/sample.parquet
+test/hive_partitioning/column0=Arthur/column1=ABC/sample.parquet
+test/hive_partitioning/column0=Arthur/column1=/sample.parquet
 OK
 TESTING THE S3CLUSTER PARTITIONING
  last	Elizabeth

tests/queries/0_stateless/03203_hive_style_partitioning.sh

@@ -69,6 +69,9 @@ SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0
 
 SELECT *, non_existing_column FROM s3('http://localhost:11111/test/hive_partitioning/non_existing_column=Elizabeth/sample.parquet') LIMIT 10;
 SELECT *, column0 FROM s3('http://localhost:11111/test/hive_partitioning/column0=*/sample.parquet') WHERE column0 = 'Elizabeth' LIMIT 10;
+
+SELECT _path FROM s3('http://localhost:11111/test/hive_partitioning/column0=Arthur/**.parquet') order by _path;
+SELECT _path FROM s3('http://localhost:11111/test/hive_partitioning/column0=Arthur/**.parquet') where column1 = '';
 """
 
 $CLICKHOUSE_CLIENT -q """