Sometimes generate a not matching structure

Author: PedroTadim

src/Client/BuzzHouse/AST/SQLProtoStr.cpp

@@ -1730,9 +1730,7 @@ CONV_FN(FileFunc, ff)
     ret += "'";
     if (ff.has_structure())
     {
-        ret += ", '";
-        ret += ff.structure();
-        ret += "'";
+        ExprToString(ret, ff.structure());
     }
     if (ff.has_fcomp())
     {
@@ -1749,9 +1747,7 @@ CONV_FN(FormatFunc, ff)
     ret += InFormat_Name(ff.format()).substr(3);
     if (ff.has_structure())
     {
-        ret += "', '";
-        ret += ff.structure();
-        ret += "'";
+        ExprToString(ret, ff.structure());
     }
     ret += ", $$\n";
     ret += ff.data();
@@ -1906,9 +1902,8 @@ CONV_FN(S3Func, sfunc)
     ret += sfunc.password();
     ret += "', '";
     ret += InOutFormat_Name(sfunc.format()).substr(6);
-    ret += "', '";
-    ret += sfunc.structure();
-    ret += "'";
+    ret += "', ";
+    ExprToString(ret, sfunc.structure());
     if (sfunc.has_fcomp())
     {
         ret += ", '";

src/Client/BuzzHouse/Generator/QueryOracle.cpp

@@ -226,11 +226,10 @@ void QueryOracle::dumpTableContent(RandomGenerator & rg, StatementGenerator & ge
 void QueryOracle::generateExportQuery(
     RandomGenerator & rg, StatementGenerator & gen, const bool test_content, const SQLTable & t, SQLQuery & sq2)
 {
-    String buf;
-    bool first = true;
     std::error_code ec;
     Insert * ins = sq2.mutable_explain()->mutable_inner_query()->mutable_insert();
     FileFunc * ff = ins->mutable_tof()->mutable_tfunc()->mutable_file();
+    Expr * expr = ff->mutable_structure();
     SelectStatementCore * sel = ins->mutable_select()->mutable_select_core();
     const std::filesystem::path & nfile = fc.db_file_path / "table.data";
     OutFormat outf = rg.pickRandomly(outIn);
@@ -244,29 +243,40 @@ void QueryOracle::generateExportQuery(
     ff->set_path(nfile.generic_string());
 
     gen.flatTableColumnPath(skip_nested_node | flat_nested, t.cols, [](const SQLColumn & c) { return c.canBeInserted(); });
-    for (const auto & entry : gen.entries)
+    if (!can_test_query_success && rg.nextSmallNumber() < 3)
+    {
+        /// Sometimes generate a not matching structure
+        gen.addRandomRelation(rg, std::nullopt, gen.entries.size(), expr);
+    }
+    else
     {
-        SQLType * tp = entry.getBottomType();
+        String buf;
+        bool first = true;
 
-        buf += fmt::format(
-            "{}{} {}{}{}{}",
-            first ? "" : ", ",
-            entry.getBottomName(),
-            entry.path.size() > 1 ? "Array(" : "",
-            tp->typeName(true),
-            entry.path.size() > 1 ? ")" : "",
-            (entry.path.size() == 1 && entry.nullable.has_value()) ? (entry.nullable.value() ? " NULL" : " NOT NULL") : "");
-        gen.columnPathRef(entry, sel->add_result_columns()->mutable_etc()->mutable_col()->mutable_path());
-        /// ArrowStream doesn't support UUID
-        if (outf == OutFormat::OUT_ArrowStream && tp->getTypeClass() == SQLTypeClass::UUID)
+        for (const auto & entry : gen.entries)
         {
-            outf = OutFormat::OUT_CSV;
+            SQLType * tp = entry.getBottomType();
+
+            buf += fmt::format(
+                "{}{} {}{}{}{}",
+                first ? "" : ", ",
+                entry.getBottomName(),
+                entry.path.size() > 1 ? "Array(" : "",
+                tp->typeName(true),
+                entry.path.size() > 1 ? ")" : "",
+                (entry.path.size() == 1 && entry.nullable.has_value()) ? (entry.nullable.value() ? " NULL" : " NOT NULL") : "");
+            gen.columnPathRef(entry, sel->add_result_columns()->mutable_etc()->mutable_col()->mutable_path());
+            /// ArrowStream doesn't support UUID
+            if (outf == OutFormat::OUT_ArrowStream && tp->getTypeClass() == SQLTypeClass::UUID)
+            {
+                outf = OutFormat::OUT_CSV;
+            }
+            first = false;
         }
-        first = false;
+        expr->mutable_lit_val()->set_string_lit(std::move(buf));
     }
     gen.entries.clear();
     ff->set_outformat(outf);
-    ff->set_structure(buf);
     if (rg.nextSmallNumber() < 4)
     {
         ff->set_fcomp(static_cast<FileCompression>((rg.nextRandomUInt32() % static_cast<uint32_t>(FileCompression_MAX)) + 1));

src/Client/BuzzHouse/Generator/SQLQuery.cpp

@@ -254,7 +254,7 @@ void StatementGenerator::setTableRemote(
             first = false;
         }
         this->remote_entries.clear();
-        sfunc->set_structure(buf);
+        sfunc->mutable_structure()->mutable_lit_val()->set_string_lit(std::move(buf));
         if (!t.file_comp.empty())
         {
             sfunc->set_fcomp(t.file_comp);
@@ -319,6 +319,72 @@ auto StatementGenerator::getQueryTableLambda()
     };
 }
 
+void StatementGenerator::addRandomRelation(RandomGenerator & rg, const std::optional<String> rel_name, const uint32_t ncols, Expr * expr)
+{
+    if (rg.nextBool())
+    {
+        /// Use generateRandomStructure function
+        SQLFuncCall * sfc = expr->mutable_comp_expr()->mutable_func_call();
+        sfc->mutable_func()->set_catalog_func(FUNCgenerateRandomStructure);
+
+        /// Number of columns parameter
+        sfc->add_args()->mutable_expr()->mutable_lit_val()->mutable_int_lit()->set_uint_lit(static_cast<uint64_t>(ncols));
+        /// Seed parameter
+        sfc->add_args()->mutable_expr()->mutable_lit_val()->mutable_int_lit()->set_uint_lit(rg.nextRandomUInt64());
+        if (rel_name.has_value())
+        {
+            SQLRelation rel(rel_name.value());
+
+            for (uint32_t i = 0; i < ncols; i++)
+            {
+                rel.cols.emplace_back(SQLRelationCol(rel_name.value(), {"c" + std::to_string(i + 1)}));
+            }
+            this->levels[this->current_level].rels.emplace_back(rel);
+        }
+    }
+    else
+    {
+        /// Use BuzzHouse approach
+        String buf;
+        bool first = true;
+        uint32_t col_counter = 0;
+        const uint32_t type_mask_backup = this->next_type_mask;
+        std::unordered_map<uint32_t, std::unique_ptr<SQLType>> centries;
+
+        this->next_type_mask = fc.type_mask;
+        for (uint32_t i = 0; i < ncols; i++)
+        {
+            const uint32_t ncame = col_counter++;
+            auto tp = std::unique_ptr<SQLType>(randomNextType(rg, this->next_type_mask, col_counter, nullptr));
+
+            buf += fmt::format("{}c{} {}", first ? "" : ", ", ncame, tp->typeName(false));
+            first = false;
+            centries[ncame] = std::move(tp);
+        }
+        this->next_type_mask = type_mask_backup;
+        expr->mutable_lit_val()->set_string_lit(std::move(buf));
+        if (rel_name.has_value())
+        {
+            SQLRelation rel(rel_name.value());
+
+            flatColumnPath(flat_tuple | flat_nested | flat_json | to_table_entries | collect_generated, centries);
+            for (const auto & entry : this->table_entries)
+            {
+                DB::Strings names;
+
+                names.reserve(entry.path.size());
+                for (const auto & path : entry.path)
+                {
+                    names.push_back(path.cname);
+                }
+                rel.cols.emplace_back(SQLRelationCol(rel_name.value(), std::move(names)));
+            }
+            this->table_entries.clear();
+            this->levels[this->current_level].rels.emplace_back(rel);
+        }
+    }
+}
+
 bool StatementGenerator::joinedTableOrFunction(
     RandomGenerator & rg, const String & rel_name, const uint32_t allowed_clauses, const bool under_remote, TableOrFunction * tof)
 {
@@ -743,66 +809,15 @@ bool StatementGenerator::joinedTableOrFunction(
             < (derived_table + cte + table + view + remote_udf + generate_series_udf + system_table + merge_udf + cluster_udf
                + merge_index_udf + loop_udf + values_udf + random_data_udf + 1))
     {
-        SQLRelation rel(rel_name);
-        const uint32_t ncols = (rg.nextSmallNumber() < 8) ? rg.nextSmallNumber() : rg.nextMediumNumber();
         GenerateRandomFunc * grf = tof->mutable_tfunc()->mutable_grandom();
         std::uniform_int_distribution<uint64_t> string_length_dist(1, 8192);
         std::uniform_int_distribution<uint64_t> nested_rows_dist(fc.min_nested_rows, fc.max_nested_rows);
 
-        if (rg.nextBool())
-        {
-            /// Use generateRandomStructure function
-            SQLFuncCall * sfc = grf->mutable_structure()->mutable_comp_expr()->mutable_func_call();
-            sfc->mutable_func()->set_catalog_func(FUNCgenerateRandomStructure);
-
-            /// Number of columns parameter
-            sfc->add_args()->mutable_expr()->mutable_lit_val()->mutable_int_lit()->set_uint_lit(static_cast<uint64_t>(ncols));
-            /// Seed parameter
-            sfc->add_args()->mutable_expr()->mutable_lit_val()->mutable_int_lit()->set_uint_lit(rg.nextRandomUInt64());
-            for (uint32_t i = 0; i < ncols; i++)
-            {
-                rel.cols.emplace_back(SQLRelationCol(rel_name, {"c" + std::to_string(i + 1)}));
-            }
-        }
-        else
-        {
-            /// Use BuzzHouse approach
-            String buf;
-            bool first = true;
-            uint32_t col_counter = 0;
-            const uint32_t type_mask_backup = this->next_type_mask;
-            std::unordered_map<uint32_t, std::unique_ptr<SQLType>> centries;
-
-            this->next_type_mask = fc.type_mask;
-            for (uint32_t i = 0; i < ncols; i++)
-            {
-                const uint32_t ncame = col_counter++;
-                auto tp = std::unique_ptr<SQLType>(randomNextType(rg, this->next_type_mask, col_counter, nullptr));
-
-                buf += fmt::format("{}c{} {}", first ? "" : ", ", ncame, tp->typeName(false));
-                first = false;
-                centries[ncame] = std::move(tp);
-            }
-            this->next_type_mask = type_mask_backup;
-            grf->mutable_structure()->mutable_lit_val()->set_string_lit(std::move(buf));
-            flatColumnPath(flat_tuple | flat_nested | flat_json | to_table_entries | collect_generated, centries);
-            for (const auto & entry : this->table_entries)
-            {
-                DB::Strings names;
-
-                names.reserve(entry.path.size());
-                for (const auto & path : entry.path)
-                {
-                    names.push_back(path.cname);
-                }
-                rel.cols.emplace_back(SQLRelationCol(rel_name, std::move(names)));
-            }
-            this->table_entries.clear();
-        }
+        addRandomRelation(
+            rg, rel_name, (rg.nextSmallNumber() < 8) ? rg.nextSmallNumber() : rg.nextMediumNumber(), grf->mutable_structure());
         grf->set_random_seed(rg.nextRandomUInt64());
         grf->set_max_string_length(string_length_dist(rg.generator));
         grf->set_max_array_length(nested_rows_dist(rg.generator));
-        this->levels[this->current_level].rels.emplace_back(rel);
     }
     else if (
         dictionary

src/Client/BuzzHouse/Generator/StatementGenerator.h

@@ -289,6 +289,7 @@ class StatementGenerator
     void flatTableColumnPath(
         uint32_t flags, const std::unordered_map<uint32_t, SQLColumn> & cols, std::function<bool(const SQLColumn & c)> col_filter);
     void flatColumnPath(uint32_t flags, const std::unordered_map<uint32_t, std::unique_ptr<SQLType>> & centries);
+    void addRandomRelation(RandomGenerator & rg, const std::optional<String> rel_name, uint32_t ncols, Expr * expr);
     void generateStorage(RandomGenerator & rg, Storage * store) const;
     void generateNextCodecs(RandomGenerator & rg, CodecList * cl);
     void generateTTLExpression(RandomGenerator & rg, const std::optional<SQLTable> & t, Expr * ttl_expr);

src/Client/BuzzHouse/Proto/SQLGrammar.proto

@@ -2421,13 +2421,13 @@ message FileFunc {
         InFormat informat = 2;
         OutFormat outformat = 3;
     }
-    optional string structure = 4;
+    optional Expr structure = 4;
     optional FileCompression fcomp = 5;
 }
 
 message FormatFunc {
     required InFormat format = 1;
-    optional string structure = 2;
+    optional Expr structure = 2;
     required string data = 3;
 }
 
@@ -2490,7 +2490,7 @@ message S3Func {
     required string user = 2;
     required string password = 3;
     required InOutFormat format = 4;
-    required string structure = 5;
+    required Expr structure = 5;
     optional string fcomp = 6;
 }