diff --git a/omniscidb/Analyzer/Analyzer.cpp b/omniscidb/Analyzer/Analyzer.cpp index caad10a1c..3f472bb04 100644 --- a/omniscidb/Analyzer/Analyzer.cpp +++ b/omniscidb/Analyzer/Analyzer.cpp @@ -142,6 +142,8 @@ const hdk::ir::Type* analyze_type_info(hdk::ir::OpType op, auto& ctx = left_type->ctx(); const hdk::ir::Type* result_type; const hdk::ir::Type* common_type; + LOG(ERROR) << "analyze_type_info - left type: " << left_type->toString() + << " right: " << right_type->toString(); *new_left_type = left_type; *new_right_type = right_type; if (hdk::ir::isLogic(op)) { @@ -306,6 +308,8 @@ const hdk::ir::Type* analyze_type_info(hdk::ir::OpType op, } else { throw std::runtime_error("invalid binary operator type."); } + LOG(ERROR) << "new left: " << (*new_left_type)->toString() + << " right: " << (*new_right_type)->toString(); result_type = result_type->withNullable(left_type->nullable() || right_type->nullable()); return result_type; diff --git a/omniscidb/ArrowStorage/ArrowStorage.cpp b/omniscidb/ArrowStorage/ArrowStorage.cpp index 70aacb127..28c1b1c7c 100644 --- a/omniscidb/ArrowStorage/ArrowStorage.cpp +++ b/omniscidb/ArrowStorage/ArrowStorage.cpp @@ -345,6 +345,7 @@ TableInfoPtr ArrowStorage::createTable(const std::string& table_name, TableInfoPtr res; int table_id; mapd_unique_lock data_lock(data_mutex_); + LOG(ERROR) << "Table ------- " << table_name << " -------"; size_t next_col_idx = 0; { mapd_unique_lock dict_lock(dict_mutex_); @@ -395,8 +396,10 @@ TableInfoPtr ArrowStorage::createTable(const std::string& table_name, type = elem_type; } } + LOG(ERROR) << "adding col info " << type->toString(); auto col_info = addColumnInfo( db_id_, table_id, columnId(next_col_idx++), col.name, type, false); + LOG(ERROR) << "added col info " << col_info->toString(); } addRowidColumn(db_id_, table_id, columnId(next_col_idx++)); } @@ -418,6 +421,7 @@ TableInfoPtr ArrowStorage::createTable(const std::string& table_name, table.fragment_size = options.fragment_size; table.schema = schema; } + LOG(ERROR) << "table info " << res->toString() << " schema: " << schema->ToString(true); return res; } diff --git a/omniscidb/IR/Expr.cpp b/omniscidb/IR/Expr.cpp index 55c8a0242..1906583c5 100644 --- a/omniscidb/IR/Expr.cpp +++ b/omniscidb/IR/Expr.cpp @@ -256,6 +256,7 @@ ExprPtr Expr::decompress() const { return makeExpr(new_type, contains_agg_, OpType::kCast, shared_from_this()); } else if (type_->id() == Type::kDate && type_->size() != 8) { auto date_type = static_cast(type_); + LOG(ERROR) << "decompress type: " << type_->toString(); return makeExpr(type_->ctx().date64(TimeUnit::kSecond, date_type->nullable()), contains_agg_, OpType::kCast, diff --git a/omniscidb/QueryBuilder/QueryBuilder.cpp b/omniscidb/QueryBuilder/QueryBuilder.cpp index 044a73f55..a41fe3576 100644 --- a/omniscidb/QueryBuilder/QueryBuilder.cpp +++ b/omniscidb/QueryBuilder/QueryBuilder.cpp @@ -864,7 +864,8 @@ BuilderExpr BuilderExpr::cast(const Type* new_type) const { return {builder_, expr_->cast(new_type), "", true}; } } else if (expr_->type()->isDate()) { - if (new_type->isDate() || new_type->isTimestamp()) { + LOG(ERROR) << "Conversion date: " << expr_->type() << " new_type: " << new_type; + if (new_type->isInteger() || new_type->isDate() || new_type->isTimestamp()) { return {builder_, expr_->cast(new_type), "", true}; } } else if (expr_->type()->isTime()) { @@ -872,7 +873,7 @@ BuilderExpr BuilderExpr::cast(const Type* new_type) const { return {builder_, expr_->cast(new_type), "", true}; } } else if (expr_->type()->isTimestamp()) { - if (new_type->isNumber() || new_type->isDate() || new_type->isTimestamp()) { + if (new_type->isInteger() || new_type->isDate() || new_type->isTimestamp()) { return {builder_, expr_->cast(new_type), "", true}; } } diff --git a/omniscidb/QueryEngine/ArrowResultSet.cpp b/omniscidb/QueryEngine/ArrowResultSet.cpp index ba46cd6c6..828a0ccbc 100644 --- a/omniscidb/QueryEngine/ArrowResultSet.cpp +++ b/omniscidb/QueryEngine/ArrowResultSet.cpp @@ -28,6 +28,7 @@ namespace { const hdk::ir::Type* type_from_arrow_field(hdk::ir::Context& ctx, const arrow::Field& field) { + LOG(ERROR) << "type_from_arrow called: " << field.type()->ToString(); switch (field.type()->id()) { case arrow::Type::INT8: return ctx.int8(field.nullable()); diff --git a/omniscidb/QueryEngine/RelAlgDagBuilder.cpp b/omniscidb/QueryEngine/RelAlgDagBuilder.cpp index 0bc45b76c..f4a52a0df 100644 --- a/omniscidb/QueryEngine/RelAlgDagBuilder.cpp +++ b/omniscidb/QueryEngine/RelAlgDagBuilder.cpp @@ -129,6 +129,7 @@ const hdk::ir::Type* buildType(hdk::ir::Context& ctx, return ctx.timestamp(precisionToTimeUnit(precision), nullable); } if (type_name == std::string("DATE")) { + LOG(ERROR) << "type_name: " << type_name; return ctx.date64(hdk::ir::TimeUnit::kSecond, nullable); } if (type_name == std::string("TIME")) { @@ -2306,6 +2307,7 @@ class RelAlgDispatcher { } std::vector parseTupleType(const rapidjson::Value& tuple_type_arr) { + LOG(ERROR) << "ParseTuple? "; CHECK(tuple_type_arr.IsArray()); std::vector tuple_type; for (auto tuple_type_arr_it = tuple_type_arr.Begin(); diff --git a/omniscidb/QueryEngine/RelAlgExecutor.cpp b/omniscidb/QueryEngine/RelAlgExecutor.cpp index e1c0d6b9c..637c36d80 100644 --- a/omniscidb/QueryEngine/RelAlgExecutor.cpp +++ b/omniscidb/QueryEngine/RelAlgExecutor.cpp @@ -218,6 +218,20 @@ ExecutionResult RelAlgExecutor::executeRelAlgQueryNoRetry(const CompilationOptio executor_->setupCaching(data_provider_, col_descs, phys_table_ids); ScopeGuard restore_metainfo_cache = [this] { executor_->clearMetaInfoCache(); }; + + auto schema_provider = executor_->getSchemaProvider(); + auto dbs = schema_provider->listDatabases(); + // Current JSON format supports a single database only. To support result + // sets in SQL queries, we add tables from the ResultSetRegistry using + // negative table ids. + auto tables = schema_provider->listTables(dbs[0]); + auto more_tables = schema_provider->listTables(dbs[0]); + for (auto table : more_tables) { + LOG(ERROR) << table->toString(); + for (auto column : schema_provider->listColumns(dbs[0], table->table_id)) { + LOG(ERROR) << column->toString(); + } + } hdk::QueryExecutionSequence query_seq(ra, executor_->getConfigPtr()); if (just_explain_plan) { std::stringstream ss; @@ -260,6 +274,7 @@ ExecutionResult RelAlgExecutor::executeRelAlgQueryNoRetry(const CompilationOptio for (auto& subquery : getSubqueries()) { auto subquery_ra = subquery->node(); CHECK(subquery_ra); + LOG(ERROR) << "subq node: " << subquery_ra->toString(); if (subquery_ra->hasContextData()) { continue; } @@ -268,6 +283,7 @@ ExecutionResult RelAlgExecutor::executeRelAlgQueryNoRetry(const CompilationOptio hdk::QueryExecutionSequence subquery_seq(subquery_ra, executor_->getConfigPtr()); ra_executor.execute(subquery_seq, co, eo, 0); } + LOG(ERROR) << "seq front: " << query_seq.steps().front()->toString(); auto shared_res = execute(query_seq, co, eo, queue_time_ms); return std::move(*shared_res); diff --git a/omniscidb/QueryEngine/RelAlgTranslator.cpp b/omniscidb/QueryEngine/RelAlgTranslator.cpp index edd9f6048..3068ec8f9 100644 --- a/omniscidb/QueryEngine/RelAlgTranslator.cpp +++ b/omniscidb/QueryEngine/RelAlgTranslator.cpp @@ -94,6 +94,7 @@ std::pair datum_from_scalar_tv(const ScalarTargetValue* scalar_tv, case hdk::ir::Type::kTimestamp: { const auto ival = boost::get(scalar_tv); CHECK(ival); + LOG(ERROR) << "translator: " << type->toString() << " scalar val: " << scalar_tv; if (*ival == inline_int_null_value(type)) { is_null_const = true; } else { diff --git a/omniscidb/ResultSetRegistry/ResultSetRegistry.cpp b/omniscidb/ResultSetRegistry/ResultSetRegistry.cpp index 755f95580..2c64463e1 100644 --- a/omniscidb/ResultSetRegistry/ResultSetRegistry.cpp +++ b/omniscidb/ResultSetRegistry/ResultSetRegistry.cpp @@ -124,7 +124,7 @@ ResultSetTableTokenPtr ResultSetRegistry::put(ResultSetTable table) { addRowidColumn(db_id_, table_id, columnId(first_rs->colCount())); // TODO: lazily compute row count and try to avoid global write - // locks for that + // locks for Date auto table_data = std::make_unique(); size_t row_count = 0; for (auto& rs : table.results()) { diff --git a/omniscidb/Shared/DateTimeParser.h b/omniscidb/Shared/DateTimeParser.h index 7ec9b35af..320dcdfc3 100644 --- a/omniscidb/Shared/DateTimeParser.h +++ b/omniscidb/Shared/DateTimeParser.h @@ -50,6 +50,30 @@ int64_t dateTimeParse(std::string_view const s, hdk::ir::TimeUnit unit) { } } +namespace { + +template < + class To, + class From, + std::enable_if_t && std::is_arithmetic_v, bool> = true> +To numeric_cast(From v) { + auto r = static_cast(v); + if (static_cast(r) != v || std::signbit(r) != std::signbit(v)) + throw std::runtime_error("numeric_cast<>() failed"); + return r; +} +} // namespace + +template +R dateTimeParse(std::string_view const s, hdk::ir::TimeUnit unit) { + if (auto const time = dateTimeParseOptional(s, unit)) { + return numeric_cast(*time); + } else { + throw std::runtime_error( + cat("Invalid date/time (", std::to_string(TYPE), ") string (", s, ')')); + } +} + template int64_t dateTimeParse(std::string_view const s, int dim) { hdk::ir::TimeUnit unit; diff --git a/omniscidb/Tests/ExecutionSequenceTest.cpp b/omniscidb/Tests/ExecutionSequenceTest.cpp index 02e9ffc66..c52d2e88d 100644 --- a/omniscidb/Tests/ExecutionSequenceTest.cpp +++ b/omniscidb/Tests/ExecutionSequenceTest.cpp @@ -88,6 +88,19 @@ class ExecutionSequenceTest : public ::testing::Test { ExecutionResult runQuery(std::unique_ptr dag, bool just_explain = false) { auto ra_executor = RelAlgExecutor(getExecutor(), getStorage(), std::move(dag)); + auto schema_provider = getStorage(); + auto dbs = schema_provider->listDatabases(); + // Current JSON format supports a single database only. To support result + // sets in SQL queries, we add tables from the ResultSetRegistry using + // negative table ids. + auto tables = schema_provider->listTables(dbs[0]); + auto more_tables = schema_provider->listTables(dbs[0]); + for (auto table : more_tables) { + LOG(ERROR) << table->toString(); + for (auto column : schema_provider->listColumns(dbs[0], table->table_id)) { + LOG(ERROR) << column->toString(); + } + } auto eo = ExecutionOptions::fromConfig(config()); eo.just_explain = just_explain; eo.allow_loop_joins = true; diff --git a/omniscidb/Tests/QueryBuilderTest.cpp b/omniscidb/Tests/QueryBuilderTest.cpp index c71a925cc..e1e2c327d 100644 --- a/omniscidb/Tests/QueryBuilderTest.cpp +++ b/omniscidb/Tests/QueryBuilderTest.cpp @@ -346,6 +346,19 @@ class QueryBuilderTest : public TestSuite { {"id": 3, "arr1":[1, null], "arr2" : [null, 5.0, null]} {"id": 4, "arr1":[1, 2], "arr2" : [4.0, 5.0, 6.0]})___"); + createTable("test_date", + {{"col_bi", ctx().int64()}, + {"col_i", ctx().int32()}, + {"col_f", ctx().fp32()}, + {"col_d", ctx().fp64()}, + {"col_dec", ctx().decimal64(10, 2)}, + {"col_b", ctx().boolean()}, + {"col_str", ctx().text()}, + {"col_date", ctx().date32(hdk::ir::TimeUnit::kDay)}}); + insertCsvValues("test_date", + "1,1,0.75,0.13444545,50.02,false,some_text,2000-01-01\n" + "2,2,30.82,0.461,7.05,true,a_text,2015-03-18\n"); + createTable("sort", {{"x", ctx().int32()}, {"y", ctx().int32()}, {"z", ctx().int32()}}); insertCsvValues("sort", @@ -362,12 +375,28 @@ class QueryBuilderTest : public TestSuite { createTable("withNull", {{"a", ctx().int64()}}); insertCsvValues("withNull", "1\nNULL"); + + LOG(ERROR) << "===========================getStorage()==========================="; + auto schema_provider = getStorage(); + auto dbs = schema_provider->listDatabases(); + auto tables = schema_provider->listTables(dbs[0]); + auto more_tables = schema_provider->listTables(dbs[0]); + for (auto table : more_tables) { + LOG(ERROR) << table->toString(); + for (auto column : schema_provider->listColumns(dbs[0], table->table_id)) { + LOG(ERROR) << " " << column->toString(); + } + } } static void TearDownTestSuite() { dropTable("test1"); dropTable("test2"); dropTable("test3"); + dropTable("test_str"); + dropTable("test_varr"); + dropTable("test_arr"); + dropTable("test_date"); dropTable("sort"); dropTable("ambiguous"); dropTable("join1"); @@ -518,6 +547,49 @@ TEST_F(QueryBuilderTest, Arithmetics) { compare_res_data(res, std::vector({0, NULL_BIGINT})); } +TEST_F(QueryBuilderTest, DateToInt) { + QueryBuilder builder(ctx(), schema_mgr_, configPtr()); + + auto tinfo_a = builder.scan("test_date"); + + auto dag = tinfo_a.proj(tinfo_a.ref("col_date").cast("int32")).finalize(); + auto res = runQuery(std::move(dag)); + LOG(ERROR) << "res: " << res.toString(); + LOG(ERROR) << "res: " << toArrow(res)->ToString(); + compare_res_data(res, std::vector({946684800, 1426636800})); + + dag = tinfo_a.proj("col_date").finalize(); + res = runQuery(std::move(dag)); + LOG(ERROR) << "second res: " << res.toString(); + LOG(ERROR) << "second res: " << toArrow(res)->ToString(); + compare_res_data( + res, + std::vector( + {dateTimeParse("2000-01-01", TimeUnit::kDay), + dateTimeParse("2015-03-18", TimeUnit::kDay)})); +} + +TEST_F(QueryBuilderTest, DateToInt2) { + QueryBuilder builder(ctx(), schema_mgr_, configPtr()); + + auto tinfo_a = builder.scan("test_date"); + + auto dag = tinfo_a.proj("col_date").finalize(); + auto res = runQuery(std::move(dag)); + LOG(ERROR) << "second res: " << res.toString(); + LOG(ERROR) << "second res: " << toArrow(res)->ToString(); + compare_res_data( + res, + std::vector( + {dateTimeParse("2000-01-01", TimeUnit::kDay), + dateTimeParse("2015-03-18", TimeUnit::kDay)})); + + dag = tinfo_a.finalize(); + res = runQuery(std::move(dag)); + LOG(ERROR) << "second res: " << res.toString(); + LOG(ERROR) << "second res: " << toArrow(res)->ToString(); +} + TEST_F(QueryBuilderTest, Arithmetics2) { QueryBuilder builder(ctx(), schema_mgr_, configPtr());