diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp index c2190be9374163..8f8a868624e53a 100644 --- a/be/src/util/arrow/row_batch.cpp +++ b/be/src/util/arrow/row_batch.cpp @@ -97,6 +97,8 @@ Status convert_to_arrow_type(const vectorized::DataTypePtr& origin_type, case TYPE_DATEV2: *result = std::make_shared(); break; + // TODO: maybe need to distinguish TYPE_DATETIME and TYPE_TIMESTAMPTZ + case TYPE_TIMESTAMPTZ: case TYPE_DATETIMEV2: if (type->get_scale() > 3) { *result = std::make_shared(arrow::TimeUnit::MICRO, timezone); diff --git a/be/src/vec/data_types/serde/data_type_timestamptz_serde.cpp b/be/src/vec/data_types/serde/data_type_timestamptz_serde.cpp index f941cb1ec5c641..0510ba9ce1508e 100644 --- a/be/src/vec/data_types/serde/data_type_timestamptz_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_timestamptz_serde.cpp @@ -17,6 +17,8 @@ #include "data_type_timestamptz_serde.h" +#include + #include "runtime/primitive_type.h" #include "vec/functions/cast/cast_parameters.h" #include "vec/functions/cast/cast_to_timestamptz.h" @@ -174,4 +176,67 @@ Status DataTypeTimeStampTzSerDe::write_column_to_mysql_binary(const IColumn& col return Status::OK(); } +Status DataTypeTimeStampTzSerDe::write_column_to_arrow(const IColumn& column, + const NullMap* null_map, + arrow::ArrayBuilder* array_builder, + int64_t start, int64_t end, + const cctz::time_zone& ctz) const { + const auto& col_data = assert_cast(column).get_data(); + auto& timestamp_builder = assert_cast(*array_builder); + std::shared_ptr timestamp_type = + std::static_pointer_cast(array_builder->type()); + static const auto& utc = cctz::utc_time_zone(); + for (size_t i = start; i < end; ++i) { + if (null_map && (*null_map)[i]) { + RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.AppendNull(), column.get_name(), + array_builder->type()->name())); + } else { + int64_t timestamp = 0; + DateV2Value datetime_val = + binary_cast>(col_data[i]); + datetime_val.unix_timestamp(×tamp, utc); + + if (_scale > 3) { + uint32_t microsecond = datetime_val.microsecond(); + timestamp = (timestamp * 1000000) + microsecond; + } else if (_scale > 0) { + uint32_t millisecond = datetime_val.microsecond() / 1000; + timestamp = (timestamp * 1000) + millisecond; + } + RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.Append(timestamp), column.get_name(), + array_builder->type()->name())); + } + } + return Status::OK(); +} + +Status DataTypeTimeStampTzSerDe::write_column_to_orc(const std::string& timezone, + const IColumn& column, const NullMap* null_map, + orc::ColumnVectorBatch* orc_col_batch, + int64_t start, int64_t end, + vectorized::Arena& arena, + const FormatOptions& options) const { + const auto& col_data = assert_cast(column).get_data(); + auto* cur_batch = dynamic_cast(orc_col_batch); + static const int64_t micro_to_nano_second = 1000; + static const auto& utc = cctz::utc_time_zone(); + for (size_t row_id = start; row_id < end; row_id++) { + if (cur_batch->notNull[row_id] == 0) { + continue; + } + + int64_t timestamp = 0; + DateV2Value datetime_val = + binary_cast>(col_data[row_id]); + if (!datetime_val.unix_timestamp(×tamp, utc)) { + return Status::InternalError("get unix timestamp error."); + } + + cur_batch->data[row_id] = timestamp; + cur_batch->nanoseconds[row_id] = datetime_val.microsecond() * micro_to_nano_second; + } + cur_batch->numElements = end - start; + return Status::OK(); +} + } // namespace doris::vectorized diff --git a/be/src/vec/data_types/serde/data_type_timestamptz_serde.h b/be/src/vec/data_types/serde/data_type_timestamptz_serde.h index 29486b9ff1d198..b51c9cbadf8957 100644 --- a/be/src/vec/data_types/serde/data_type_timestamptz_serde.h +++ b/be/src/vec/data_types/serde/data_type_timestamptz_serde.h @@ -63,6 +63,15 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe(orc_type->getMaximumLength())); case orc::TypeKind::TIMESTAMP_INSTANT: + if (_scan_params.__isset.enable_mapping_timestamp_tz && + _scan_params.enable_mapping_timestamp_tz) { + return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_TIMESTAMPTZ, + true, 0, 6); + } return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, true, 0, 6); case orc::TypeKind::LIST: { @@ -1853,6 +1858,8 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, // : std::make_unique(capacity, memoryPool); return _decode_string_column(col_name, data_column, orc_column_type->getKind(), cvb, num_values); + case PrimitiveType::TYPE_TIMESTAMPTZ: + return _decode_timestamp_tz_column(col_name, data_column, cvb, num_values); case PrimitiveType::TYPE_ARRAY: { if (orc_column_type->getKind() != orc::TypeKind::LIST) { return Status::InternalError( @@ -2054,7 +2061,8 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name, default: break; } - return Status::InternalError("Unsupported type for column '{}'", col_name); + return Status::InternalError("Unsupported type {} for column '{}'", data_type->get_name(), + col_name); } template diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h index 617b0b0d447565..5c7aa3dad5439f 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.h +++ b/be/src/vec/exec/format/orc/vorc_reader.h @@ -42,6 +42,7 @@ #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/sargs/Literal.hh" +#include "runtime/primitive_type.h" #include "util/runtime_profile.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column_array.h" @@ -552,6 +553,40 @@ class OrcReader : public GenericReader { return Status::OK(); } + template + Status _decode_timestamp_tz_column(const std::string& col_name, + const MutableColumnPtr& data_column, + const orc::ColumnVectorBatch* cvb, size_t num_values) { + SCOPED_RAW_TIMER(&_statistics.decode_value_time); + const auto* data = dynamic_cast(cvb); + if (data == nullptr) { + return Status::InternalError( + "Wrong data type for timestamp_tz column '{}', expected {}", col_name, + cvb->toString()); + } + auto& column_data = assert_cast(*data_column).get_data(); + auto origin_size = column_data.size(); + column_data.resize(origin_size + num_values); + UInt8* __restrict filter_data; + if constexpr (is_filter) { + filter_data = _filter->data(); + } + static const cctz::time_zone utc_time_zone = cctz::utc_time_zone(); + for (int i = 0; i < num_values; ++i) { + auto& v = reinterpret_cast&>( + column_data[origin_size + i]); + if constexpr (is_filter) { + if (!filter_data[i]) { + continue; + } + } + v.from_unixtime(data->data[i], utc_time_zone); + // nanoseconds will lose precision. only keep microseconds. + v.set_microsecond(data->nanoseconds[i] / 1000); + } + return Status::OK(); + } + template Status _decode_string_column(const std::string& col_name, const MutableColumnPtr& data_column, const orc::TypeKind& type_kind, const orc::ColumnVectorBatch* cvb, diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp index 41ef2057ea29ab..d834bbcc6ecbe5 100644 --- a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp +++ b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp @@ -268,6 +268,10 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv DCHECK(src_physical_type == tparquet::Type::BYTE_ARRAY) << src_physical_type; physical_converter = std::make_unique(); } + } else if (src_logical_primitive == TYPE_TIMESTAMPTZ) { + DCHECK(src_physical_type == tparquet::Type::INT64) << src_physical_type; + DCHECK(parquet_schema.logicalType.__isset.TIMESTAMP) << parquet_schema.name; + physical_converter = std::make_unique(); } else { physical_converter = std::make_unique(src_physical_type, src_logical_type); diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.h b/be/src/vec/exec/format/parquet/parquet_column_convert.h index 888d890dd9d37f..bcee0436d0dd0c 100644 --- a/be/src/vec/exec/format/parquet/parquet_column_convert.h +++ b/be/src/vec/exec/format/parquet/parquet_column_convert.h @@ -677,6 +677,31 @@ struct Int64ToTimestamp : public PhysicalToLogicalConverter { } }; +struct Int64ToTimestampTz : public PhysicalToLogicalConverter { + Status physical_convert(ColumnPtr& src_physical_col, ColumnPtr& src_logical_column) override { + ColumnPtr src_col = remove_nullable(src_physical_col); + MutableColumnPtr dst_col = remove_nullable(src_logical_column)->assume_mutable(); + + size_t rows = src_col->size(); + size_t start_idx = dst_col->size(); + dst_col->resize(start_idx + rows); + + const auto& src_data = assert_cast(src_col.get())->get_data(); + auto& dest_data = assert_cast(dst_col.get())->get_data(); + static const cctz::time_zone utc = cctz::utc_time_zone(); + + for (int i = 0; i < rows; i++) { + int64_t x = src_data[i]; + auto& num = dest_data[start_idx + i]; + auto& value = reinterpret_cast&>(num); + value.from_unixtime(x / _convert_params->second_mask, utc); + value.set_microsecond((x % _convert_params->second_mask) * + (_convert_params->scale_to_nano_factor / 1000)); + } + return Status::OK(); + } +}; + struct Int96toTimestamp : public PhysicalToLogicalConverter { Status physical_convert(ColumnPtr& src_physical_col, ColumnPtr& src_logical_column) override { ColumnPtr src_col = remove_nullable(src_physical_col); diff --git a/be/src/vec/exec/format/parquet/parquet_thrift_util.h b/be/src/vec/exec/format/parquet/parquet_thrift_util.h index 3b23ef7f76a70d..58b6828535c6ab 100644 --- a/be/src/vec/exec/format/parquet/parquet_thrift_util.h +++ b/be/src/vec/exec/format/parquet/parquet_thrift_util.h @@ -39,7 +39,8 @@ constexpr size_t INIT_META_SIZE = 48 * 1024; // 48k static Status parse_thrift_footer(io::FileReaderSPtr file, std::unique_ptr* file_metadata, size_t* meta_size, - io::IOContext* io_ctx, const bool enable_mapping_varbinary) { + io::IOContext* io_ctx, const bool enable_mapping_varbinary, + const bool enable_mapping_timestamp_tz) { size_t file_size = file->size(); size_t bytes_read = std::min(file_size, INIT_META_SIZE); std::vector footer(bytes_read); @@ -81,7 +82,8 @@ static Status parse_thrift_footer(io::FileReaderSPtr file, // deserialize footer RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata)); *file_metadata = std::make_unique(t_metadata, metadata_size); - RETURN_IF_ERROR((*file_metadata)->init_schema(enable_mapping_varbinary)); + RETURN_IF_ERROR( + (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); *meta_size = PARQUET_FOOTER_SIZE + metadata_size; return Status::OK(); } diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp index e9062753f57448..fbea6948ba1fe8 100644 --- a/be/src/vec/exec/format/parquet/schema_desc.cpp +++ b/be/src/vec/exec/format/parquet/schema_desc.cpp @@ -301,6 +301,15 @@ std::pair FieldDescriptor::convert_to_doris_type( } else if (logicalType.__isset.TIME) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); } else if (logicalType.__isset.TIMESTAMP) { + if (_enable_mapping_timestamp_tz) { + if (logicalType.TIMESTAMP.isAdjustedToUTC) { + // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_TIMESTAMPTZ, nullable, 0, + logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + return ans; + } + } ans.first = DataTypeFactory::instance().create_data_type( TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); } else if (logicalType.__isset.JSON) { diff --git a/be/src/vec/exec/format/parquet/schema_desc.h b/be/src/vec/exec/format/parquet/schema_desc.h index 23cdee50ddf3c0..672059512d947a 100644 --- a/be/src/vec/exec/format/parquet/schema_desc.h +++ b/be/src/vec/exec/format/parquet/schema_desc.h @@ -91,6 +91,7 @@ class FieldDescriptor { size_t _next_schema_pos; // useful for parse_node_field to decide whether to convert byte_array to VARBINARY type bool _enable_mapping_varbinary = false; + bool _enable_mapping_timestamp_tz = false; private: void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, @@ -164,6 +165,7 @@ class FieldDescriptor { const FieldSchema* find_column_by_id(uint64_t column_id) const; void set_enable_mapping_varbinary(bool enable) { _enable_mapping_varbinary = enable; } + void set_enable_mapping_timestamp_tz(bool enable) { _enable_mapping_timestamp_tz = enable; } }; #include "common/compile_check_end.h" diff --git a/be/src/vec/exec/format/parquet/vparquet_file_metadata.cpp b/be/src/vec/exec/format/parquet/vparquet_file_metadata.cpp index fcbece254fdf7c..5578e698a8afa4 100644 --- a/be/src/vec/exec/format/parquet/vparquet_file_metadata.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_file_metadata.cpp @@ -38,11 +38,13 @@ FileMetaData::~FileMetaData() { ExecEnv::GetInstance()->parquet_meta_tracker()->release(_mem_size); } -Status FileMetaData::init_schema(const bool enable_mapping_varbinary) { +Status FileMetaData::init_schema(const bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { if (_metadata.schema[0].num_children <= 0) { return Status::Corruption("Invalid parquet schema"); } _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); + _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); return _schema.parse_from_thrift(_metadata.schema); } diff --git a/be/src/vec/exec/format/parquet/vparquet_file_metadata.h b/be/src/vec/exec/format/parquet/vparquet_file_metadata.h index 6ad5ad7cea917f..19735e4fab86a8 100644 --- a/be/src/vec/exec/format/parquet/vparquet_file_metadata.h +++ b/be/src/vec/exec/format/parquet/vparquet_file_metadata.h @@ -29,7 +29,7 @@ class FileMetaData { public: FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size); ~FileMetaData(); - Status init_schema(const bool enable_mapping_varbinary); + Status init_schema(const bool enable_mapping_varbinary, const bool enable_mapping_timestamp_tz); const FieldDescriptor& schema() const { return _schema; } FieldDescriptor& schema() { return _schema; } const tparquet::FileMetaData& to_thrift() const; diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp index 10621f0ca5f5e2..85a6fd4e199bc7 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp @@ -261,10 +261,14 @@ Status ParquetReader::_open_file() { bool enable_mapping_varbinary = _scan_params.__isset.enable_mapping_varbinary ? _scan_params.enable_mapping_varbinary : false; + bool enable_mapping_timestamp_tz = _scan_params.__isset.enable_mapping_timestamp_tz + ? _scan_params.enable_mapping_timestamp_tz + : false; if (_meta_cache == nullptr) { // wrap _file_metadata with unique ptr, so that it can be released finally. RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, - &meta_size, _io_ctx, enable_mapping_varbinary)); + &meta_size, _io_ctx, enable_mapping_varbinary, + enable_mapping_timestamp_tz)); _file_metadata = _file_metadata_ptr.get(); // parse magic number & parse meta data _reader_statistics.file_footer_read_calls += 1; @@ -273,7 +277,8 @@ Status ParquetReader::_open_file() { FileMetaCache::get_key(_tracing_file_reader, _file_description); if (!_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, - &meta_size, _io_ctx, enable_mapping_varbinary)); + &meta_size, _io_ctx, enable_mapping_varbinary, + enable_mapping_timestamp_tz)); // _file_metadata_ptr.release() : move control of _file_metadata to _meta_cache_handle _meta_cache->insert(file_meta_cache_key, _file_metadata_ptr.release(), &_meta_cache_handle); diff --git a/be/src/vec/exec/format/table/parquet_metadata_reader.cpp b/be/src/vec/exec/format/table/parquet_metadata_reader.cpp index 144371a9f50f05..6333ba6aab9ed0 100644 --- a/be/src/vec/exec/format/table/parquet_metadata_reader.cpp +++ b/be/src/vec/exec/format/table/parquet_metadata_reader.cpp @@ -864,7 +864,8 @@ Status ParquetMetadataReader::_append_file_rows(const std::string& path, std::unique_ptr file_metadata; size_t meta_size = 0; io::IOContext io_ctx; - RETURN_IF_ERROR(parse_thrift_footer(file_reader, &file_metadata, &meta_size, &io_ctx, false)); + RETURN_IF_ERROR( + parse_thrift_footer(file_reader, &file_metadata, &meta_size, &io_ctx, true, true)); if (_mode_handler == nullptr) { return Status::InternalError( diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index 57210b4e17d6d3..0d7c348528ba6f 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -497,6 +497,10 @@ std::string JniConnector::get_jni_type(const DataTypePtr& data_type) { buffer << "datetimev2(" << type->get_scale() << ")"; return buffer.str(); } + case TYPE_TIMESTAMPTZ: { + buffer << "timestamptz(" << type->get_scale() << ")"; + return buffer.str(); + } case TYPE_BINARY: return "binary"; case TYPE_DECIMALV2: { @@ -588,6 +592,10 @@ std::string JniConnector::get_jni_type_with_different_string(const DataTypePtr& buffer << "datetimev2(" << data_type->get_scale() << ")"; return buffer.str(); } + case TYPE_TIMESTAMPTZ: { + buffer << "timestamptz(" << data_type->get_scale() << ")"; + return buffer.str(); + } case TYPE_BINARY: return "binary"; case TYPE_CHAR: { diff --git a/be/src/vec/runtime/vorc_transformer.cpp b/be/src/vec/runtime/vorc_transformer.cpp index 373d9c7c84c3f6..a42a8eb84d48e3 100644 --- a/be/src/vec/runtime/vorc_transformer.cpp +++ b/be/src/vec/runtime/vorc_transformer.cpp @@ -263,6 +263,10 @@ std::unique_ptr VOrcTransformer::_build_orc_type( type = orc::createPrimitiveType(orc::BINARY); break; } + case TYPE_TIMESTAMPTZ: { + type = orc::createPrimitiveType(orc::TIMESTAMP_INSTANT); + break; + } case TYPE_STRUCT: { type = orc::createStructType(); const auto* type_struct = diff --git a/be/test/vec/exec/format/parquet/parquet_expr_test.cpp b/be/test/vec/exec/format/parquet/parquet_expr_test.cpp index f7182b93a87475..1996bef6cc70ce 100644 --- a/be/test/vec/exec/format/parquet/parquet_expr_test.cpp +++ b/be/test/vec/exec/format/parquet/parquet_expr_test.cpp @@ -288,7 +288,7 @@ class ParquetExprTest : public testing::Test { size_t meta_size; static_cast(parse_thrift_footer(p_reader->_file_reader, &doris_file_metadata, - &meta_size, nullptr, false)); + &meta_size, nullptr, false, false)); doris_metadata = doris_file_metadata->to_thrift(); p_reader->_ctz = &ctz; diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp index 6ae25f936aa3ab..1d9c62ebfad851 100644 --- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp @@ -80,7 +80,7 @@ TEST_F(ParquetThriftReaderTest, normal) { std::unique_ptr meta_data; size_t meta_size; - static_cast(parse_thrift_footer(reader, &meta_data, &meta_size, nullptr, true)); + static_cast(parse_thrift_footer(reader, &meta_data, &meta_size, nullptr, true, true)); tparquet::FileMetaData t_metadata = meta_data->to_thrift(); LOG(WARNING) << "====================================="; @@ -113,7 +113,7 @@ TEST_F(ParquetThriftReaderTest, complex_nested_file) { std::unique_ptr metadata; size_t meta_size; - static_cast(parse_thrift_footer(reader, &metadata, &meta_size, nullptr, true)); + static_cast(parse_thrift_footer(reader, &metadata, &meta_size, nullptr, true, true)); tparquet::FileMetaData t_metadata = metadata->to_thrift(); FieldDescriptor schemaDescriptor; static_cast(schemaDescriptor.parse_from_thrift(t_metadata.schema)); @@ -401,7 +401,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file, std::unique_ptr metadata; size_t meta_size; - static_cast(parse_thrift_footer(reader, &metadata, &meta_size, nullptr, true)); + static_cast(parse_thrift_footer(reader, &metadata, &meta_size, nullptr, true, true)); tparquet::FileMetaData t_metadata = metadata->to_thrift(); FieldDescriptor schema_descriptor; static_cast(schema_descriptor.parse_from_thrift(t_metadata.schema)); diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/bucket-0/data-0ef44e1e-2bc4-417a-a262-fd8b66fb5a67-0.parquet b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/bucket-0/data-0ef44e1e-2bc4-417a-a262-fd8b66fb5a67-0.parquet new file mode 100644 index 00000000000000..f3bf7de5be2fb3 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/bucket-0/data-0ef44e1e-2bc4-417a-a262-fd8b66fb5a67-0.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/bucket-0/data-7855782c-3776-4332-8288-efa84aad3144-0.parquet b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/bucket-0/data-7855782c-3776-4332-8288-efa84aad3144-0.parquet new file mode 100644 index 00000000000000..3123e9fc06bf89 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/bucket-0/data-7855782c-3776-4332-8288-efa84aad3144-0.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-46ccff55-ca24-4577-8249-bff2967480fd-0 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-46ccff55-ca24-4577-8249-bff2967480fd-0 new file mode 100644 index 00000000000000..e82becd7ffc5cf Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-46ccff55-ca24-4577-8249-bff2967480fd-0 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-df8f8bcc-8680-48fd-8484-40d88e17e834-0 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-df8f8bcc-8680-48fd-8484-40d88e17e834-0 new file mode 100644 index 00000000000000..892f7c4c65e881 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-df8f8bcc-8680-48fd-8484-40d88e17e834-0 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-0 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-0 new file mode 100644 index 00000000000000..8f9d14dabac506 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-0 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-1 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-1 new file mode 100644 index 00000000000000..92f74aef4c49e7 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-1 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-0 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-0 new file mode 100644 index 00000000000000..327da8b3f4f92f Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-0 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-1 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-1 new file mode 100644 index 00000000000000..bf9ed2e4e437aa Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/manifest/manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-1 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/schema/schema-0 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/schema/schema-0 new file mode 100644 index 00000000000000..34820cdb48b042 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/schema/schema-0 @@ -0,0 +1,19 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "ts_ltz", + "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE" + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { }, + "comment" : "", + "timeMillis" : 1767687184848 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/EARLIEST b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/EARLIEST new file mode 100644 index 00000000000000..56a6051ca2b02b --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/LATEST b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/LATEST new file mode 100644 index 00000000000000..d8263ee9860594 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/LATEST @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/snapshot-1 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/snapshot-1 new file mode 100644 index 00000000000000..f5325e0b38cdae --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-0", + "deltaManifestList" : "manifest-list-b2b87161-8fcb-466b-afc0-d6000640181e-1", + "changelogManifestList" : null, + "commitUser" : "92bd74ff-540b-40e4-a033-4fdc7fb10821", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1767687193032, + "logOffsets" : { }, + "totalRecordCount" : 2, + "deltaRecordCount" : 2, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/snapshot-2 b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/snapshot-2 new file mode 100644 index 00000000000000..92789e26615c96 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/t_ltz/snapshot/snapshot-2 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "id" : 2, + "schemaId" : 0, + "baseManifestList" : "manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-0", + "deltaManifestList" : "manifest-list-a5224c7a-7873-4139-aebb-1592e62a21c5-1", + "changelogManifestList" : null, + "commitUser" : "e220749e-c9ee-4db7-a757-3bd82952a410", + "commitIdentifier" : 9223372036854775807, + "commitKind" : "APPEND", + "timeMillis" : 1767706744573, + "logOffsets" : { }, + "totalRecordCount" : 3, + "deltaRecordCount" : 1, + "changelogRecordCount" : 0, + "watermark" : -9223372036854775808 +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run25.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run25.sql new file mode 100644 index 00000000000000..f82bb0286a4b5d --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run25.sql @@ -0,0 +1,64 @@ +create database if not exists demo.test_timestamp_tz; +USE demo.test_timestamp_tz; + +DROP TABLE IF EXISTS test_ice_timestamp_tz_orc; +DROP TABLE IF EXISTS test_ice_timestamp_tz_parquet; + +SET spark.sql.session.timeZone = Asia/Shanghai; + +CREATE TABLE test_ice_timestamp_tz_orc ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING iceberg +TBLPROPERTIES( + 'write.format.default' = 'orc', + 'format-version' = '1' +); + +CREATE TABLE test_ice_timestamp_tz_parquet ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING iceberg +TBLPROPERTIES( + 'write.format.default' = 'parquet', + 'format-version' = '1' +); + +INSERT INTO test_ice_timestamp_tz_orc VALUES (1, TIMESTAMP_LTZ '2025-01-01 00:00:00'); +INSERT INTO test_ice_timestamp_tz_orc VALUES (2, TIMESTAMP_LTZ '2025-06-01 12:34:56.789'); +INSERT INTO test_ice_timestamp_tz_orc VALUES (3, TIMESTAMP_LTZ '2025-12-31 23:59:59.999999'); +INSERT INTO test_ice_timestamp_tz_orc VALUES (4, NULL); + + +INSERT INTO test_ice_timestamp_tz_parquet VALUES (1, TIMESTAMP_LTZ '2025-01-01 00:00:00'); +INSERT INTO test_ice_timestamp_tz_parquet VALUES (2, TIMESTAMP_LTZ '2025-06-01 12:34:56.789'); +INSERT INTO test_ice_timestamp_tz_parquet VALUES (3, TIMESTAMP_LTZ '2025-12-31 23:59:59.999999'); +INSERT INTO test_ice_timestamp_tz_parquet VALUES (4, NULL); + +SELECT * FROM test_ice_timestamp_tz_orc; +SELECT * FROM test_ice_timestamp_tz_parquet; + +DROP TABLE IF EXISTS test_ice_timestamp_tz_orc_write_with_mapping; +DROP TABLE IF EXISTS test_ice_timestamp_tz_parquet_write_with_mapping; + +CREATE TABLE test_ice_timestamp_tz_orc_write_with_mapping ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING iceberg +TBLPROPERTIES( + 'write.format.default' = 'orc', + 'format-version' = '1' +); + +CREATE TABLE test_ice_timestamp_tz_parquet_write_with_mapping ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING iceberg +TBLPROPERTIES( + 'write.format.default' = 'parquet', + 'format-version' = '1' +); diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run10.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run10.sql new file mode 100644 index 00000000000000..fa8f99618d44fb --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run10.sql @@ -0,0 +1,65 @@ +use paimon; +create database if not exists paimon_test_timestamp_tz; +USE paimon_test_timestamp_tz; + +DROP TABLE IF EXISTS test_ice_timestamp_tz_orc; +DROP TABLE IF EXISTS test_ice_timestamp_tz_parquet; + +SET spark.sql.session.timeZone = Asia/Shanghai; + +CREATE TABLE test_ice_timestamp_tz_orc ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING paimon +TBLPROPERTIES( + 'file.format' = 'orc', + 'primary-key' = 'id' +); + +CREATE TABLE test_ice_timestamp_tz_parquet ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING paimon +TBLPROPERTIES( + 'file.format' = 'parquet', + 'primary-key' = 'id' +); + +INSERT INTO test_ice_timestamp_tz_orc VALUES (1, TIMESTAMP_LTZ '2025-01-01 00:00:00'); +INSERT INTO test_ice_timestamp_tz_orc VALUES (2, TIMESTAMP_LTZ '2025-06-01 12:34:56.789'); +INSERT INTO test_ice_timestamp_tz_orc VALUES (3, TIMESTAMP_LTZ '2025-12-31 23:59:59.999999'); +INSERT INTO test_ice_timestamp_tz_orc VALUES (4, NULL); + + +INSERT INTO test_ice_timestamp_tz_parquet VALUES (1, TIMESTAMP_LTZ '2025-01-01 00:00:00'); +INSERT INTO test_ice_timestamp_tz_parquet VALUES (2, TIMESTAMP_LTZ '2025-06-01 12:34:56.789'); +INSERT INTO test_ice_timestamp_tz_parquet VALUES (3, TIMESTAMP_LTZ '2025-12-31 23:59:59.999999'); +INSERT INTO test_ice_timestamp_tz_parquet VALUES (4, NULL); + +SELECT * FROM test_ice_timestamp_tz_orc; +SELECT * FROM test_ice_timestamp_tz_parquet; + +DROP TABLE IF EXISTS test_ice_timestamp_tz_orc_write_with_mapping; +DROP TABLE IF EXISTS test_ice_timestamp_tz_parquet_write_with_mapping; + +CREATE TABLE test_ice_timestamp_tz_orc_write_with_mapping ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING paimon +TBLPROPERTIES( + 'file.format' = 'orc', + 'primary-key' = 'id' +); + +CREATE TABLE test_ice_timestamp_tz_parquet_write_with_mapping ( + id INT, + ts_tz TIMESTAMP_LTZ +) +USING paimon +TBLPROPERTIES( + 'file.format' = 'parquet', + 'primary-key' = 'id' +); \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/mysql/init/01-drop-db.sql b/docker/thirdparties/docker-compose/mysql/init/01-drop-db.sql index 272f4f546552da..2da54e8fac3d7b 100644 --- a/docker/thirdparties/docker-compose/mysql/init/01-drop-db.sql +++ b/docker/thirdparties/docker-compose/mysql/init/01-drop-db.sql @@ -18,3 +18,4 @@ drop database if exists doris_test; drop database if exists show_test_do_not_modify; drop database if exists test_varbinary_db; +drop database if exists test_timestamp_tz_db; diff --git a/docker/thirdparties/docker-compose/mysql/init/02-create-db.sql b/docker/thirdparties/docker-compose/mysql/init/02-create-db.sql index a22b654432add8..df6b34ab4b572f 100644 --- a/docker/thirdparties/docker-compose/mysql/init/02-create-db.sql +++ b/docker/thirdparties/docker-compose/mysql/init/02-create-db.sql @@ -21,3 +21,4 @@ create database Doris; create database doris; create database show_test_do_not_modify; create database test_varbinary_db; +create database test_timestamp_tz_db; diff --git a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql index 789dcc2080ab5d..09a983a3f0c915 100644 --- a/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/mysql/init/03-create-table.sql @@ -528,3 +528,10 @@ CREATE TABLE test_varbinary_db.`test_varbinary_udf` ( `varbinary_c` varbinary(100) ); +CREATE TABLE test_timestamp_tz_db.ts_test ( + id INT, + ts_ts TIMESTAMP NULL, + ts_dt DATETIME NULL +); + + diff --git a/docker/thirdparties/docker-compose/mysql/init/04-insert.sql b/docker/thirdparties/docker-compose/mysql/init/04-insert.sql index 0677c6e9096dea..02c1771a78ab78 100644 --- a/docker/thirdparties/docker-compose/mysql/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/mysql/init/04-insert.sql @@ -1212,6 +1212,15 @@ INSERT INTO doris_test.`test_cast` VALUES (2, '2', '2022-01-02', '2022-01-02 00: INSERT INTO test_varbinary_db.`test_varbinary` VALUES (1, X'48656C6C6F20576F726C64'), (2, X'48656C6C6F20576F726C6421'); INSERT INTO test_varbinary_db.`test_varbinary_udf` VALUES (1, X'48656C6C6F20576F726C64'), (2, X'48656C6C6F20576F726C6421'), (3, NULL), (4, X'AB'), (5, X'ABCDEF'); + + + + + +SET time_zone = '+08:00'; +INSERT INTO test_timestamp_tz_db.ts_test VALUES (1,'2025-01-01 12:00:00','2025-01-01 12:00:00'); +INSERT INTO test_timestamp_tz_db.ts_test VALUES (2,NULL,NULL); + ANALYZE TABLE Doris.doris; ANALYZE TABLE Doris.DORIS; ANALYZE TABLE Doris.Doris; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql index 7b1601ca302d2a..e6cdaa287a89cb 100644 --- a/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/oracle/init/03-create-table.sql @@ -263,3 +263,8 @@ CREATE TABLE doris_test.varbinary_test( "NAME" VARCHAR2(20), "BLOB_COL" BLOB ); + +CREATE TABLE doris_test.ltz_test ( + id NUMBER, + ts_ltz TIMESTAMP WITH LOCAL TIME ZONE +); diff --git a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql index 7dd0afe0bd9c05..8fb187fdd3200f 100644 --- a/docker/thirdparties/docker-compose/oracle/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/oracle/init/04-insert.sql @@ -159,4 +159,10 @@ INSERT INTO doris_test.varbinary_test VALUES (1, 'empty', EMPTY_BLOB()); INSERT INTO doris_test.varbinary_test VALUES (2, 'NULL', NULL); INSERT INTO doris_test.varbinary_test VALUES (3, 'normal', HEXTORAW('48656C6C6F20576F726C64')); + +ALTER SESSION SET TIME_ZONE = '+08:00'; +INSERT INTO doris_test.ltz_test VALUES (1,TIMESTAMP '2025-01-01 12:00:00 +08:00'); +INSERT INTO doris_test.ltz_test VALUES (NULL,NULL); + + commit; diff --git a/docker/thirdparties/docker-compose/postgresql/init/01-create-schema.sql b/docker/thirdparties/docker-compose/postgresql/init/01-create-schema.sql index 3e3c0c05901cbc..cf433264e76d44 100644 --- a/docker/thirdparties/docker-compose/postgresql/init/01-create-schema.sql +++ b/docker/thirdparties/docker-compose/postgresql/init/01-create-schema.sql @@ -18,3 +18,4 @@ create schema doris_test; create schema catalog_pg_test; create schema cdc_test; +create schema test_timestamp_tz_db; diff --git a/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql b/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql index 1279a1b9354e49..b3ba617d6c10ab 100644 --- a/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql +++ b/docker/thirdparties/docker-compose/postgresql/init/02-create-table.sql @@ -431,3 +431,10 @@ CREATE TABLE catalog_pg_test.extreme_test_multi_block ( uuid_val uuid ); +CREATE TABLE test_timestamp_tz_db.ts_test ( + id int, + ts_tz timestamptz NULL, + ts_ntz timestamp NULL +); + + diff --git a/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql b/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql index ccad42ab6fee74..4d25043838c871 100644 --- a/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/postgresql/init/04-insert.sql @@ -3020,3 +3020,8 @@ insert into catalog_pg_test.extreme_test_multi_block select * from catalog_pg_te insert into catalog_pg_test.extreme_test_multi_block select * from catalog_pg_test.extreme_test_multi_block; insert into catalog_pg_test.extreme_test_multi_block select * from catalog_pg_test.extreme_test_multi_block; insert into catalog_pg_test.extreme_test_multi_block select * from catalog_pg_test.extreme_test; + + +SET TIME ZONE 'Asia/Shanghai'; +INSERT INTO test_timestamp_tz_db.ts_test VALUES (1,'2025-01-01 12:00:00+08','2025-01-01 12:00:00'); +INSERT INTO test_timestamp_tz_db.ts_test VALUES (2,NULL,NULL); \ No newline at end of file diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java index 77c6fba37dc679..70070ee2cb80bb 100644 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/AvroColumnValue.java @@ -136,6 +136,11 @@ public LocalDateTime getDateTime() { return null; } + @Override + public LocalDateTime getTimeStampTz() { + return null; + } + @Override public byte[] getBytes() { return (byte[]) inspectObject(); diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java index ae0199d07d27c5..e69f7e4440758d 100644 --- a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java +++ b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java @@ -111,6 +111,11 @@ public byte getByte() { throw new UnsupportedOperationException("Hoodie type does not support tinyint"); } + @Override + public LocalDateTime getTimeStampTz() { + return ((Timestamp) fieldData).toLocalDateTime(); + } + @Override public BigDecimal getDecimal() { return ((HiveDecimal) inspectObject()).bigDecimalValue(); diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java index 70814f27f752a2..1e404809b7737d 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/main/java/org/apache/doris/iceberg/IcebergSysTableColumnValue.java @@ -140,6 +140,12 @@ public LocalDateTime getDateTime() { return LocalDateTime.ofInstant(instant, ZoneId.of(timezone)); } + @Override + public LocalDateTime getTimeStampTz() { + Instant instant = Instant.ofEpochMilli((((long) fieldData) / 1000)); + return LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); + } + @Override public byte[] getBytes() { // https://github.com/apache/iceberg/blob/8626ef5137024c1a69daaff97a832af6b0ae37ea/api/src/main/java/org/apache/iceberg/types/Type.java#L45C5-L45C30 diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java index ecf3dd670cf486..5460af27dca58d 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java @@ -118,6 +118,11 @@ public byte[] getStringAsBytes() { throw new UnsupportedOperationException(); } + @Override + public LocalDateTime getTimeStampTz() { + throw new UnsupportedOperationException(); + } + @Override public LocalDate getDate() { return LocalDate.now(); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index c39d2aa1a9f452..dfd40b1541e877 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -49,6 +49,7 @@ public enum Type { DATEV2(4), DATETIME(8), DATETIMEV2(8), + TIMESTAMPTZ(8), CHAR(-1), VARCHAR(-1), BINARY(-1), @@ -192,7 +193,8 @@ public boolean isDateTimeV2() { public boolean isPrimitive() { return type == Type.BOOLEAN || type == Type.BYTE || type == Type.TINYINT || type == Type.SMALLINT - || type == Type.INT || type == Type.BIGINT || type == Type.FLOAT || type == Type.DOUBLE; + || type == Type.INT || type == Type.BIGINT || type == Type.FLOAT || type == Type.DOUBLE + || type == Type.TIMESTAMPTZ; } public Type getType() { @@ -332,7 +334,14 @@ public static ColumnType parseType(String columnName, String hiveType) { type = Type.VARBINARY; break; default: - if (lowerCaseType.startsWith("timestamp") + if (lowerCaseType.startsWith("timestamptz")) { + type = Type.TIMESTAMPTZ; + precision = 6; // default + Matcher match = digitPattern.matcher(lowerCaseType); + if (match.find()) { + precision = Integer.parseInt(match.group(1).trim()); + } + } else if (lowerCaseType.startsWith("timestamp") || lowerCaseType.startsWith("datetime") || lowerCaseType.startsWith("datetimev2")) { type = Type.DATETIMEV2; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java index 640fd775c98013..8911a19f0bd5bd 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java @@ -73,6 +73,8 @@ default boolean canGetCharAsBytes() { LocalDateTime getDateTime(); + LocalDateTime getTimeStampTz(); + byte[] getBytes(); void unpackArray(List values); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java index e82f05c7d0a367..e38d860d6d01e7 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java @@ -198,6 +198,11 @@ public LocalDateTime getDateTime() { return LocalDateTime.now(); } + @Override + public LocalDateTime getTimeStampTz() { + throw new UnsupportedOperationException(); + } + @Override public byte[] getBytes() { return (byte[]) inspectObject(); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 594033a096f72c..36d2329da89976 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -447,6 +447,8 @@ public int appendNull(ColumnType.Type typeValue) { case DATETIME: case DATETIMEV2: return appendDateTime(LocalDateTime.MIN); + case TIMESTAMPTZ: + return appendTimeStampTz(LocalDateTime.MIN); case CHAR: case VARCHAR: case STRING: @@ -1164,6 +1166,50 @@ private void putDateTime(int rowId, LocalDateTime v) { OffHeap.putLong(null, data + rowId * 8L, time); } + public int appendTimeStampTz(LocalDateTime v) { + reserve(appendIndex + 1); + putTimeStampTz(appendIndex, v); + return appendIndex++; + } + + private void putTimeStampTz(int rowId, LocalDateTime v) { + // TimeStampTz use the same storage format as DateTimeV2 + long time = TypeNativeBytes.convertToDateTimeV2(v.getYear(), v.getMonthValue(), v.getDayOfMonth(), v.getHour(), + v.getMinute(), v.getSecond(), v.getNano() / 1000); + OffHeap.putLong(null, data + rowId * 8L, time); + } + + public LocalDateTime[] getTimeStampTzColumn(int start, int end) { + LocalDateTime[] result = new LocalDateTime[end - start]; + for (int i = start; i < end; ++i) { + if (!isNullAt(i)) { + result[i - start] = getTimeStampTz(i); + } + } + return result; + } + + public LocalDateTime getTimeStampTz(int rowId) { + long time = OffHeap.getLong(null, data + rowId * 8L); + return TypeNativeBytes.convertToJavaDateTimeV2(time); + } + + public void appendTimeStampTz(LocalDateTime[] batch, boolean isNullable) { + if (!isNullable) { + checkNullable(batch, batch.length); + } + reserve(appendIndex + batch.length); + for (LocalDateTime v : batch) { + if (v == null) { + putNull(appendIndex); + putTimeStampTz(appendIndex, LocalDateTime.MIN); + } else { + putTimeStampTz(appendIndex, v); + } + appendIndex++; + } + } + private void putBytes(int rowId, byte[] src, int offset, int length) { OffHeap.copyMemory(src, OffHeap.BYTE_ARRAY_OFFSET + offset, null, data + rowId, length); } @@ -1638,6 +1684,9 @@ public Object[] newObjectContainerArray(ColumnType.Type type, int size) { case DATETIME: case DATETIMEV2: return new LocalDateTime[size]; + case TIMESTAMPTZ: + // MAYBE use LocalDateTime is not appropriate + return new LocalDateTime[size]; case CHAR: case VARCHAR: case STRING: @@ -1698,6 +1747,10 @@ public void appendObjectColumn(Object[] batch, boolean isNullable) { case DATETIMEV2: appendDateTime((LocalDateTime[]) batch, isNullable); break; + case TIMESTAMPTZ: + // MAYBE use LocalDateTime is not appropriate + appendTimeStampTz((LocalDateTime[]) batch, isNullable); + break; case CHAR: case VARCHAR: case STRING: @@ -1762,6 +1815,8 @@ public Object[] getObjectColumn(int start, int end) { case DATETIME: case DATETIMEV2: return getDateTimeColumn(start, end); + case TIMESTAMPTZ: + return getTimeStampTzColumn(start, end); case CHAR: case VARCHAR: case STRING: @@ -1826,6 +1881,9 @@ public void appendValue(ColumnValue o) { case DATETIMEV2: appendDateTime(o.getDateTime()); break; + case TIMESTAMPTZ: + appendTimeStampTz(o.getTimeStampTz()); + break; case CHAR: if (o.canGetCharAsBytes()) { appendBytesAndOffset(o.getCharAsBytes()); @@ -1917,6 +1975,9 @@ public void dump(StringBuilder sb, int i) { case DATETIMEV2: sb.append(getDateTime(i)); break; + case TIMESTAMPTZ: + sb.append(getTimeStampTz(i)); + break; case CHAR: case VARCHAR: case STRING: diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java index 3160ff6fd0f291..c9eb3aea507c09 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java @@ -622,6 +622,10 @@ private void insertColumn(int rowIdx, int colIdx, VectorColumn column) throws SQ preparedStatement.setTimestamp( parameterIndex, Timestamp.valueOf(column.getDateTime(rowIdx))); break; + case TIMESTAMPTZ: + preparedStatement.setObject( + parameterIndex, Timestamp.valueOf(column.getTimeStampTz(rowIdx))); + break; case CHAR: case VARCHAR: case STRING: @@ -675,6 +679,9 @@ private void insertNullColumn(int parameterIndex, ColumnType.Type dorisType) case DATETIMEV2: preparedStatement.setNull(parameterIndex, Types.TIMESTAMP); break; + case TIMESTAMPTZ: + preparedStatement.setNull(parameterIndex, Types.TIMESTAMP_WITH_TIMEZONE); + break; case CHAR: case VARCHAR: case STRING: diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java index 85d53991dd6ea8..266eecce933fef 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java @@ -167,6 +167,9 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla } return data; } + case TIMESTAMPTZ: { + return resultSet.getObject(columnIndex + 1, LocalDateTime.class); + } default: throw new IllegalArgumentException("Unsupported column type: " + type.getType()); } diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java index e46ef1e676ea93..c076fdf77e1e1d 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java @@ -103,6 +103,9 @@ private Object newGetColumnValue(int columnIndex, ColumnType type, String[] repl return resultSet.getObject(columnIndex + 1); case VARBINARY: return resultSet.getObject(columnIndex + 1, byte[].class); + case TIMESTAMPTZ: + Timestamp ts = resultSet.getObject(columnIndex + 1, Timestamp.class); + return ts == null ? null : LocalDateTime.ofInstant(ts.toInstant(), java.time.ZoneOffset.UTC); default: throw new IllegalArgumentException("Unsupported column type: " + type.getType()); } diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java index 1c0ecb6919b919..62fd54a8f673be 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/PostgreSQLJdbcExecutor.java @@ -47,7 +47,8 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int VectorTable outputTable) { for (int i = 0; i < columnCount; ++i) { if (outputTable.getColumnType(i).getType() == Type.DATETIME - || outputTable.getColumnType(i).getType() == Type.DATETIMEV2) { + || outputTable.getColumnType(i).getType() == Type.DATETIMEV2 + || outputTable.getColumnType(i).getType() == Type.TIMESTAMPTZ) { block.add(new Object[batchSizeNum]); } else if (outputTable.getColumnType(i).getType() == Type.STRING || outputTable.getColumnType(i).getType() == Type.ARRAY) { @@ -89,6 +90,13 @@ protected Object getColumnValue(int columnIndex, ColumnType type, String[] repla return resultSet.getObject(columnIndex + 1); case VARBINARY: return resultSet.getBytes(columnIndex + 1); + case TIMESTAMPTZ: + OffsetDateTime offsetDateTime = resultSet.getObject(columnIndex + 1, OffsetDateTime.class); + if (offsetDateTime == null) { + return null; + } else { + return Timestamp.from(offsetDateTime.toInstant()); + } case ARRAY: java.sql.Array array = resultSet.getArray(columnIndex + 1); return array == null ? null : convertArrayToList(array.getArray()); @@ -111,6 +119,14 @@ protected ColumnValueConverter getOutputConverter(ColumnType columnType, String return input; } }, LocalDateTime.class); + case TIMESTAMPTZ: + return createConverter(input -> { + if (input instanceof Timestamp) { + return LocalDateTime.ofInstant(((Timestamp) input).toInstant(), java.time.ZoneOffset.UTC); + } else { + return input; + } + }, LocalDateTime.class); case CHAR: return createConverter( input -> trimSpaces(input.toString()), String.class); diff --git a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java index ec6c40a1376cb0..dc326c3823fad0 100644 --- a/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java +++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java @@ -246,6 +246,11 @@ public LocalDateTime getDateTime() { return result; } + @Override + public LocalDateTime getTimeStampTz() { + return convertToLocalDateTime((TimeStampMicroTZVector) column, idx); + } + @Override public byte[] getBytes() { VarBinaryVector binaryCol = (VarBinaryVector) column; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index af8a13149e1001..d6db3d4d954f6e 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -149,6 +149,15 @@ public LocalDateTime getDateTime() { } } + @Override + public LocalDateTime getTimeStampTz() { + Timestamp ts = record.getTimestamp(idx, dorisType.getPrecision()); + LocalDateTime v = ts.toInstant() + .atZone(ZoneId.of("UTC")) + .toLocalDateTime(); + return v; + } + @Override public boolean isNull() { return record.isNullAt(idx); diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorColumnValue.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorColumnValue.java index 35a2308ea00b0a..12164043c0b724 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorColumnValue.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorColumnValue.java @@ -168,6 +168,12 @@ public LocalDateTime getDateTime() { return null; } + @Override + public LocalDateTime getTimeStampTz() { + Object o = trinoType.getObjectValue(connectorSession, block, position); + return ((SqlTimestampWithTimeZone) o).toZonedDateTime().toLocalDateTime(); + } + @Override public boolean isNull() { return block.isNull(position); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 2d3cd457130cfd..cc7ecf52c4e6a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -264,6 +264,9 @@ private String dorisTypeToOrcTypeMap(Type dorisType) throws AnalysisException { case DATETIMEV2: orcType = "timestamp"; break; + case TIMESTAMPTZ: + orcType = "timestamp with local time zone"; + break; case CHAR: orcType = "char(" + dorisType.getLength() + ")"; break; @@ -387,6 +390,10 @@ private void analyzeForOrcFormat(List resultExprs, List colLabels) case DATETIMEV2: checkOrcType(schema.second, "timestamp", true, resultType.getPrimitiveType().toString()); break; + case TIMESTAMPTZ: + checkOrcType(schema.second, "timestamp with local time zone", true, + resultType.getPrimitiveType().toString()); + break; case CHAR: checkOrcType(schema.second, "char", false, resultType.getPrimitiveType().toString()); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index c9d21e7174db84..bc5aef34a123ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -135,7 +135,8 @@ public class JdbcResource extends Resource { CONNECTION_POOL_KEEP_ALIVE, TEST_CONNECTION, ExternalCatalog.USE_META_CACHE, - CatalogProperty.ENABLE_MAPPING_VARBINARY + CatalogProperty.ENABLE_MAPPING_VARBINARY, + CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ ).build(); // The default value of optional properties @@ -157,6 +158,7 @@ public class JdbcResource extends Resource { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.USE_META_CACHE, String.valueOf(ExternalCatalog.DEFAULT_USE_META_CACHE)); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CatalogProperty.ENABLE_MAPPING_VARBINARY, "false"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ, "false"); } // timeout for both connection and read. 10 seconds is long enough. diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java index 3a77e792acd321..774ee4e6e838f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java @@ -49,6 +49,7 @@ public class FileFormatConstants { public static final String PROP_PATH_PARTITION_KEYS = "path_partition_keys"; public static final String PROP_ENCLOSE = "enclose"; public static final String PROP_ENABLE_MAPPING_VARBINARY = "enable_mapping_varbinary"; + public static final String PROP_ENABLE_MAPPING_TIMESTAMP_TZ = "enable_mapping_timestamp_tz"; // decimal(p,s) public static final Pattern DECIMAL_TYPE_PATTERN = Pattern.compile("decimal\\((\\d+),(\\d+)\\)"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java index 0ad7b66c39bf49..0f118251197f30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java @@ -45,6 +45,8 @@ public class CatalogProperty { // Default: false, mapping BINARY types to STRING for compatibility public static final String ENABLE_MAPPING_VARBINARY = "enable.mapping.varbinary"; + // Default: false, mapping TIMESTAMP_TZ types to DATETIME for compatibility + public static final String ENABLE_MAPPING_TIMESTAMP_TZ = "enable.mapping.timestamp_tz"; @Deprecated @SerializedName(value = "resource") @@ -111,6 +113,21 @@ public void setEnableMappingVarbinary(boolean enable) { addProperty(ENABLE_MAPPING_VARBINARY, String.valueOf(enable)); } + /** + * @return true if timestamp_tz mapping is enabled, false otherwise + */ + public boolean getEnableMappingTimestampTz() { + return Boolean.parseBoolean(getOrDefault(ENABLE_MAPPING_TIMESTAMP_TZ, "false")); + } + + /** + * Set enable mapping timestamp_tz property. + * @param enable true to enable timestamp_tz mapping, false to disable + */ + public void setEnableMappingTimestampTz(boolean enable) { + addProperty(ENABLE_MAPPING_TIMESTAMP_TZ, String.valueOf(enable)); + } + public void modifyCatalogProps(Map props) { synchronized (this) { properties.putAll(props); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index c0e89a3978ae93..39188224e388f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -127,7 +127,8 @@ public abstract class ExternalCatalog public static final Set HIDDEN_PROPERTIES = Sets.newHashSet( CREATE_TIME, USE_META_CACHE, - CatalogProperty.ENABLE_MAPPING_VARBINARY); + CatalogProperty.ENABLE_MAPPING_VARBINARY, + CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ); protected static final int ICEBERG_CATALOG_EXECUTOR_THREAD_NUM = Runtime.getRuntime().availableProcessors(); @@ -246,12 +247,19 @@ public void setDefaultPropsIfMissing(boolean isReplay) { if (catalogProperty.getOrDefault(CatalogProperty.ENABLE_MAPPING_VARBINARY, "").isEmpty()) { catalogProperty.setEnableMappingVarbinary(false); } + if (catalogProperty.getOrDefault(CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ, "").isEmpty()) { + catalogProperty.setEnableMappingTimestampTz(false); + } } public boolean getEnableMappingVarbinary() { return catalogProperty.getEnableMappingVarbinary(); } + public boolean getEnableMappingTimestampTz() { + return catalogProperty.getEnableMappingTimestampTz(); + } + // we need check auth fallback for kerberos or simple public boolean ifNotSetFallbackToSimpleAuth() { return catalogProperty.getOrDefault(DFSFileSystem.PROP_ALLOW_FALLBACK_TO_SIMPLE_AUTH, "").isEmpty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 2614006b6a1185..f54ff4734a875d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -163,6 +163,7 @@ protected void initSchemaParams() throws UserException { params.setSrcTupleId(-1); // Set enable_mapping_varbinary from catalog or TVF params.setEnableMappingVarbinary(getEnableMappingVarbinary()); + params.setEnableMappingTimestampTz(getEnableMappingTimestampTz()); } private void updateRequiredSlots() throws UserException { @@ -579,6 +580,30 @@ protected boolean getEnableMappingVarbinary() { return false; } + protected boolean getEnableMappingTimestampTz() { + try { + TableIf table = getTargetTable(); + // For External Catalog tables get from catalog properties + if (table instanceof ExternalTable) { + ExternalTable externalTable = (ExternalTable) table; + CatalogIf catalog = externalTable.getCatalog(); + if (catalog instanceof ExternalCatalog) { + return ((ExternalCatalog) catalog).getEnableMappingTimestampTz(); + } + } + // For TVF read directly from fileFormatProperties + if (table instanceof FunctionGenTable) { + FunctionGenTable functionGenTable = (FunctionGenTable) table; + ExternalFileTableValuedFunction tvf = (ExternalFileTableValuedFunction) functionGenTable.getTvf(); + return tvf.fileFormatProperties.enableMappingTimestampTz; + } + } catch (Exception e) { + LOG.info("Failed to get enable_mapping_timestamp_tz from catalog, use default value false. Error: {}", + e.getMessage()); + } + return false; + } + protected abstract List getPathPartitionKeys() throws UserException; protected abstract TableIf getTargetTable() throws UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 47ed5ff6a6a8d3..e2ad32ebeeef27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -724,7 +724,8 @@ private Optional initHiveSchema() { String fieldName = field.getName().toLowerCase(Locale.ROOT); String defaultValue = colDefaultValues.getOrDefault(fieldName, null); columns.add(new Column(fieldName, - HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType(), catalog.getEnableMappingVarbinary()), + HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType(), catalog.getEnableMappingVarbinary(), + catalog.getEnableMappingTimestampTz()), true, null, true, defaultValue, field.getComment(), true, -1)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java index 5ca5c7f24cf782..58d6789b59cc9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java @@ -614,15 +614,17 @@ public static String dorisTypeToHiveType(Type dorisType) { /** * Convert hive type to doris type. */ - public static Type hiveTypeToDorisType(String hiveType, boolean enableMappingVarbinary) { + public static Type hiveTypeToDorisType(String hiveType, boolean enableMappingVarbinary, + boolean enableMappingTimeStampTz) { // use the largest scale as default time scale. - return hiveTypeToDorisType(hiveType, 6, enableMappingVarbinary); + return hiveTypeToDorisType(hiveType, 6, enableMappingVarbinary, enableMappingTimeStampTz); } /** * Convert hive type to doris type with timescale. */ - public static Type hiveTypeToDorisType(String hiveType, int timeScale, boolean enableMappingVarbinary) { + public static Type hiveTypeToDorisType(String hiveType, int timeScale, boolean enableMappingVarbinary, + boolean enableMappingTimeStampTz) { String lowerCaseType = hiveType.toLowerCase(); switch (lowerCaseType) { case "boolean": @@ -655,7 +657,7 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale, boolean e if (lowerCaseType.startsWith("array")) { if (lowerCaseType.indexOf("<") == 5 && lowerCaseType.lastIndexOf(">") == lowerCaseType.length() - 1) { Type innerType = hiveTypeToDorisType(lowerCaseType.substring(6, lowerCaseType.length() - 1), - enableMappingVarbinary); + enableMappingVarbinary, enableMappingTimeStampTz); return ArrayType.create(innerType, true); } } @@ -665,8 +667,11 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale, boolean e String keyValue = lowerCaseType.substring(4, lowerCaseType.length() - 1); int index = findNextNestedField(keyValue); if (index != keyValue.length() && index != 0) { - return new MapType(hiveTypeToDorisType(keyValue.substring(0, index), enableMappingVarbinary), - hiveTypeToDorisType(keyValue.substring(index + 1), enableMappingVarbinary)); + return new MapType( + hiveTypeToDorisType(keyValue.substring(0, index), enableMappingVarbinary, + enableMappingTimeStampTz), + hiveTypeToDorisType(keyValue.substring(index + 1), enableMappingVarbinary, + enableMappingTimeStampTz)); } } } @@ -680,7 +685,8 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale, boolean e int pivot = listFields.indexOf(':'); if (pivot > 0 && pivot < listFields.length() - 1) { fields.add(new StructField(listFields.substring(0, pivot), - hiveTypeToDorisType(listFields.substring(pivot + 1, index), enableMappingVarbinary))); + hiveTypeToDorisType(listFields.substring(pivot + 1, index), enableMappingVarbinary, + enableMappingTimeStampTz))); listFields = listFields.substring(Math.min(index + 1, listFields.length())); } else { break; @@ -717,6 +723,10 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale, boolean e } return ScalarType.createDecimalV3Type(precision, scale); } + if (lowerCaseType.startsWith("timestamp with local time zone")) { + return enableMappingTimeStampTz ? ScalarType.createTimeStampTzType(timeScale) + : ScalarType.createDatetimeV2Type(timeScale); + } return Type.UNSUPPORTED; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index c381239bc0cc8c..0eef3764566a34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -557,7 +557,7 @@ public static PartitionSpec solveIcebergPartitionSpec(PartitionDesc partitionDes } private static Type icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Type.PrimitiveType primitive, - boolean enableMappingVarbinary) { + boolean enableMappingVarbinary, boolean enableMappingTimestampTz) { switch (primitive.typeId()) { case BOOLEAN: return Type.BOOLEAN; @@ -586,6 +586,9 @@ private static Type icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Typ case DATE: return ScalarType.createDateV2Type(); case TIMESTAMP: + if (enableMappingTimestampTz && ((TimestampType) primitive).shouldAdjustToUTC()) { + return ScalarType.createTimeStampTzType(ICEBERG_DATETIME_SCALE_MS); + } return ScalarType.createDatetimeV2Type(ICEBERG_DATETIME_SCALE_MS); case TIME: return Type.UNSUPPORTED; @@ -594,24 +597,28 @@ private static Type icebergPrimitiveTypeToDorisType(org.apache.iceberg.types.Typ } } - public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type, boolean enableMappingVarbinary) { + public static Type icebergTypeToDorisType(org.apache.iceberg.types.Type type, boolean enableMappingVarbinary, + boolean enableMappingTimestampTz) { if (type.isPrimitiveType()) { return icebergPrimitiveTypeToDorisType((org.apache.iceberg.types.Type.PrimitiveType) type, - enableMappingVarbinary); + enableMappingVarbinary, enableMappingTimestampTz); } switch (type.typeId()) { case LIST: Types.ListType list = (Types.ListType) type; - return ArrayType.create(icebergTypeToDorisType(list.elementType(), enableMappingVarbinary), true); + return ArrayType.create( + icebergTypeToDorisType(list.elementType(), enableMappingVarbinary, enableMappingTimestampTz), + true); case MAP: Types.MapType map = (Types.MapType) type; return new MapType( - icebergTypeToDorisType(map.keyType(), enableMappingVarbinary), - icebergTypeToDorisType(map.valueType(), enableMappingVarbinary)); + icebergTypeToDorisType(map.keyType(), enableMappingVarbinary, enableMappingTimestampTz), + icebergTypeToDorisType(map.valueType(), enableMappingVarbinary, enableMappingTimestampTz)); case STRUCT: Types.StructType struct = (Types.StructType) type; ArrayList nestedTypes = struct.fields().stream().map( - x -> new StructField(x.name(), icebergTypeToDorisType(x.type(), enableMappingVarbinary))) + x -> new StructField(x.name(), + icebergTypeToDorisType(x.type(), enableMappingVarbinary, enableMappingTimestampTz))) .collect(Collectors.toCollection(ArrayList::new)); return new StructType(nestedTypes); default: @@ -939,7 +946,8 @@ private static List getSchema(ExternalTable dorisTable, long schemaId, b Preconditions.checkNotNull(schema, "Schema for " + type + " " + dorisTable.getCatalog().getName() + "." + dorisTable.getDbName() + "." + dorisTable.getName() + " is null"); - return parseSchema(schema, dorisTable.getCatalog().getEnableMappingVarbinary()); + return parseSchema(schema, dorisTable.getCatalog().getEnableMappingVarbinary(), + dorisTable.getCatalog().getEnableMappingTimestampTz()); }); } catch (Exception e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); @@ -950,13 +958,15 @@ private static List getSchema(ExternalTable dorisTable, long schemaId, b /** * Parse iceberg schema to doris schema */ - public static List parseSchema(Schema schema, boolean enableMappingVarbinary) { + public static List parseSchema(Schema schema, boolean enableMappingVarbinary, + boolean enableMappingTimestampTz) { List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { - Column column = new Column(field.name().toLowerCase(Locale.ROOT), - IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary), true, null, - true, field.doc(), true, -1); + Column column = new Column(field.name().toLowerCase(Locale.ROOT), + IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), + true, null, + true, field.doc(), true, -1); updateIcebergColumnUniqueId(column, field); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 817649fda746ed..0d23bb1cb43745 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -252,7 +252,8 @@ private JdbcClient createJdbcClient() { .setConnectionPoolMaxLifeTime(getConnectionPoolMaxLifeTime()) .setConnectionPoolMaxWaitTime(getConnectionPoolMaxWaitTime()) .setConnectionPoolKeepAlive(isConnectionPoolKeepAlive()) - .setEnableMappingVarbinary(getEnableMappingVarbinary()); + .setEnableMappingVarbinary(getEnableMappingVarbinary()) + .setEnableMappingTimestampTz(getEnableMappingTimestampTz()); return JdbcClient.createJdbcClient(jdbcClientConfig); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 0b2b81f4d0dbf9..7fa454d7813c5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -68,6 +68,7 @@ public abstract class JdbcClient { protected Map includeDatabaseMap; protected Map excludeDatabaseMap; protected boolean enableMappingVarbinary; + protected boolean enableMappingTimestampTz; public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -113,6 +114,7 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { initializeClassLoader(jdbcClientConfig); initializeDataSource(jdbcClientConfig); this.enableMappingVarbinary = jdbcClientConfig.isEnableMappingVarbinary(); + this.enableMappingTimestampTz = jdbcClientConfig.isEnableMappingTimestampTz(); } protected void setJdbcDriverSystemProperties() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java index 1b67e639ca6da8..2d13fbc964b6e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java @@ -43,6 +43,8 @@ public class JdbcClientConfig implements Cloneable { // Whether to enable mapping BINARY to doris VARBINARY // default: false, mapping to doris string type private boolean enableMappingVarbinary; + // default: false, mapping to doris datetime type + private boolean enableMappingTimestampTz; private Map includeDatabaseMap; private Map excludeDatabaseMap; @@ -67,6 +69,8 @@ public JdbcClientConfig() { this.customizedProperties = Maps.newHashMap(); this.enableMappingVarbinary = Boolean.parseBoolean( JdbcResource.getDefaultPropertyValue(CatalogProperty.ENABLE_MAPPING_VARBINARY)); + this.enableMappingTimestampTz = Boolean.parseBoolean( + JdbcResource.getDefaultPropertyValue(CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ)); } @Override @@ -241,6 +245,15 @@ public boolean isEnableMappingVarbinary() { return enableMappingVarbinary; } + public JdbcClientConfig setEnableMappingTimestampTz(boolean enableMappingTimestampTz) { + this.enableMappingTimestampTz = enableMappingTimestampTz; + return this; + } + + public boolean isEnableMappingTimestampTz() { + return enableMappingTimestampTz; + } + public void setCustomizedProperties(Map customizedProperties) { this.customizedProperties = customizedProperties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index 09208fd4c2157c..bc0112040a9f16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -277,7 +277,18 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { fieldSchema.setAllowNull(true); } return ScalarType.createDateV2Type(); - case "TIMESTAMP": + case "TIMESTAMP": { + int columnSize = fieldSchema.requiredColumnSize(); + int scale = columnSize > 19 ? columnSize - 20 : 0; + if (scale > 6) { + scale = 6; + } + if (convertDateToNull) { + fieldSchema.setAllowNull(true); + } + return enableMappingTimestampTz ? ScalarType.createTimeStampTzType(scale) + : ScalarType.createDatetimeV2Type(scale); + } case "DATETIME": { // mysql can support microsecond // use columnSize to calculate the precision of timestamp/datetime diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 6f0363685dff06..996608545cdb2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -155,15 +155,23 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { if (oracleType.startsWith("INTERVAL")) { oracleType = oracleType.substring(0, 8); } else if (oracleType.startsWith("TIMESTAMP")) { - if (oracleType.contains("TIME ZONE") || oracleType.contains("LOCAL TIME ZONE")) { - return Type.UNSUPPORTED; - } // oracle can support nanosecond, will lose precision int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { scale = 6; } - return ScalarType.createDatetimeV2Type(scale); + if (oracleType.contains("LOCAL TIME ZONE")) { + //TIMESTAMP(s) WITH LOCAL TIME ZONE + return enableMappingTimestampTz ? ScalarType.createTimeStampTzType(scale) + : ScalarType.createDatetimeV2Type(scale); + } else if (oracleType.contains("TIME ZONE")) { + //TIMESTAMP(s) WITH TIME ZONE + return Type.UNSUPPORTED; + } else { + //TIMESTAMP(s) + oracleType = "TIMESTAMP"; + return ScalarType.createDatetimeV2Type(scale); + } } switch (oracleType) { /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java index 099b7cc1437b09..5505c4340044e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java @@ -129,8 +129,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { return Type.DOUBLE; case "bpchar": return ScalarType.createCharType(fieldSchema.requiredColumnSize()); - case "timestamp": - case "timestamptz": { + case "timestamp": { // postgres can support microsecond int scale = fieldSchema.getDecimalDigits().orElse(0); if (scale > 6) { @@ -138,6 +137,14 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { } return ScalarType.createDatetimeV2Type(scale); } + case "timestamptz": { + int scale = fieldSchema.getDecimalDigits().orElse(0); + if (scale > 6) { + scale = 6; + } + return enableMappingTimestampTz ? ScalarType.createTimeStampTzType(scale) + : ScalarType.createDatetimeV2Type(scale); + } case "date": return ScalarType.createDateV2Type(); case "bool": diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 6b39413d1d4161..d50f9580e106b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -304,7 +304,9 @@ public Optional initSchema(SchemaCacheKey key) { List partitionColumns = Lists.newArrayList(); for (DataField field : columns) { Column column = new Column(field.name().toLowerCase(), - PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary()), true, + PaimonUtil.paimonTypeToDorisType(field.type(), getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz()), + true, null, true, field.description(), true, -1); PaimonUtil.updatePaimonColumnUniqueId(column, field); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 730ba2ff810b45..b9285b4a7578fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -195,7 +195,7 @@ public static ListPartitionItem toListPartitionItem(String partitionName, List fields = rowType.getFields(); return new org.apache.doris.catalog.StructType(fields.stream() .map(field -> new org.apache.doris.catalog.StructField(field.name(), - paimonTypeToDorisType(field.type(), enableVarbinaryMapping))) + paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping))) .collect(Collectors.toCollection(ArrayList::new))); case TIME_WITHOUT_TIME_ZONE: return Type.UNSUPPORTED; @@ -281,8 +286,9 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT } } - public static Type paimonTypeToDorisType(org.apache.paimon.types.DataType type, boolean enableVarbinaryMapping) { - return paimonPrimitiveTypeToDorisType(type, enableVarbinaryMapping); + public static Type paimonTypeToDorisType(org.apache.paimon.types.DataType type, boolean enableVarbinaryMapping, + boolean enableTimestampTzMapping) { + return paimonPrimitiveTypeToDorisType(type, enableVarbinaryMapping, enableTimestampTzMapping); } public static void updatePaimonColumnUniqueId(Column column, DataType dataType) { @@ -316,7 +322,8 @@ public static void updatePaimonColumnUniqueId(Column column, DataField field) { updatePaimonColumnUniqueId(column, field.type()); } - public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMapping) { + public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMapping, + boolean enableTimestampTzMapping) { TField field = new TField(); field.setIsOptional(dataType.isNullable()); TNestedField nestedField = new TNestedField(); @@ -325,7 +332,8 @@ public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMap TArrayField listField = new TArrayField(); org.apache.paimon.types.ArrayType paimonArrayType = (org.apache.paimon.types.ArrayType) dataType; TFieldPtr fieldPtr = new TFieldPtr(); - fieldPtr.setFieldPtr(getSchemaInfo(paimonArrayType.getElementType(), enableVarbinaryMapping)); + fieldPtr.setFieldPtr(getSchemaInfo(paimonArrayType.getElementType(), enableVarbinaryMapping, + enableTimestampTzMapping)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); field.setNestedField(nestedField); @@ -339,10 +347,12 @@ public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMap TMapField mapField = new TMapField(); org.apache.paimon.types.MapType mapType = (org.apache.paimon.types.MapType) dataType; TFieldPtr keyField = new TFieldPtr(); - keyField.setFieldPtr(getSchemaInfo(mapType.getKeyType(), enableVarbinaryMapping)); + keyField.setFieldPtr( + getSchemaInfo(mapType.getKeyType(), enableVarbinaryMapping, enableTimestampTzMapping)); mapField.setKeyField(keyField); TFieldPtr valueField = new TFieldPtr(); - valueField.setFieldPtr(getSchemaInfo(mapType.getValueType(), enableVarbinaryMapping)); + valueField.setFieldPtr( + getSchemaInfo(mapType.getValueType(), enableVarbinaryMapping, enableTimestampTzMapping)); mapField.setValueField(valueField); nestedField.setMapField(mapField); field.setNestedField(nestedField); @@ -354,7 +364,8 @@ public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMap } case ROW: { RowType rowType = (RowType) dataType; - TStructField structField = getSchemaInfo(rowType.getFields(), enableVarbinaryMapping); + TStructField structField = getSchemaInfo(rowType.getFields(), enableVarbinaryMapping, + enableTimestampTzMapping); nestedField.setStructField(structField); field.setNestedField(nestedField); @@ -364,16 +375,18 @@ public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMap break; } default: - field.setType(paimonPrimitiveTypeToDorisType(dataType, enableVarbinaryMapping).toColumnTypeThrift()); + field.setType(paimonPrimitiveTypeToDorisType(dataType, enableVarbinaryMapping, enableTimestampTzMapping) + .toColumnTypeThrift()); break; } return field; } - public static TStructField getSchemaInfo(List paimonFields, boolean enableVarbinaryMapping) { + public static TStructField getSchemaInfo(List paimonFields, boolean enableVarbinaryMapping, + boolean enableTimestampTzMapping) { TStructField structField = new TStructField(); for (DataField paimonField : paimonFields) { - TField childField = getSchemaInfo(paimonField.type(), enableVarbinaryMapping); + TField childField = getSchemaInfo(paimonField.type(), enableVarbinaryMapping, enableTimestampTzMapping); childField.setName(paimonField.name()); childField.setId(paimonField.id()); TFieldPtr fieldPtr = new TFieldPtr(); @@ -383,23 +396,27 @@ public static TStructField getSchemaInfo(List paimonFields, boolean e return structField; } - public static TSchema getSchemaInfo(TableSchema paimonTableSchema, boolean enableVarbinaryMapping) { + public static TSchema getSchemaInfo(TableSchema paimonTableSchema, boolean enableVarbinaryMapping, + boolean enableTimestampTzMapping) { TSchema tSchema = new TSchema(); tSchema.setSchemaId(paimonTableSchema.id()); - tSchema.setRootField(getSchemaInfo(paimonTableSchema.fields(), enableVarbinaryMapping)); + tSchema.setRootField( + getSchemaInfo(paimonTableSchema.fields(), enableVarbinaryMapping, enableTimestampTzMapping)); return tSchema; } - public static List parseSchema(Table table, boolean enableVarbinaryMapping) { + public static List parseSchema(Table table, boolean enableVarbinaryMapping, + boolean enableTimestampTzMapping) { List primaryKeys = table.primaryKeys(); - return parseSchema(table.rowType(), primaryKeys, enableVarbinaryMapping); + return parseSchema(table.rowType(), primaryKeys, enableVarbinaryMapping, enableTimestampTzMapping); } - public static List parseSchema(RowType rowType, List primaryKeys, boolean enableVarbinaryMapping) { + public static List parseSchema(RowType rowType, List primaryKeys, boolean enableVarbinaryMapping, + boolean enableTimestampTzMapping) { List resSchema = Lists.newArrayListWithCapacity(rowType.getFields().size()); rowType.getFields().forEach(field -> { resSchema.add(new Column(field.name().toLowerCase(), - PaimonUtil.paimonTypeToDorisType(field.type(), enableVarbinaryMapping), + PaimonUtil.paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping), primaryKeys.contains(field.name()), null, field.type().isNullable(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 4e28c924f33f03..eecddd31b446c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -205,7 +205,8 @@ private void putHistorySchemaInfo(Long schemaId) { TableSchema tableSchema = Env.getCurrentEnv().getExtMetaCacheMgr().getPaimonMetadataCache() .getPaimonSchemaCacheValue(table.getOrBuildNameMapping(), schemaId).getTableSchema(); params.addToHistorySchemaInfo( - PaimonUtil.getSchemaInfo(tableSchema, source.getCatalog().getEnableMappingVarbinary())); + PaimonUtil.getSchemaInfo(tableSchema, source.getCatalog().getEnableMappingVarbinary(), + source.getCatalog().getEnableMappingTimestampTz())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java index 819e6fbed42e63..ac2512d88b51c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/fileformat/FileFormatProperties.java @@ -49,6 +49,7 @@ public abstract class FileFormatProperties { // Default: false, mapping BINARY types to STRING for compatibility // When enabled, BINARY types map to VARBINARY public boolean enableMappingVarbinary = false; + public boolean enableMappingTimestampTz = false; public FileFormatProperties(TFileFormatType fileFormatType, String formatName) { this.fileFormatType = fileFormatType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 766634f1977769..30a423058b5f7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -199,6 +199,11 @@ protected Map parseCommonProperties(Map properti FileFormatConstants.PROP_ENABLE_MAPPING_VARBINARY, "false"); fileFormatProperties.enableMappingVarbinary = Boolean.parseBoolean(enableMappingVarbinaryStr); + // Parse enable_mapping_timestamp_tz property + String enableMappingTimestampTzStr = getOrDefaultAndRemove(copiedProps, + FileFormatConstants.PROP_ENABLE_MAPPING_TIMESTAMP_TZ, "false"); + fileFormatProperties.enableMappingTimestampTz = Boolean.parseBoolean(enableMappingTimestampTzStr); + fileFormatProperties.analyzeFileFormatProperties(copiedProps, true); if (fileFormatProperties instanceof CsvFileFormatProperties @@ -445,6 +450,7 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti fileScanRangeParams.setLoadId(ctx.queryId()); // table function fetch schema, whether to enable mapping varbinary fileScanRangeParams.setEnableMappingVarbinary(fileFormatProperties.enableMappingVarbinary); + fileScanRangeParams.setEnableMappingTimestampTz(fileFormatProperties.enableMappingTimestampTz); if (getTFileType() == TFileType.FILE_STREAM) { fileStatuses.add(new TBrokerFileStatus("", false, -1, true)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java index 82ed9a68f78514..03d400d6e58e37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java @@ -126,7 +126,8 @@ public IcebergTableValuedFunction(TableNameInfo icebergTableName, String queryTy throw new AnalysisException("Unrecognized queryType for iceberg metadata: " + queryType); } this.sysTable = MetadataTableUtils.createMetadataTableInstance(icebergTable, tableType); - this.schema = IcebergUtils.parseSchema(sysTable.schema(), externalCatalog.getEnableMappingVarbinary()); + this.schema = IcebergUtils.parseSchema(sysTable.schema(), externalCatalog.getEnableMappingVarbinary(), + externalCatalog.getEnableMappingTimestampTz()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java index aaa0372941ac4d..06da287090c0b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java @@ -96,7 +96,8 @@ public PaimonTableValuedFunction(TableNameInfo paimonTableName, String queryType this.paimonSysTable = paimonExternalCatalog.getPaimonTable(buildNameMapping, "main", queryType); - this.schema = PaimonUtil.parseSchema(paimonSysTable, paimonExternalCatalog.getEnableMappingVarbinary()); + this.schema = PaimonUtil.parseSchema(paimonSysTable, paimonExternalCatalog.getEnableMappingVarbinary(), + paimonExternalCatalog.getEnableMappingTimestampTz()); } public static PaimonTableValuedFunction create(Map params) throws AnalysisException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 25024295940016..e06b7dee7bf43a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -30,8 +30,8 @@ public class PaimonUtilTest { public void testSchemaForVarcharAndChar() { DataField c1 = new DataField(1, "c1", new VarCharType(32)); DataField c2 = new DataField(2, "c2", new CharType(14)); - Type type1 = PaimonUtil.paimonTypeToDorisType(c1.type(), true); - Type type2 = PaimonUtil.paimonTypeToDorisType(c2.type(), true); + Type type1 = PaimonUtil.paimonTypeToDorisType(c1.type(), true, true); + Type type2 = PaimonUtil.paimonTypeToDorisType(c2.type(), true, true); Assert.assertTrue(type1.isVarchar()); Assert.assertEquals(32, type1.getLength()); Assert.assertEquals(14, type2.getLength()); diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index f7a8e8d978d50c..06156ded920e08 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -488,6 +488,7 @@ struct TFileScanRangeParams { 27: optional string paimon_predicate // enable mapping varbinary type for Doris external table and TVF 28: optional bool enable_mapping_varbinary = false; + 29: optional bool enable_mapping_timestamp_tz = false; } struct TFileRangeDesc { diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out new file mode 100644 index 00000000000000..aba59213ddb1f5 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.out @@ -0,0 +1,49 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_desc_orc -- +id int Yes true \N +ts_tz timestamptz(6) Yes true \N WITH_TIMEZONE + +-- !select_desc_parquet -- +id int Yes true \N +ts_tz timestamptz(6) Yes true \N WITH_TIMEZONE + +-- !select_tvf0 -- +1 2025-01-01T00:00 +2 2025-06-01T12:34:56.789 +3 2025-12-31T23:59:59.999999 +4 \N + +-- !select_tvf0_desc -- +id int Yes false \N NONE +ts_tz datetime(6) Yes false \N NONE + +-- !select_tvf1 -- +1 2025-01-01T00:00 +2 2025-06-01T12:34:56.789 +3 2025-12-31T23:59:59.999999 +4 \N + +-- !select_tvf1_desc -- +id int Yes false \N NONE +ts_tz datetime(6) Yes false \N NONE + +-- !select_tvf2 -- +1 2025-01-01 00:00:00+08:00 +2 2025-06-01 12:34:56+08:00 +3 2025-12-31 23:59:59+08:00 +4 \N + +-- !select_tvf2_desc -- +id int Yes false \N NONE +ts_tz timestamptz Yes false \N NONE + +-- !select_tvf3 -- +1 2025-01-01 00:00:00+08:00 +2 2025-06-01 12:34:56+08:00 +3 2025-12-31 23:59:59+08:00 +4 \N + +-- !select_tvf3_desc -- +id int Yes false \N NONE +ts_tz timestamptz Yes false \N NONE + diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_tz.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_tz.out new file mode 100644 index 00000000000000..d4430978f79b9a --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_timestamp_tz.out @@ -0,0 +1,83 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_with_mapping1 -- +id int Yes true \N +ts_tz timestamptz(6) Yes true \N WITH_TIMEZONE + +-- !desc_with_mapping2 -- +id int Yes true \N +ts_tz timestamptz(6) Yes true \N WITH_TIMEZONE + +-- !select_UTC81_mapping -- +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 +4 \N + +-- !select_UTC82_mapping -- +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 +4 \N + +-- !select_UTC03_mapping -- +1 2024-12-31 16:00:00.000000+00:00 +2 2025-06-01 04:34:56.789000+00:00 +3 2025-12-31 15:59:59.999999+00:00 +4 \N + +-- !select_UTC04_mapping -- +1 2024-12-31 16:00:00.000000+00:00 +2 2025-06-01 04:34:56.789000+00:00 +3 2025-12-31 15:59:59.999999+00:00 +4 \N + +-- !select12 -- +4 + +-- !select13 -- +1 + +-- !select14 -- +1 + +-- !select15 -- +\N \N +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 +4 \N +7 2020-01-01 10:10:10.000000+08:00 + +-- !select16 -- +\N \N +1 2024-12-31 16:00:00.000000+00:00 +2 2025-06-01 04:34:56.789000+00:00 +3 2025-12-31 15:59:59.999999+00:00 +4 \N +7 2020-01-01 02:10:10.000000+00:00 + +-- !select17 -- +4 + +-- !select18 -- +1 + +-- !select19 -- +1 + +-- !select20 -- +\N \N +1 2025-01-01 00:00:00.000000+08:00 +2 2025-06-01 12:34:56.789000+08:00 +3 2025-12-31 23:59:59.999999+08:00 +4 \N +7 2020-01-01 10:10:10.000000+08:00 + +-- !select21 -- +\N \N +1 2024-12-31 16:00:00.000000+00:00 +2 2025-06-01 04:34:56.789000+00:00 +3 2025-12-31 15:59:59.999999+00:00 +4 \N +7 2020-01-01 02:10:10.000000+00:00 + diff --git a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out index 61aa3523d36681..7950b410c588a4 100644 --- a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out @@ -7,6 +7,7 @@ information_schema mysql pg_catalog public +test_timestamp_tz_db -- !test0 -- 123 abc @@ -2302,6 +2303,7 @@ information_schema mysql pg_catalog public +test_timestamp_tz_db -- !specified_database_4 -- information_schema diff --git a/regression-test/data/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.out b/regression-test/data/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.out index 5d7e2cc76b3af5..25fc197cbd3dfe 100644 --- a/regression-test/data/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.out +++ b/regression-test/data/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.out @@ -139,3 +139,30 @@ varbinary_c varbinary(100) Yes true \N 4 \N 5 0xAB +-- !desc_timestamp_tz -- +id int Yes true \N +ts_ts timestamptz Yes true \N +ts_dt datetime Yes true \N + +-- !select_timestamp_tz -- +1 2025-01-01 12:00:00+08:00 2025-01-01T12:00 +2 \N \N + +-- !select_timestamp_tz2 -- +0 + +-- !select_timestamp_tz3 -- +0 + +-- !select_timestamp_tz5 -- +1 2025-01-01 12:00:00+08:00 2025-01-01T12:00 +2 \N \N +3 1999-10-10 12:00:00+08:00 1999-10-10T12:00 +4 \N \N + +-- !select_timestamp_tz6 -- +1 2025-01-01 04:00:00+00:00 2025-01-01T12:00 +2 \N \N +3 1999-10-10 04:00:00+00:00 1999-10-10T12:00 +4 \N \N + diff --git a/regression-test/data/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.out b/regression-test/data/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.out index 6e196d63614ce5..ff3c799662c183 100644 --- a/regression-test/data/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.out +++ b/regression-test/data/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.out @@ -59,3 +59,15 @@ id int Yes true \N 3 normal 0x48656C6C6F20576F726C64 4 insert 0xAB +-- !desc_timestamp_tz -- +ID text Yes true \N +TS_LTZ timestamptz(6) Yes true \N + +-- !select_timestamp_tz -- +\N \N +1 2025-01-01 12:00:00.000000+08:00 + +-- !select_timestamp_tz6 -- +\N \N +1 2025-01-01 04:00:00.000000+00:00 + diff --git a/regression-test/data/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.out b/regression-test/data/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.out index 863f5e8e36e59b..69819e7e10ab0b 100644 --- a/regression-test/data/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.out +++ b/regression-test/data/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.out @@ -35,3 +35,30 @@ uuid_val text Yes true \N -- !select_all_types_multi_block -- 4100 +-- !desc_timestamp_tz -- +id int Yes true \N +ts_tz timestamptz(6) Yes true \N +ts_ntz datetime(6) Yes true \N + +-- !select_timestamp_tz -- +1 2025-01-01 12:00:00.000000+08:00 2025-01-01T12:00 +2 \N \N + +-- !select_timestamp_tz2 -- +0 + +-- !select_timestamp_tz3 -- +0 + +-- !select_timestamp_tz5 -- +1 2025-01-01 12:00:00.000000+08:00 2025-01-01T12:00 +2 \N \N +3 1999-10-10 04:00:00.000000+08:00 1999-10-10T12:00 +4 \N \N + +-- !select_timestamp_tz6 -- +1 2025-01-01 04:00:00.000000+00:00 2025-01-01T12:00 +2 \N \N +3 1999-10-09 20:00:00.000000+00:00 1999-10-10T12:00 +4 \N \N + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out new file mode 100644 index 00000000000000..4f9965aaed9e7a --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.out @@ -0,0 +1,38 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc_1 -- +id int Yes true \N +ts_ltz timestamptz(3) Yes true \N WITH_TIMEZONE + +-- !jni_1 -- +1 2024-01-01T10:00 +2 2026-01-06T16:13:12 +3 2024-11-11T11:11:11.000000123 + +-- !jni_1_cast -- +1 2024-01-01 10:00:00.000+08:00 +2 2026-01-06 16:13:12.000+08:00 +3 2024-11-11 11:11:11.123+08:00 + +-- !desc_2 -- +id int Yes true \N +ts_ltz timestamptz(3) Yes true \N WITH_TIMEZONE + +-- !native_1 -- +1 2024-01-01T10:00 +2 2026-01-06T16:13:12 +3 2024-11-11T11:11:11.000000123 + +-- !native_1_cast -- +1 2024-01-01 10:00:00.000+08:00 +2 2026-01-06 16:13:12.000+08:00 +3 2024-11-11 11:11:11.123+08:00 + +-- !mapping_tz -- +1 2024-01-01 10:00:00+08:00 +2 2026-01-06 16:13:12+08:00 +3 2024-11-11 11:11:11+08:00 + +-- !mapping_tz_desc -- +id int Yes false \N NONE +ts_ltz timestamptz Yes false \N NONE + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.groovy new file mode 100644 index 00000000000000..b0a4c54db40c18 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.groovy @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_iceberg_export_timestamp_tz", "external,hive,external_docker") { + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + for (String hivePrefix : ["hive2"]) { + setHivePrefix(hivePrefix) + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + // It's okay to use random `hdfsUser`, but can not be empty. + def hdfsUserName = "doris" + def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" + def outfile_path = "/user/doris/tmp_data" + def uri = "${defaultFS}" + "${outfile_path}/exp_" + + def outfile_to_HDFS = {format,export_table_name -> + // select ... into outfile ... + def uuid = UUID.randomUUID().toString() + outfile_path = "/user/doris/tmp_data/${uuid}" + uri = "${defaultFS}" + "${outfile_path}/exp_" + + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY id + INTO OUTFILE "${uri}" + FORMAT AS ${format} + PROPERTIES ( + "fs.defaultFS"="${defaultFS}", + "hadoop.username" = "${hdfsUserName}" + ); + """ + logger.info("outfile success path: " + res[0][3]); + return res[0][3] + } + + try { + String catalog_name_with_export = "test_iceberg_timestamp_tz_with_mapping_export" + String db_name = "test_timestamp_tz" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + sql """set time_zone = 'Asia/Shanghai';""" + + sql """drop catalog if exists ${catalog_name_with_export}""" + sql """ + CREATE CATALOG ${catalog_name_with_export} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1", + "s3.path.style.access" = "true", + "s3.connection.ssl.enabled" = "false", + "enable.mapping.varbinary"="true", + "enable.mapping.timestamp_tz"="true" + );""" + + + sql """switch ${catalog_name_with_export}""" + sql """use ${db_name}""" + order_qt_select_desc_orc """ desc test_ice_timestamp_tz_orc; """ + order_qt_select_desc_parquet """ desc test_ice_timestamp_tz_parquet; """ + // TODO: seems write to parquet with timestamp_tz has some problem + def format = "parquet" + def export_table_name = "test_ice_timestamp_tz_parquet" + + def outfile_url0 = outfile_to_HDFS(format, export_table_name) + order_qt_select_tvf0 """ select * from HDFS( + "uri" = "${outfile_url0}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_timestamp_tz"="true", + "enable_mapping_varbinary"="true", + "format" = "${format}"); + """ + order_qt_select_tvf0_desc """ desc function HDFS( + "uri" = "${outfile_url0}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_timestamp_tz"="true", + "enable_mapping_varbinary"="true", + "format" = "${format}"); + """ + + format = "parquet" + export_table_name = "test_ice_timestamp_tz_orc" + def outfile_url1 = outfile_to_HDFS(format, export_table_name) + order_qt_select_tvf1 """ select * from HDFS( + "uri" = "${outfile_url1}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_timestamp_tz"="true", + "enable_mapping_varbinary"="true", + "format" = "${format}"); + """ + order_qt_select_tvf1_desc """ desc function HDFS( + "uri" = "${outfile_url1}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_timestamp_tz"="true", + "enable_mapping_varbinary"="true", + "format" = "${format}"); + """ + + format = "orc" + export_table_name = "test_ice_timestamp_tz_parquet" + def outfile_url2 = outfile_to_HDFS(format, export_table_name) + order_qt_select_tvf2 """ select * from HDFS( + "uri" = "${outfile_url2}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_varbinary"="true", + "enable_mapping_timestamp_tz"="true", + "format" = "${format}"); + """ + order_qt_select_tvf2_desc """ desc function HDFS( + "uri" = "${outfile_url2}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_varbinary"="true", + "enable_mapping_timestamp_tz"="true", + "format" = "${format}"); + """ + + + format = "orc" + export_table_name = "test_ice_timestamp_tz_orc" + def outfile_url3 = outfile_to_HDFS(format, export_table_name) + order_qt_select_tvf3 """ select * from HDFS( + "uri" = "${outfile_url3}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_varbinary"="true", + "enable_mapping_timestamp_tz"="true", + "format" = "${format}"); + """ + order_qt_select_tvf3_desc """ desc function HDFS( + "uri" = "${outfile_url3}.${format}", + "hadoop.username" = "${hdfsUserName}", + "enable_mapping_varbinary"="true", + "enable_mapping_timestamp_tz"="true", + "format" = "${format}"); + """ + } finally { + } + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_tz.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_tz.groovy new file mode 100644 index 00000000000000..21a0a8a6e88a4d --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_timestamp_tz.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_timestamp_tz", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalog_name_with_mapping = "test_iceberg_timestamp_tz_with_mapping" + String db_name = "test_timestamp_tz" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name_with_mapping}""" + sql """ + CREATE CATALOG ${catalog_name_with_mapping} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1", + "s3.path.style.access" = "true", + "s3.connection.ssl.enabled" = "false", + "enable.mapping.varbinary"="true", + "enable.mapping.timestamp_tz"="true" + );""" + + + sql """switch ${catalog_name_with_mapping}""" + sql """use ${db_name}""" + sql """set time_zone = 'Asia/Shanghai';""" + order_qt_desc_with_mapping1 """desc test_ice_timestamp_tz_orc;""" + order_qt_desc_with_mapping2 """desc test_ice_timestamp_tz_parquet;""" + + qt_select_UTC81_mapping """ + select * from test_ice_timestamp_tz_orc order by id; + """ + + qt_select_UTC82_mapping """ + select * from test_ice_timestamp_tz_parquet order by id; + """ + + sql """set time_zone = 'UTC';""" + + qt_select_UTC03_mapping """ + select * from test_ice_timestamp_tz_orc order by id; + """ + qt_select_UTC04_mapping """ + select * from test_ice_timestamp_tz_parquet order by id; + """ + + // with mapping orc + sql """set time_zone = 'Asia/Shanghai';""" + qt_select12 """ + insert into test_ice_timestamp_tz_orc_write_with_mapping select * from test_ice_timestamp_tz_orc; + """ + + qt_select13 """ + insert into test_ice_timestamp_tz_orc_write_with_mapping values(NULL,NULL); + """ + + qt_select14 """ + insert into test_ice_timestamp_tz_orc_write_with_mapping values(7,cast("2020-01-01 10:10:10 +08:00" as timestamptz)); + """ + + qt_select15 """ + select * from test_ice_timestamp_tz_orc_write_with_mapping order by id; + """ + + sql """set time_zone = 'UTC';""" + qt_select16 """ + select * from test_ice_timestamp_tz_orc_write_with_mapping order by id; + """ + + // with mapping parquet + sql """set time_zone = 'Asia/Shanghai';""" + qt_select17 """ + insert into test_ice_timestamp_tz_parquet_write_with_mapping select * from test_ice_timestamp_tz_parquet; + """ + + qt_select18 """ + insert into test_ice_timestamp_tz_parquet_write_with_mapping values(NULL,NULL); + """ + + qt_select19 """ + insert into test_ice_timestamp_tz_parquet_write_with_mapping values(7,cast("2020-01-01 10:10:10 +08:00" as timestamptz)); + """ + + qt_select20 """ + select * from test_ice_timestamp_tz_parquet_write_with_mapping order by id; + """ + sql """set time_zone = 'UTC';""" + qt_select21 """ + select * from test_ice_timestamp_tz_parquet_write_with_mapping order by id; + """ +} diff --git a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.groovy b/regression-test/suites/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.groovy index e9396412062166..0f59b71f2f24b1 100644 --- a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.groovy +++ b/regression-test/suites/external_table_p0/jdbc/type_test/select/test_mysql_all_types_select.groovy @@ -58,5 +58,26 @@ suite("test_mysql_all_types_select", "p0,external,mysql,external_docker,external qt_select_varbinary_type5 """select * from test_varbinary order by id;""" sql """drop catalog if exists mysql_all_type_test """ + + sql """drop catalog if exists mysql_timestamp_tz_type_test """ + sql """create catalog if not exists mysql_timestamp_tz_type_test properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "enable.mapping.varbinary" = "true", + "enable.mapping.timestamp_tz" = "true" + );""" + sql """SET time_zone = '+08:00';""" + sql """use mysql_timestamp_tz_type_test.test_timestamp_tz_db""" + qt_desc_timestamp_tz """desc ts_test;""" + qt_select_timestamp_tz """select * from ts_test order by id;""" + qt_select_timestamp_tz2 """insert into ts_test values(3,"1999-10-10 12:00:00+08:00","1999-10-10 12:00:00");""" + qt_select_timestamp_tz3 """insert into ts_test values(4,NULL, NULL);""" + qt_select_timestamp_tz5 """select * from ts_test order by id;""" + sql """SET time_zone = '+00:00';""" + qt_select_timestamp_tz6 """select * from ts_test order by id;""" } } diff --git a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.groovy b/regression-test/suites/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.groovy index 317e30088f539a..c70cbae55a6daa 100644 --- a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.groovy +++ b/regression-test/suites/external_table_p0/jdbc/type_test/select/test_oracle_all_types_select.groovy @@ -61,5 +61,27 @@ suite("test_oracle_all_types_select", "p0,external,oracle,external_docker,extern qt_select_varbinary_types "select * from VARBINARY_TEST order by id;" sql """ insert into VARBINARY_TEST values (4,"insert",X"AB");""" qt_select_varbinary_types2 "select * from VARBINARY_TEST order by id;" + + sql """drop catalog if exists oracle_timestamp_type_test """ + sql """create catalog if not exists oracle_timestamp_type_test properties( + "type"="jdbc", + "user"="doris_test", + "password"="123456", + "jdbc_url" = "jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}", + "driver_url" = "${driver_url}", + "driver_class" = "oracle.jdbc.driver.OracleDriver", + "enable.mapping.varbinary"="true", + "enable.mapping.timestamp_tz"="true" + );""" + sql """SET time_zone = '+08:00';""" + sql """use oracle_timestamp_type_test.DORIS_TEST""" + qt_desc_timestamp_tz """desc LTZ_TEST;""" + qt_select_timestamp_tz """select * from LTZ_TEST order by id;""" + // TODO: need more time to investigate the insert behavior + // qt_select_timestamp_tz2 """insert into LTZ_TEST values(3,"1999-10-10 12:00:00+08:00");""" + // qt_select_timestamp_tz3 """insert into LTZ_TEST values(4,NULL);""" + // qt_select_timestamp_tz5 """select * from LTZ_TEST order by id;""" + sql """SET time_zone = '+00:00';""" + qt_select_timestamp_tz6 """select * from LTZ_TEST order by id;""" } } diff --git a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.groovy b/regression-test/suites/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.groovy index 39e445dc7eed66..f42d962ebd6254 100644 --- a/regression-test/suites/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.groovy +++ b/regression-test/suites/external_table_p0/jdbc/type_test/select/test_pg_all_types_select.groovy @@ -72,5 +72,26 @@ suite("test_pg_all_types_select", "p0,external,pg,external_docker,external_docke qt_select_all_types_multi_block """select count(*) from catalog_pg_test.extreme_test_multi_block;""" sql """drop catalog if exists pg_all_type_test """ + + sql """drop catalog if exists pg_timestamp_tz_type_test """ + sql """create catalog if not exists pg_timestamp_tz_type_test properties( + "type"="jdbc", + "user"="postgres", + "password"="123456", + "jdbc_url" = "jdbc:postgresql://${externalEnvIp}:${pg_port}/postgres?currentSchema=test_timestamp_tz_db&useSSL=false", + "driver_url" = "${driver_url}", + "driver_class" = "org.postgresql.Driver", + "enable.mapping.timestamp_tz" = "true" + );""" + + sql """SET time_zone = '+08:00';""" + sql """use pg_timestamp_tz_type_test.test_timestamp_tz_db""" + qt_desc_timestamp_tz """desc ts_test;""" + qt_select_timestamp_tz """select * from ts_test order by id;""" + qt_select_timestamp_tz2 """insert into ts_test values(3,"1999-10-10 12:00:00+08:00","1999-10-10 12:00:00");""" + qt_select_timestamp_tz3 """insert into ts_test values(4,NULL, NULL);""" + qt_select_timestamp_tz5 """select * from ts_test order by id;""" + sql """SET time_zone = '+00:00';""" + qt_select_timestamp_tz6 """select * from ts_test order by id;""" } } diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.groovy new file mode 100644 index 00000000000000..0c054569a8cede --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_timestamp_tz.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_catalog_timestamp_tz", "p0,external,doris,external_docker,external_docker_doris,new_catalog_property") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort") + String catalog_name_with_mapping = "test_paimon_timestamp_tz_with_mapping" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name_with_mapping}""" + sql """create catalog if not exists ${catalog_name_with_mapping} properties ( + "type" = "paimon", + "paimon.catalog.type"="filesystem", + "enable.mapping.varbinary"="true", + "enable.mapping.timestamp_tz"="true", + "warehouse" = "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1" + );""" + sql """set time_zone = 'Asia/Shanghai';""" + // with mapping + sql """use `${catalog_name_with_mapping}`.`db1`""" + sql """ set force_jni_scanner=true; """ + explain { + sql "select * from t_ltz order by id;" + contains "paimonNativeReadSplits=0/1" + } + order_qt_desc_1 """ desc t_ltz; """ + qt_jni_1 """ select * from t_ltz order by id; """ + qt_jni_1_cast """ select id, cast(ts_ltz as string) from t_ltz order by id; """ + sql """ set force_jni_scanner=false; """ + explain { + sql "select * from t_ltz order by id;" + contains "paimonNativeReadSplits=2/2" + } + order_qt_desc_2 """ desc t_ltz; """ + qt_native_1 """ select * from t_ltz order by id; """ + qt_native_1_cast """ select id, cast(ts_ltz as string) from t_ltz order by id; """ + + // with mapping + qt_mapping_tz """ + select * from hdfs( + "uri" = "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1/db1.db/t_ltz/bucket-0/*.parquet", + "fs.defaultFS" = "hdfs://${externalEnvIp}:${hdfs_port}", + "hadoop.username" = "doris", + "enable_mapping_varbinary" = "true", + "enable_mapping_timestamp_tz" = "true", + "format" = "parquet") order by id; + """ + + order_qt_mapping_tz_desc """ + desc function hdfs( + "uri" = "hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1/db1.db/t_ltz/bucket-0/*.parquet", + "fs.defaultFS" = "hdfs://${externalEnvIp}:${hdfs_port}", + "hadoop.username" = "doris", + "enable_mapping_varbinary" = "true", + "enable_mapping_timestamp_tz" = "true", + "format" = "parquet"); + """ + } +} + + + diff --git a/regression-test/suites/manager/test_manager_interface_3.groovy b/regression-test/suites/manager/test_manager_interface_3.groovy index 28ffa19a4acc13..23d63c6b68ab4d 100644 --- a/regression-test/suites/manager/test_manager_interface_3.groovy +++ b/regression-test/suites/manager/test_manager_interface_3.groovy @@ -425,7 +425,7 @@ suite('test_manager_interface_3',"p0") { } } log.info("x1 = ${x}") - assertTrue(x == 21) + assertTrue(x == 22) connect(user, "${pwd}", url) { result = sql """ show resources """ @@ -437,7 +437,7 @@ suite('test_manager_interface_3',"p0") { } } log.info("x2 = ${x}") - assertTrue(x == 21) + assertTrue(x == 22) } checkNereidsExecute("show all grants"); @@ -481,7 +481,7 @@ suite('test_manager_interface_3',"p0") { } } log.info("x3 = ${x}") - assertTrue(x == 21) + assertTrue(x == 22) } sql """ drop RESOURCE if exists ${resource_name} """ sql """drop user if exists ${user}"""