diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp index 1f255078b5d6cf..947b951c30ba22 100644 --- a/be/src/exec/operator/operator.cpp +++ b/be/src/exec/operator/operator.cpp @@ -64,6 +64,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -833,6 +834,7 @@ DECLARE_OPERATOR(OlapTableSinkV2LocalState) DECLARE_OPERATOR(HiveTableSinkLocalState) DECLARE_OPERATOR(TVFTableSinkLocalState) DECLARE_OPERATOR(IcebergTableSinkLocalState) +DECLARE_OPERATOR(PaimonTableSinkLocalState) DECLARE_OPERATOR(SpillIcebergTableSinkLocalState) DECLARE_OPERATOR(IcebergDeleteSinkLocalState) DECLARE_OPERATOR(IcebergMergeSinkLocalState) diff --git a/be/src/exec/operator/paimon_table_sink_operator.cpp b/be/src/exec/operator/paimon_table_sink_operator.cpp new file mode 100644 index 00000000000000..e177682bb2ccfb --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.cpp @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/operator/paimon_table_sink_operator.h" + +#include "common/logging.h" + +namespace doris { + +Status PaimonTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { + return Base::init(state, info); +} + +Status PaimonTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool eos) { + auto& local_state = get_local_state(state); + SCOPED_TIMER(local_state.exec_time_counter()); + COUNTER_UPDATE(local_state.rows_input_counter(), static_cast(in_block->rows())); + + // Delegate directly to the single AsyncWriterSink → VPaimonTableWriter. + // Partition and bucket routing is handled internally by the Paimon SDK + // inside IPaimonWriter::write(). + return local_state.sink(state, in_block, eos); +} + +} // namespace doris diff --git a/be/src/exec/operator/paimon_table_sink_operator.h b/be/src/exec/operator/paimon_table_sink_operator.h new file mode 100644 index 00000000000000..3ee0181129a33e --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/operator/operator.h" +#include "exec/sink/writer/paimon/vpaimon_table_writer.h" +#include "runtime/runtime_state.h" + +namespace doris { + +/// Paimon table sink operator — simple pass-through to AsyncWriterSink. +/// +/// A single VPaimonTableWriter (one AsyncResultWriter, one IO thread) +/// handles the entire table. Partition and bucket routing is performed +/// internally by the Paimon SDK (Java or Rust) inside +/// IPaimonWriter::write(). Doris does not compute partition values +/// or bucket ids; it passes complete Blocks to the SDK. +/// +/// This mirrors Iceberg's approach: IcebergTableSinkOperatorX simply +/// delegates to AsyncWriterSink, and all +/// partition routing happens inside VIcebergTableWriter::write(). +class PaimonTableSinkOperatorX; + +class PaimonTableSinkLocalState final + : public AsyncWriterSink { +public: + using Base = AsyncWriterSink; + using Parent = PaimonTableSinkOperatorX; + ENABLE_FACTORY_CREATOR(PaimonTableSinkLocalState); + PaimonTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) + : Base(parent, state) {} + Status init(RuntimeState* state, LocalSinkStateInfo& info) override; + Status open(RuntimeState* state) override { + SCOPED_TIMER(exec_time_counter()); + SCOPED_TIMER(_open_timer); + return Base::open(state); + } + + friend class PaimonTableSinkOperatorX; +}; + +class PaimonTableSinkOperatorX final : public DataSinkOperatorX { +public: + using Base = DataSinkOperatorX; + PaimonTableSinkOperatorX(ObjectPool* pool, int operator_id, const RowDescriptor& row_desc, + const std::vector& t_output_expr) + : Base(operator_id, 0, 0), + _row_desc(row_desc), + _t_output_expr(t_output_expr), + _pool(pool) {} + + Status init(const TDataSink& thrift_sink) override { + RETURN_IF_ERROR(Base::init(thrift_sink)); + DCHECK(thrift_sink.__isset.paimon_table_sink); + RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs)); + return Status::OK(); + } + + Status prepare(RuntimeState* state) override { + RETURN_IF_ERROR(Base::prepare(state)); + RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); + return VExpr::open(_output_vexpr_ctxs, state); + } + + Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; + +private: + friend class PaimonTableSinkLocalState; + template + requires(std::is_base_of_v) + friend class AsyncWriterSink; + + const RowDescriptor& _row_desc; + VExprContextSPtrs _output_vexpr_ctxs; + const std::vector& _t_output_expr; + ObjectPool* _pool = nullptr; +}; + +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index f8bd3120846a80..634c964ce3b264 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -87,6 +87,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -1371,6 +1372,14 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS output_exprs); break; } + case TDataSinkType::PAIMON_TABLE_SINK: { + if (!thrift_sink.__isset.paimon_table_sink) { + return Status::InternalError("Missing paimon table sink."); + } + _sink = std::make_shared(pool, next_sink_operator_id(), row_desc, + output_exprs); + break; + } case TDataSinkType::JDBC_TABLE_SINK: { if (!thrift_sink.__isset.jdbc_table_sink) { return Status::InternalError("Missing data jdbc sink."); @@ -2515,6 +2524,20 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } } + if (auto pcm = req.runtime_state->paimon_commit_messages(); !pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), pcm.begin(), + pcm.end()); + } else if (!req.runtime_states.empty()) { + for (auto* rs : req.runtime_states) { + if (auto rs_pcm = rs->paimon_commit_messages(); !rs_pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), + rs_pcm.begin(), rs_pcm.end()); + } + } + } + req.runtime_state->get_unreported_errors(&(params.error_log)); params.__isset.error_log = (!params.error_log.empty()); diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp new file mode 100644 index 00000000000000..a11f3385a8a5b5 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" + +namespace doris { + +Status FfiPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* state) { + return Status::NotSupported( + "FFI (Rust) Paimon write backend is not yet implemented. " + "Currently only the JNI (Java) backend is available. " + "The FFI backend will be introduced in v2 for high-throughput append-only writes."); +} + +Status FfiPaimonWriteBackend::create_writer(std::unique_ptr* writer) { + return Status::NotSupported("FFI backend: create_writer not yet implemented"); +} + +Status FfiPaimonWriteBackend::create_committer(std::unique_ptr* committer) { + return Status::NotSupported("FFI backend: create_committer not yet implemented"); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h new file mode 100644 index 00000000000000..fe9cfd49e18b48 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +/// Stub implementation of Paimon write backend via Rust C FFI. +/// +/// This is a placeholder for future use. In v1, all methods return +/// Status::NotSupported. When paimon-rust is integrated (v2), this +/// will be replaced with a real implementation that: +/// +/// 1. Dynamically loads the paimon-rust shared library (libpaimon_c.so) +/// 2. Converts Doris Block → Arrow C Data Interface (zero-copy) +/// 3. Calls Rust TableWrite::write_arrow_batch() via FFI +/// 4. Calls Rust TableCommit::commit() via FFI +/// +/// v1 scope: stub only. Upper layers (VPaimonTableWriter) are coded +/// against IPaimonWriteBackend and never need to change. +class FfiPaimonWriteBackend final : public IPaimonWriteBackend { +public: + FfiPaimonWriteBackend() = default; + ~FfiPaimonWriteBackend() override = default; + + Status open(const TPaimonTableSink& sink, RuntimeState* state) override; + + Status create_writer(std::unique_ptr* writer) override; + + Status create_committer(std::unique_ptr* committer) override; + + PaimonBackendType type() const override { return PaimonBackendType::FFI; } + + // Rust currently supports a subset of Java's features + bool supports_compaction() const override { return false; } + bool supports_lookup_changelog_producer() const override { return false; } + bool supports_full_compaction_changelog_producer() const override { return false; } + bool supports_partial_update_with_dv() const override { return false; } + bool supports_aggregation_with_dv() const override { return false; } +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp new file mode 100644 index 00000000000000..04116fffd09fd7 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp @@ -0,0 +1,427 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" + +#include +#include +#include +#include +#include + +#include "common/logging.h" +#include "format/arrow/arrow_block_convertor.h" +#include "format/arrow/arrow_row_batch.h" +#include "runtime/runtime_state.h" +#include "util/jni-util.h" + +namespace doris { + +// ──────────────────────────────────────────────────────────── +// JniPaimonWriteBackend +// ──────────────────────────────────────────────────────────── + +static constexpr const char* PAIMON_JNI_WRITER_CLASS = "org/apache/doris/paimon/PaimonJniWriter"; +static constexpr const char* PAIMON_OUTPUT_COLUMN_NAMES_KEY = "doris.output_column_names"; +static constexpr char PAIMON_COLUMN_NAME_SEPARATOR = '\x01'; + +JniPaimonWriteBackend::~JniPaimonWriteBackend() { + JNIEnv* env = nullptr; + if (_get_jni_env(&env).ok()) { + if (_jni_writer_obj != nullptr) { + env->DeleteGlobalRef(_jni_writer_obj); + _jni_writer_obj = nullptr; + } + if (_jni_writer_cls != nullptr) { + env->DeleteGlobalRef(_jni_writer_cls); + _jni_writer_cls = nullptr; + } + } +} + +Status JniPaimonWriteBackend::_get_jni_env(JNIEnv** env) { + JavaVM* jvm = nullptr; + jsize n_vms = 0; + jint result = JNI_GetCreatedJavaVMs(&jvm, 1, &n_vms); + if (result != JNI_OK || n_vms == 0) { + return Status::InternalError("Failed to get created JavaVM"); + } + result = jvm->GetEnv(reinterpret_cast(env), JNI_VERSION_1_8); + if (result == JNI_EDETACHED) { + result = jvm->AttachCurrentThread(reinterpret_cast(env), nullptr); + if (result != JNI_OK) { + return Status::InternalError("Failed to attach current thread to JVM"); + } + } else if (result != JNI_OK) { + return Status::InternalError("Failed to get JNIEnv"); + } + return Status::OK(); +} + +Status JniPaimonWriteBackend::_check_jni_exception(JNIEnv* env, const std::string& method_name) { + if (env->ExceptionCheck()) { + Status st = + Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in " + method_name + ": "); + LOG(WARNING) << st.to_string(); + return st; + } + return Status::OK(); +} + +static std::vector _split_column_names(const std::string& column_names_str) { + std::vector names; + size_t begin = 0; + while (begin <= column_names_str.size()) { + size_t end = column_names_str.find(PAIMON_COLUMN_NAME_SEPARATOR, begin); + if (end == std::string::npos) { + names.emplace_back(column_names_str.substr(begin)); + break; + } + names.emplace_back(column_names_str.substr(begin, end - begin)); + begin = end + 1; + } + return names; +} + +static jobject _to_java_options(JNIEnv* env, const std::map& options) { + jclass map_cls = env->FindClass("java/util/HashMap"); + jmethodID map_ctor = env->GetMethodID(map_cls, "", "()V"); + jmethodID put_method = env->GetMethodID( + map_cls, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + + jobject map_obj = env->NewObject(map_cls, map_ctor); + for (const auto& kv : options) { + jstring key = env->NewStringUTF(kv.first.c_str()); + jstring val = env->NewStringUTF(kv.second.c_str()); + env->CallObjectMethod(map_obj, put_method, key, val); + env->DeleteLocalRef(key); + env->DeleteLocalRef(val); + } + env->DeleteLocalRef(map_cls); + return map_obj; +} + +Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* state) { + _sink = sink; + _arrow_pool = std::make_unique>(); + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + + // Find the Java writer class + jclass local_cls = env->FindClass(PAIMON_JNI_WRITER_CLASS); + RETURN_IF_ERROR(_check_jni_exception(env, "FindClass")); + if (local_cls == nullptr) { + return Status::InternalError("Failed to find Java class: {}", PAIMON_JNI_WRITER_CLASS); + } + _jni_writer_cls = static_cast(env->NewGlobalRef(local_cls)); + env->DeleteLocalRef(local_cls); + + // Cache method IDs + _open_id = env->GetMethodID(_jni_writer_cls, "open", + "(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;)V"); + _write_id = env->GetMethodID(_jni_writer_cls, "write", "(JI)V"); + _prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B"); + _abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V"); + _close_id = env->GetMethodID(_jni_writer_cls, "close", "()V"); + RETURN_IF_ERROR(_check_jni_exception(env, "GetMethodID")); + + // Create the JNI writer object + jmethodID ctor_id = env->GetMethodID(_jni_writer_cls, "", "()V"); + jobject local_obj = env->NewObject(_jni_writer_cls, ctor_id); + RETURN_IF_ERROR(_check_jni_exception(env, "NewObject")); + _jni_writer_obj = env->NewGlobalRef(local_obj); + env->DeleteLocalRef(local_obj); + + // Build options map + std::map jni_options; + if (sink.__isset.paimon_options) { + jni_options.insert(sink.paimon_options.begin(), sink.paimon_options.end()); + } + if (sink.__isset.hadoop_config) { + for (const auto& [key, value] : sink.hadoop_config) { + jni_options[key] = value; + } + } + jni_options["db_name"] = sink.db_name; + jni_options["table_name"] = sink.tb_name; + if (sink.__isset.serialized_table && !sink.serialized_table.empty()) { + jni_options["serialized_table"] = sink.serialized_table; + } + + // Build column names array + jstring j_location = env->NewStringUTF(sink.table_location.c_str()); + jobject j_options = _to_java_options(env, jni_options); + + jclass string_cls = env->FindClass("java/lang/String"); + jobjectArray j_cols = nullptr; + if (sink.__isset.column_names && !sink.column_names.empty()) { + j_cols = env->NewObjectArray(static_cast(sink.column_names.size()), string_cls, + nullptr); + for (size_t i = 0; i < sink.column_names.size(); ++i) { + jstring str = env->NewStringUTF(sink.column_names[i].c_str()); + env->SetObjectArrayElement(j_cols, static_cast(i), str); + env->DeleteLocalRef(str); + } + } else { + j_cols = env->NewObjectArray(0, string_cls, nullptr); + } + + // Call Java open() + env->CallVoidMethod(_jni_writer_obj, _open_id, j_location, j_options, j_cols); + Status st = _check_jni_exception(env, "open"); + + env->DeleteLocalRef(j_location); + env->DeleteLocalRef(j_options); + env->DeleteLocalRef(j_cols); + env->DeleteLocalRef(string_cls); + + if (st.ok()) { + _opened = true; + } + return st; +} + +Status JniPaimonWriteBackend::create_writer(std::unique_ptr* writer) { + DCHECK(_opened) << "Backend must be opened before creating writers"; + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + + auto jni_writer = std::make_unique( + env, _jni_writer_obj, _write_id, _prepare_commit_id, _abort_id, + std::make_unique>(), _sink); + *writer = std::move(jni_writer); + return Status::OK(); +} + +Status JniPaimonWriteBackend::create_committer(std::unique_ptr* committer) { + auto c = std::make_unique(_sink); + *committer = std::move(c); + return Status::OK(); +} + +// ──────────────────────────────────────────────────────────── +// JniPaimonWriter +// ──────────────────────────────────────────────────────────── + +JniPaimonWriter::JniPaimonWriter(JNIEnv* env, jobject jni_writer_obj, jmethodID write_id, + jmethodID prepare_commit_id, jmethodID abort_id, + std::unique_ptr> arrow_pool, + const TPaimonTableSink& sink) + : _env(env), + _jni_writer_obj(jni_writer_obj), + _write_id(write_id), + _prepare_commit_id(prepare_commit_id), + _abort_id(abort_id), + _arrow_pool(std::move(arrow_pool)), + _sink(sink) {} + +JniPaimonWriter::~JniPaimonWriter() = default; + +Status JniPaimonWriter::_write_projected_block(Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + _row_count += block.rows(); + + // Determine output column names + if (_sink.__isset.paimon_options) { + auto it = _sink.paimon_options.find(PAIMON_OUTPUT_COLUMN_NAMES_KEY); + if (it != _sink.paimon_options.end()) { + auto output_names = _split_column_names(it->second); + DCHECK_EQ(output_names.size(), block.columns()); + for (size_t i = 0; i < output_names.size(); ++i) { + block.get_by_position(i).name = output_names[i]; + } + } + } + + // Convert Block → Arrow RecordBatch → IPC Stream + std::shared_ptr arrow_schema; + RETURN_IF_ERROR(get_arrow_schema_from_block(block, &arrow_schema, "UTC")); + + std::shared_ptr record_batch; + RETURN_IF_ERROR(convert_to_arrow_batch(block, arrow_schema, _arrow_pool.get(), &record_batch, + cctz::utc_time_zone())); + + auto out_stream_res = arrow::io::BufferOutputStream::Create(); + if (!out_stream_res.ok()) { + return Status::InternalError("Arrow BufferOutputStream create failed: {}", + out_stream_res.status().ToString()); + } + auto out_stream = *out_stream_res; + + auto writer_res = arrow::ipc::MakeStreamWriter(out_stream, arrow_schema); + if (!writer_res.ok()) { + return Status::InternalError("Arrow StreamWriter create failed: {}", + writer_res.status().ToString()); + } + auto ipc_writer = *writer_res; + if (!ipc_writer->WriteRecordBatch(*record_batch).ok()) { + return Status::InternalError("Arrow WriteRecordBatch failed"); + } + if (!ipc_writer->Close().ok()) { + return Status::InternalError("Arrow StreamWriter close failed"); + } + + auto buffer_res = out_stream->Finish(); + if (!buffer_res.ok()) { + return Status::InternalError("Arrow output stream finish failed: {}", + buffer_res.status().ToString()); + } + std::shared_ptr buffer = *buffer_res; + + // Pass to Java via JNI + JNIEnv* env = nullptr; + { + JavaVM* jvm = nullptr; + jsize n_vms = 0; + JNI_GetCreatedJavaVMs(&jvm, 1, &n_vms); + jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_8); + if (env == nullptr) { + jvm->AttachCurrentThread(reinterpret_cast(&env), nullptr); + } + } + + auto address = reinterpret_cast(buffer->data()); + jint length = static_cast(buffer->size()); + env->CallVoidMethod(_jni_writer_obj, _write_id, address, length); + RETURN_IF_ERROR( + Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in JniPaimonWriter::write: ")); + return Status::OK(); +} + +Status JniPaimonWriter::write(RuntimeState* state, Block& block) { + return _write_projected_block(block); +} + +Status JniPaimonWriter::prepare_commit(std::vector& messages) { + JNIEnv* env = nullptr; + { + JavaVM* jvm = nullptr; + jsize n_vms = 0; + JNI_GetCreatedJavaVMs(&jvm, 1, &n_vms); + jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_8); + if (env == nullptr) { + jvm->AttachCurrentThread(reinterpret_cast(&env), nullptr); + } + } + + jobject j_payloads_obj = env->CallObjectMethod(_jni_writer_obj, _prepare_commit_id); + Status st = Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in prepareCommit: "); + if (!st.ok()) { + return st; + } + + if (j_payloads_obj == nullptr) { + return Status::OK(); // no data written + } + + auto* j_payloads = static_cast(j_payloads_obj); + jsize num_payloads = env->GetArrayLength(j_payloads); + + for (jsize i = 0; i < num_payloads; ++i) { + jbyteArray j_bytes = static_cast(env->GetObjectArrayElement(j_payloads, i)); + if (j_bytes == nullptr) { + continue; + } + jsize len = env->GetArrayLength(j_bytes); + if (len > 0) { + jbyte* bytes = env->GetByteArrayElements(j_bytes, nullptr); + if (bytes != nullptr) { + std::string payload(reinterpret_cast(bytes), static_cast(len)); + TPaimonCommitMessage msg; + msg.__set_payload(payload); + messages.emplace_back(std::move(msg)); + env->ReleaseByteArrayElements(j_bytes, bytes, JNI_ABORT); + } + } + env->DeleteLocalRef(j_bytes); + } + env->DeleteLocalRef(j_payloads); + return Status::OK(); +} + +Status JniPaimonWriter::compact(bool full_compaction) { + // Compaction is triggered by Java's prepareCommit, not as a separate call + // from BE. For explicit compaction, we would need an additional JNI call. + return Status::OK(); +} + +Status JniPaimonWriter::abort() { + JNIEnv* env = nullptr; + { + JavaVM* jvm = nullptr; + jsize n_vms = 0; + JNI_GetCreatedJavaVMs(&jvm, 1, &n_vms); + jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_8); + if (env == nullptr) { + jvm->AttachCurrentThread(reinterpret_cast(&env), nullptr); + } + } + env->CallVoidMethod(_jni_writer_obj, _abort_id); + return Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in abort: "); +} + +// ──────────────────────────────────────────────────────────── +// JniPaimonCommitter +// ──────────────────────────────────────────────────────────── + +JniPaimonCommitter::JniPaimonCommitter(const TPaimonTableSink& sink) : _sink(sink) {} + +Status JniPaimonCommitter::_commit_impl( + const std::vector& messages, bool overwrite_mode, + const std::map* static_partition) { + // The actual commit is performed by FE (PaimonTransaction). + // BE only collects and forwards the CommitMessages. + // The commit logic is implemented in FE because it requires + // Paimon SDK classes (StreamTableCommit) and the coordinator role. + // + // This method exists for future use when we implement + // commit-via-CommitBE pattern (see design doc Section 5.3, Option 3). + return Status::NotSupported( + "Commit is handled by FE Coordinator (PaimonTransaction); " + "JniPaimonCommitter is reserved for future Commit-BE pattern"); +} + +Status JniPaimonCommitter::commit(const std::vector& messages) { + return _commit_impl(messages, false, nullptr); +} + +Status JniPaimonCommitter::overwrite(const std::vector& messages, + const std::map& static_partition) { + return _commit_impl(messages, true, &static_partition); +} + +Status JniPaimonCommitter::truncate_table() { + return Status::NotSupported("truncate_table is handled by FE Coordinator"); +} + +Status JniPaimonCommitter::truncate_partitions( + const std::vector>& partitions) { + return Status::NotSupported("truncate_partitions is handled by FE Coordinator"); +} + +Status JniPaimonCommitter::abort(const std::vector& messages) { + // Abort is best-effort: delete the data files. + // In the current architecture, FE handles abort through PaimonTransaction. + return Status::NotSupported("abort is handled by FE Coordinator"); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h new file mode 100644 index 00000000000000..97dd06af70321e --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +class RuntimeState; + +/// JNI-based Paimon write backend. +/// +/// Bridges Doris BE (C++) to the Paimon Java SDK via JNI. +/// On open() it loads the Java class `PaimonJniWriter` and caches +/// method IDs. On create_writer() it instantiates a wrapper that +/// delegates Block writes to the shared Java BatchTableWrite +/// instance; partition/bucket routing is handled by the Java SDK. +/// +/// Data path: +/// Block → Arrow RecordBatch → IPC Stream → JNI direct buffer → +/// Java ArrowStreamReader → Paimon InternalRow → BatchTableWrite +/// +/// Commit path: +/// Java prepareCommit() → CommitMessageSerializer → byte[][] +/// → C++ TPaimonCommitMessage → RuntimeState → FE Coordinator +class JniPaimonWriteBackend final : public IPaimonWriteBackend { +public: + JniPaimonWriteBackend() = default; + ~JniPaimonWriteBackend() override; + + Status open(const TPaimonTableSink& sink, RuntimeState* state) override; + + Status create_writer(std::unique_ptr* writer) override; + + Status create_committer(std::unique_ptr* committer) override; + + PaimonBackendType type() const override { return PaimonBackendType::JNI; } + + bool supports_compaction() const override { return true; } + bool supports_lookup_changelog_producer() const override { return true; } + bool supports_full_compaction_changelog_producer() const override { return true; } + bool supports_partial_update_with_dv() const override { return true; } + bool supports_aggregation_with_dv() const override { return true; } + +private: + Status _get_jni_env(JNIEnv** env); + Status _check_jni_exception(JNIEnv* env, const std::string& method_name); + + // JNI global references + jclass _jni_writer_cls = nullptr; + jobject _jni_writer_obj = nullptr; + + // Cached method IDs for the Java PaimonJniWriter class + jmethodID _open_id = nullptr; + jmethodID _write_id = nullptr; + jmethodID _prepare_commit_id = nullptr; + jmethodID _abort_id = nullptr; + jmethodID _close_id = nullptr; + + // Arrow memory pool for Block → Arrow conversion + std::unique_ptr> _arrow_pool; + + // Parsed sink configuration + TPaimonTableSink _sink; + + // Whether the JNI writer has been opened + bool _opened = false; +}; + +/// Writer backed by the shared JNI BatchTableWrite context. +/// Partition and bucket routing is handled by the Java Paimon SDK +/// internally during write(). +class JniPaimonWriter final : public IPaimonWriter { +public: + JniPaimonWriter(JNIEnv* env, jobject jni_writer_obj, jmethodID write_id, + jmethodID prepare_commit_id, jmethodID abort_id, + std::unique_ptr> arrow_pool, const TPaimonTableSink& sink); + ~JniPaimonWriter() override; + + Status write(RuntimeState* state, Block& block) override; + Status prepare_commit(std::vector& messages) override; + Status compact(bool full_compaction) override; + Status abort() override; + +private: + Status _write_projected_block(Block& block); + + JNIEnv* _env; + jobject _jni_writer_obj; + jmethodID _write_id; + jmethodID _prepare_commit_id; + jmethodID _abort_id; + std::unique_ptr> _arrow_pool; + TPaimonTableSink _sink; + int64_t _row_count = 0; +}; + +/// Committer that delegates to the Java Paimon SDK. +class JniPaimonCommitter final : public IPaimonCommitter { +public: + explicit JniPaimonCommitter(const TPaimonTableSink& sink); + ~JniPaimonCommitter() override = default; + + Status commit(const std::vector& messages) override; + Status overwrite(const std::vector& messages, + const std::map& static_partition) override; + Status truncate_table() override; + Status truncate_partitions( + const std::vector>& partitions) override; + Status abort(const std::vector& messages) override; + +private: + Status _commit_impl(const std::vector& messages, bool overwrite_mode, + const std::map* static_partition); + + TPaimonTableSink _sink; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend.h b/be/src/exec/sink/writer/paimon/paimon_write_backend.h new file mode 100644 index 00000000000000..595a355cba4bc0 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend.h @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "common/status.h" +#include "core/block/block.h" + +namespace doris { + +class RuntimeState; + +// ──────────────────────────────────────────────────────────── +// Backend type enumeration +// ──────────────────────────────────────────────────────────── + +enum class PaimonBackendType { + JNI, // Java SDK via JNI + FFI // Rust via C FFI +}; + +// ──────────────────────────────────────────────────────────── +// IPaimonWriter — manages all (partition, bucket) pairs +// ──────────────────────────────────────────────────────────── +// +// A single IPaimonWriter handles the entire table. Partition and +// bucket routing is performed internally by the Paimon SDK (Java +// or Rust), not by Doris. This is analogous to Iceberg's +// VIcebergPartitionWriter — Doris passes a full Block to write(), +// and the SDK routes rows to the correct file writers. + +class IPaimonWriter { +public: + virtual ~IPaimonWriter() = default; + + /// Write a Doris columnar block to this writer. + /// The block may contain rows belonging to different partitions and + /// buckets. Partition computation (BinaryRow partition), bucket + /// computation (hash % numBuckets), and routing to the correct + /// file writer (AppendOnlyWriter / KeyValueFileWriter) are all + /// handled internally by the Paimon SDK. + virtual Status write(RuntimeState* state, Block& block) = 0; + + /// Prepare commit: close files and produce commit messages. + /// The caller owns the returned messages and is responsible for forwarding + /// them to the commit coordinator (FE). + virtual Status prepare_commit(std::vector& messages) = 0; + + /// Trigger compaction for this (partition, bucket). + /// Only supported by the JNI backend; FFI backend returns NotSupported. + virtual Status compact(bool full_compaction) = 0; + + /// Abort: discard all data written by this writer. + /// Best-effort cleanup; errors are logged but not propagated. + virtual Status abort() = 0; +}; + +// ──────────────────────────────────────────────────────────── +// IPaimonCommitter — snapshot commit coordinator +// ──────────────────────────────────────────────────────────── + +class IPaimonCommitter { +public: + virtual ~IPaimonCommitter() = default; + + /// Commit new files in APPEND mode. + /// All messages are committed atomically in one snapshot. + virtual Status commit(const std::vector& messages) = 0; + + /// Commit in OVERWRITE mode, optionally scoped to static partitions. + virtual Status overwrite(const std::vector& messages, + const std::map& static_partition) = 0; + + /// Truncate the entire table. + virtual Status truncate_table() = 0; + + /// Truncate specific partitions. + virtual Status truncate_partitions( + const std::vector>& partitions) = 0; + + /// Abort: delete data files that were prepared but not yet committed. + virtual Status abort(const std::vector& messages) = 0; +}; + +// ──────────────────────────────────────────────────────────── +// IPaimonWriteBackend — factory for writers and committers +// ──────────────────────────────────────────────────────────── +// +// Java (JNI) and Rust (FFI) are interchangeable implementations of +// this interface. Rust is a functional subset of Java. The selection +// logic prefers Rust (performance) and falls back to Java when the +// table requires features that Rust does not yet support. +// +// v1 only implements JniPaimonWriteBackend. FfiPaimonWriteBackend +// exists as a stub that returns NotSupported for all methods. + +class IPaimonWriteBackend { +public: + virtual ~IPaimonWriteBackend() = default; + + /// Initialize the backend from the Thrift sink description. + virtual Status open(const TPaimonTableSink& sink, RuntimeState* state) = 0; + + /// Create a writer for the table. One writer manages all partitions + /// and buckets; routing is handled internally by the Paimon SDK. + virtual Status create_writer(std::unique_ptr* writer) = 0; + + /// Create a committer. One per sink; shared across all writers. + virtual Status create_committer(std::unique_ptr* committer) = 0; + + /// Which backend type is this? + virtual PaimonBackendType type() const = 0; + + // ──── Capability queries ──────────────────────────────── + + virtual bool supports_compaction() const = 0; + virtual bool supports_lookup_changelog_producer() const = 0; + virtual bool supports_full_compaction_changelog_producer() const = 0; + virtual bool supports_partial_update_with_dv() const = 0; + virtual bool supports_aggregation_with_dv() const = 0; +}; + +// ──────────────────────────────────────────────────────────── +// Factory +// ──────────────────────────────────────────────────────────── + +class PaimonWriteBackendFactory { +public: + /// Create the appropriate backend based on the sink configuration. + static Status create(const TPaimonTableSink& sink, + std::unique_ptr* backend); + + /// Select the best backend type for the given table configuration. + /// v1: always returns JNI. + /// v2: returns FFI when the table does not require Java-only features. + static PaimonBackendType select_backend_type(const TPaimonTableSink& sink); +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp new file mode 100644 index 00000000000000..61cda74725a283 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +Status PaimonWriteBackendFactory::create(const TPaimonTableSink& sink, + std::unique_ptr* backend) { + PaimonBackendType type = select_backend_type(sink); + + switch (type) { + case PaimonBackendType::JNI: { + *backend = std::make_unique(); + return Status::OK(); + } + case PaimonBackendType::FFI: { + *backend = std::make_unique(); + return Status::OK(); + } + } + return Status::InternalError("Unknown Paimon backend type: {}", static_cast(type)); +} + +PaimonBackendType PaimonWriteBackendFactory::select_backend_type(const TPaimonTableSink& sink) { + // v1: Always use JNI backend. + // + // v2: When Rust FFI backend is ready, use the following logic: + // if (sink.__isset.backend_type && sink.backend_type == TPaimonWriteBackendType::FFI) { + // return PaimonBackendType::FFI; + // } + // return PaimonBackendType::JNI; // default fallback + // + // The FFI backend is preferred when: + // - Table is append-only (no primary keys) + // - No Lookup/FullCompaction changelog producer + // - No DeletionVectors + PartialUpdate/Aggregation + if (sink.__isset.backend_type) { + if (sink.backend_type == TPaimonWriteBackendType::FFI) { + return PaimonBackendType::FFI; + } + } + return PaimonBackendType::JNI; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/vpaimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/vpaimon_table_writer.cpp new file mode 100644 index 00000000000000..a217829eae6516 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/vpaimon_table_writer.cpp @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/vpaimon_table_writer.h" + +#include "common/logging.h" +#include "core/block/block.h" +#include "runtime/runtime_state.h" + +namespace doris { + +VPaimonTableWriter::VPaimonTableWriter(const TDataSink& t_sink, + const VExprContextSPtrs& output_exprs, + std::shared_ptr dep, + std::shared_ptr fin_dep) + : AsyncResultWriter(output_exprs, std::move(dep), std::move(fin_dep)), _t_sink(t_sink) { + DCHECK(_t_sink.__isset.paimon_table_sink); +} + +Status VPaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { + _state = state; + _operator_profile = profile; + + // Register profile counters + _written_rows_counter = ADD_COUNTER(_operator_profile, "WrittenRows", TUnit::UNIT); + _written_bytes_counter = ADD_COUNTER(_operator_profile, "WrittenBytes", TUnit::BYTES); + _send_data_timer = ADD_TIMER(_operator_profile, "SendDataTime"); + _project_timer = ADD_CHILD_TIMER(_operator_profile, "ProjectTime", "SendDataTime"); + _arrow_convert_timer = ADD_CHILD_TIMER(_operator_profile, "ArrowConvertTime", "SendDataTime"); + _file_store_write_timer = + ADD_CHILD_TIMER(_operator_profile, "FileStoreWriteTime", "SendDataTime"); + _open_timer = ADD_TIMER(_operator_profile, "OpenTime"); + _close_timer = ADD_TIMER(_operator_profile, "CloseTime"); + _prepare_commit_timer = ADD_TIMER(_operator_profile, "PrepareCommitTime"); + _serialize_commit_messages_timer = ADD_TIMER(_operator_profile, "SerializeCommitMessagesTime"); + _commit_payload_count = ADD_COUNTER(_operator_profile, "CommitPayloadCount", TUnit::UNIT); + _commit_payload_bytes_counter = + ADD_COUNTER(_operator_profile, "CommitPayloadBytes", TUnit::BYTES); + + SCOPED_TIMER(_open_timer); + + // Create backend via factory — one independent JNI/FFI object per writer + RETURN_IF_ERROR(PaimonWriteBackendFactory::create(_t_sink.paimon_table_sink, &_backend)); + RETURN_IF_ERROR(_backend->open(_t_sink.paimon_table_sink, state)); + + // Create a single writer — routing handled internally by Paimon SDK + RETURN_IF_ERROR(_backend->create_writer(&_writer)); + + LOG(INFO) << "VPaimonTableWriter opened: table=" << _t_sink.paimon_table_sink.tb_name + << ", backend=" << static_cast(_backend->type()); + return Status::OK(); +} + +Status VPaimonTableWriter::write(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + SCOPED_TIMER(_send_data_timer); + + // Project: apply output expressions + Block output_block; + { + SCOPED_TIMER(_project_timer); + RETURN_IF_ERROR(_projection_block(block, &output_block)); + } + + COUNTER_UPDATE(_written_rows_counter, block.rows()); + COUNTER_UPDATE(_written_bytes_counter, block.bytes()); + _state->update_num_rows_load_total(block.rows()); + _state->update_num_bytes_load_total(block.bytes()); + + if (_writer) { + SCOPED_TIMER(_file_store_write_timer); + RETURN_IF_ERROR(_writer->write(_state, output_block)); + } + _written_rows += block.rows(); + return Status::OK(); +} + +Status VPaimonTableWriter::close(Status status) { + SCOPED_TIMER(_close_timer); + + std::vector messages; + if (status.ok() && _writer) { + { + SCOPED_TIMER(_prepare_commit_timer); + Status prep_st = _writer->prepare_commit(messages); + if (!prep_st.ok()) { + status = prep_st; + } + } + + if (status.ok()) { + COUNTER_UPDATE(_commit_payload_count, static_cast(messages.size())); + for (const auto& msg : messages) { + if (msg.__isset.payload) { + COUNTER_UPDATE(_commit_payload_bytes_counter, + static_cast(msg.payload.size())); + } + } + + if (!messages.empty()) { + _state->add_paimon_commit_messages(messages); + LOG(INFO) << "Paimon writer closed: " << messages.size() + << " commit messages, total rows=" << _written_rows; + } + } + } + + if (!status.ok()) { + LOG(WARNING) << "Paimon writer closing with error: " << status.to_string(); + if (_writer) { + Status abort_st = _writer->abort(); + if (!abort_st.ok()) { + LOG(WARNING) << "Paimon writer abort failed: " << abort_st.to_string(); + } + } + } + + _writer.reset(); + _backend.reset(); + return status; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/vpaimon_table_writer.h b/be/src/exec/sink/writer/paimon/vpaimon_table_writer.h new file mode 100644 index 00000000000000..b54d254ac9a725 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/vpaimon_table_writer.h @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/sink/writer/async_result_writer.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "exprs/vexpr_fwd.h" +#include "runtime/runtime_profile.h" + +namespace doris { + +class ObjectPool; +class RuntimeState; + +/// VPaimonTableWriter is the single AsyncResultWriter for a Paimon table +/// sink. One writer = one IO thread, analogous to Iceberg's +/// VIcebergTableWriter. +/// +/// Partition and bucket routing is delegated to the Paimon SDK (Java or +/// Rust) via IPaimonWriter::write(). Doris passes complete Blocks — the +/// SDK internally computes partition values, bucket ids, and routes rows +/// to the correct file writers. +/// +/// Architecture: +/// PaimonTableSinkOperatorX +/// │ sink_impl() → AsyncWriterSink::sink() (no routing) +/// ▼ +/// VPaimonTableWriter (single AsyncResultWriter) +/// │ write() → IPaimonWriter::write() +/// │ → Paimon SDK internal routing +/// │ → AppendOnlyWriter / KeyValueFileWriter +/// │ → CompactManager (auto compaction) +/// ▼ +/// close() → prepareCommit() → CommitMessage[] +/// +/// Commit flow: +/// close() → writer->prepare_commit() +/// → collect TPaimonCommitMessage[] +/// → RuntimeState::add_paimon_commit_messages() +/// → RPC to FE Coordinator +class VPaimonTableWriter final : public AsyncResultWriter { +public: + VPaimonTableWriter(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs, + std::shared_ptr dep, std::shared_ptr fin_dep); + + ~VPaimonTableWriter() override = default; + + Status init_properties(ObjectPool* pool) { return Status::OK(); } + + Status open(RuntimeState* state, RuntimeProfile* profile) override; + + Status write(RuntimeState* state, Block& block) override; + + Status close(Status status) override; + +private: + TDataSink _t_sink; + RuntimeState* _state = nullptr; + + // The backend abstraction — JNI or FFI, one per writer + std::unique_ptr _backend; + + // Single writer managing all partitions and buckets (SDK-internal routing) + std::unique_ptr _writer; + + // Statistics (note: _written_rows is inherited from ResultWriter) + int64_t _written_bytes = 0; + + // Profile counters + RuntimeProfile::Counter* _written_rows_counter = nullptr; + RuntimeProfile::Counter* _written_bytes_counter = nullptr; + RuntimeProfile::Counter* _send_data_timer = nullptr; + RuntimeProfile::Counter* _project_timer = nullptr; + RuntimeProfile::Counter* _arrow_convert_timer = nullptr; + RuntimeProfile::Counter* _file_store_write_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _prepare_commit_timer = nullptr; + RuntimeProfile::Counter* _serialize_commit_messages_timer = nullptr; + RuntimeProfile::Counter* _commit_payload_count = nullptr; + RuntimeProfile::Counter* _commit_payload_bytes_counter = nullptr; +}; + +} // namespace doris diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 2a58dc31fc9247..22259533a7961e 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -543,6 +543,17 @@ class RuntimeState { _mc_commit_datas.emplace_back(mc_commit_data); } + std::vector paimon_commit_messages() const { + std::lock_guard lock(_paimon_commit_messages_mutex); + return _paimon_commit_messages; + } + + void add_paimon_commit_messages(const std::vector& commit_messages) { + std::lock_guard lock(_paimon_commit_messages_mutex); + _paimon_commit_messages.insert(_paimon_commit_messages.end(), commit_messages.begin(), + commit_messages.end()); + } + // local runtime filter mgr, the runtime filter do not have remote target or // not need local merge should regist here. the instance exec finish, the local // runtime filter mgr can release the memory of local runtime filter @@ -975,6 +986,9 @@ class RuntimeState { mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; + mutable std::mutex _paimon_commit_messages_mutex; + std::vector _paimon_commit_messages; + std::vector> _op_id_to_local_state; std::unique_ptr _sink_local_state; diff --git a/fe/be-java-extensions/paimon-scanner/pom.xml b/fe/be-java-extensions/paimon-scanner/pom.xml index 60ff39c2681b11..b1c74ef1f647eb 100644 --- a/fe/be-java-extensions/paimon-scanner/pom.xml +++ b/fe/be-java-extensions/paimon-scanner/pom.xml @@ -61,6 +61,11 @@ under the License. paimon-format + + org.apache.arrow + arrow-vector + + diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java new file mode 100644 index 00000000000000..2eac6e95f56229 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java @@ -0,0 +1,982 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.doris.common.security.authentication.PreExecutionAuthenticator; +import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.holders.NullableTimeStampMicroHolder; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.disk.IOManagerImpl; +import org.apache.paimon.io.BundleRecords; +import org.apache.paimon.io.DataOutputSerializer; +import org.apache.paimon.memory.HeapMemorySegmentPool; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageSerializer; +import org.apache.paimon.table.sink.TableWriteImpl; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarBinaryType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.lang.reflect.Constructor; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * JNI entry point for Paimon write operations. + * + * Called from C++ (VPaimonTableWriter) via JNI. Data path: + * + * C++ Block → Arrow IPC Stream → direct memory address + * → PaimonJniWriter.write(address, length) + * → ArrowStreamReader → VectorSchemaRoot + * → extractColumnValues (column-major typed extraction) + * → group by writer.getPartition(row) + writer.getBucket(row) + * → writer.writeBundle(partition, bucket, bundle) [batch write] + * + * Commit path: + * + * VPaimonTableWriter::close() → JNI → PaimonJniWriter.prepareCommit() + * → CommitMessageSerializer → byte[][] + * → C++ collects TPaimonCommitMessage → FE PaimonTransaction + */ +public class PaimonJniWriter { + private static final Logger LOG = LoggerFactory.getLogger(PaimonJniWriter.class); + + private static final String OPTION_PREFIX_PAIMON = "paimon."; + private static final String OPTION_PREFIX_HADOOP = "hadoop."; + private static final String KEY_SERIALIZED_TABLE = "serialized_table"; + private static final String KEY_COMMIT_IDENTIFIER = "doris.commit_identifier"; + private static final String KEY_COMMIT_USER = "doris.commit_user"; + private static final String KEY_SPILL_DIR = "paimon_jni_spill_dir"; + + static final int COMMIT_PAYLOAD_HEADER_BYTES = 12; + static final int MAX_COMMIT_PAYLOAD_BYTES = 8 * 1024 * 1024; + static final int DEFAULT_COMMIT_CHUNK_SIZE = 512; + + private BatchTableWrite writer; + private TableWriteImpl tableWrite; + + private final BufferAllocator allocator; + private final CommitMessageSerializer serializer = new CommitMessageSerializer(); + private final ClassLoader classLoader; + + private String tableLocation; + private PreExecutionAuthenticator preExecutionAuthenticator; + + private Map paimonFieldMap; + private DataType[] targetTypes; + private long commitIdentifier; + + private IOManager ioManager; + private HeapMemorySegmentPool memorySegmentPool; + + private static volatile Constructor directByteBufferConstructor; + + public PaimonJniWriter() { + this.allocator = new RootAllocator(Long.MAX_VALUE); + this.classLoader = this.getClass().getClassLoader(); + configureLogLevels(); + } + + // ──────────────────────────────────────────────────────────── + // JNI entry points (called from C++) + // ──────────────────────────────────────────────────────────── + + /** + * Initialize the writer. Called once per BE fragment. + * + * @param tableLocation Paimon table root path + * @param options merged options map (Paimon keys prefixed "paimon.", + * Hadoop keys prefixed "hadoop.") + + * @param columnNames output column names + */ + public void open(String tableLocation, Map options, + String[] columnNames) throws Exception { + this.tableLocation = tableLocation; + Thread.currentThread().setContextClassLoader(classLoader); + + Map paimonOpts = extractPrefixed(options, OPTION_PREFIX_PAIMON); + Map hadoopOpts = extractPrefixed(options, OPTION_PREFIX_HADOOP); + + if (!paimonOpts.containsKey("warehouse") && tableLocation != null) { + paimonOpts.put("warehouse", tableLocation); + } + + this.preExecutionAuthenticator = PreExecutionAuthenticatorCache.getAuthenticator(options); + preExecutionAuthenticator.execute(() -> { + try { + LOG.info("PaimonJniWriter opening: location={}, columns={}", + tableLocation, columnNames != null ? columnNames.length : 0); + + Table table = loadTable(options, paimonOpts, hadoopOpts); + this.commitIdentifier = Long.parseLong( + options.getOrDefault(KEY_COMMIT_IDENTIFIER, "-1")); + + initFieldMap(table.rowType().getFields()); + RowType writeType = buildWriteType(columnNames); + initWriter(table, options); + + if (!isFullRowWrite(table.rowType(), writeType)) { + if (!table.primaryKeys().isEmpty()) { + throw new UnsupportedOperationException( + "Paimon primary-key write requires full table row type"); + } + this.writer.withWriteType(writeType); + } + return null; + } catch (Throwable t) { + throw new RuntimeException("PaimonJniWriter open failed: location=" + + tableLocation, t); + } + }); + } + + /** + * Write a batch of rows. Called once per Arrow IPC buffer from C++. + * + * @param address native memory address of the Arrow IPC Stream bytes + * @param length number of bytes + */ + public void write(long address, int length) throws Exception { + preExecutionAuthenticator.execute(() -> { + try { + ByteBuffer directBuffer = getDirectBuffer(address, length); + try (ArrowStreamReader reader = new ArrowStreamReader( + new DirectBufInputStream(directBuffer), allocator)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + while (reader.loadNextBatch()) { + writeBatch(root); + } + } + return null; + } catch (Throwable t) { + throw new RuntimeException("PaimonJniWriter write failed: addr=" + + address + ", len=" + length, t); + } + }); + } + + /** + * Prepare commit: flush all in-memory data, close files, serialize commit messages. + * + * @return byte[][] each element is a DPCM-framed serialized CommitMessage chunk + */ + public byte[][] prepareCommit() throws Exception { + if (writer == null) { + return new byte[0][]; + } + return preExecutionAuthenticator.execute(() -> { + try { + List messages = doPrepareCommit(); + if (messages == null || messages.isEmpty()) { + LOG.info("PaimonJniWriter prepareCommit: empty, location={}", tableLocation); + return new byte[0][]; + } + LOG.info("PaimonJniWriter prepareCommit: {} messages", messages.size()); + return serializeMessages(messages); + } catch (Throwable t) { + throw new RuntimeException("PaimonJniWriter prepareCommit failed: location=" + + tableLocation, t); + } + }); + } + + /** + * Abort: discard written data files. Called on error. + */ + public void abort() throws Exception { + try { + if (preExecutionAuthenticator != null) { + preExecutionAuthenticator.execute(() -> { + closeWriterResources(false); + return null; + }); + } + } catch (Exception e) { + LOG.error("PaimonJniWriter abort failed", e); + throw e; + } + } + + /** + * Close: release all resources. + */ + public void close() throws Exception { + try { + if (preExecutionAuthenticator != null) { + preExecutionAuthenticator.execute(() -> { + closeWriterResources(true); + return null; + }); + } + } catch (Exception e) { + LOG.warn("PaimonJniWriter close error", e); + throw e; + } + } + + // ──────────────────────────────────────────────────────────── + // Initialization helpers + // ──────────────────────────────────────────────────────────── + + private static Map extractPrefixed(Map options, + String prefix) { + return options.entrySet().stream() + .filter(kv -> kv.getKey().startsWith(prefix)) + .collect(Collectors.toMap( + kv -> kv.getKey().substring(prefix.length()), + Map.Entry::getValue)); + } + + private Table loadTable(Map options, + Map paimonOpts, + Map hadoopOpts) throws Exception { + if (options.containsKey(KEY_SERIALIZED_TABLE)) { + return PaimonUtils.deserialize(options.get(KEY_SERIALIZED_TABLE)); + } + Catalog catalog = createCatalog(paimonOpts, hadoopOpts); + String dbName = options.getOrDefault("db_name", "default"); + String tblName = options.getOrDefault("table_name", "paimon_table"); + return catalog.getTable(Identifier.create(dbName, tblName)); + } + + private void initFieldMap(List fields) { + this.paimonFieldMap = new HashMap<>(); + for (DataField field : fields) { + this.paimonFieldMap.put(field.name(), field); + } + } + + private void initWriter(Table table, Map options) throws Exception { + if (!(table instanceof FileStoreTable)) { + org.apache.paimon.table.sink.BatchWriteBuilder builder = + table.newBatchWriteBuilder(); + if (isOverwriteMode(options)) { + Map staticPart = getStaticPartition(options); + if (staticPart != null && !staticPart.isEmpty()) { + builder.withOverwrite(staticPart); + } else { + builder.withOverwrite(); + } + } + this.writer = builder.newWrite(); + return; + } + + FileStoreTable fileStoreTable = (FileStoreTable) table; + CoreOptions coreOptions = CoreOptions.fromMap(fileStoreTable.options()); + + String commitUser = options.get(KEY_COMMIT_USER); + if (commitUser == null || commitUser.isEmpty()) { + commitUser = coreOptions.createCommitUser(); + } + + this.tableWrite = fileStoreTable.newWrite(commitUser); + if (isOverwriteMode(options)) { + this.tableWrite.withIgnorePreviousFiles(true); + } + this.writer = this.tableWrite; + initSpillIfNeeded(coreOptions, options); + } + + private static boolean isOverwriteMode(Map options) { + String mode = options.get("doris.write_mode"); + return "OVERWRITE".equalsIgnoreCase(mode); + } + + private static Map getStaticPartition(Map options) { + Map result = new HashMap<>(); + String prefix = "doris.static_partition."; + for (Map.Entry e : options.entrySet()) { + if (e.getKey().startsWith(prefix)) { + result.put(e.getKey().substring(prefix.length()), e.getValue()); + } + } + return result; + } + + private void initSpillIfNeeded(CoreOptions coreOptions, Map options) + throws Exception { + if (!coreOptions.writeBufferSpillable()) { + return; + } + String spillDir = options.get(KEY_SPILL_DIR); + if (spillDir == null || spillDir.isEmpty()) { + spillDir = System.getProperty("java.io.tmpdir"); + } else { + Files.createDirectories(Paths.get(spillDir)); + } + this.ioManager = new IOManagerImpl(spillDir); + this.memorySegmentPool = new HeapMemorySegmentPool( + coreOptions.writeBufferSize(), coreOptions.pageSize()); + this.writer.withIOManager(ioManager).withMemoryPool(memorySegmentPool); + LOG.info("PaimonJniWriter spill enabled: dir={}", spillDir); + } + + // ──────────────────────────────────────────────────────────── + // Data writing + // ──────────────────────────────────────────────────────────── + + private void writeBatch(VectorSchemaRoot root) throws Exception { + int rowCount = root.getRowCount(); + if (rowCount == 0) { + return; + } + List fields = root.getSchema().getFields(); + List vectors = root.getFieldVectors(); + int colCount = fields.size(); + DataType[] currentTypes = resolveTargetTypes(fields); + + // Column-major extraction: one instanceof per column, N direct getter calls. + Object[][] columnValues = new Object[colCount][]; + for (int col = 0; col < colCount; col++) { + columnValues[col] = extractColumnValues(vectors.get(col), fields.get(col), + currentTypes[col], rowCount); + } + + // Group rows by (partition, bucket) using the Paimon SDK's routing methods, + // then write each group via writeBundle for batch-level I/O optimization. + Map> groups = new LinkedHashMap<>(); + for (int r = 0; r < rowCount; r++) { + GenericRow row = new GenericRow(colCount); + for (int c = 0; c < colCount; c++) { + row.setField(c, columnValues[c][r]); + } + BinaryRow partition = writer.getPartition(row); + int bucket = writer.getBucket(row); + groups.computeIfAbsent(new PartitionBucketKey(partition, bucket), + k -> new ArrayList<>()).add(row); + } + + for (Map.Entry> entry : groups.entrySet()) { + PartitionBucketKey key = entry.getKey(); + writer.writeBundle(key.partition, key.bucket, new ListBundleRecords(entry.getValue())); + } + } + + /** + * Extract all values for one column using a type-specific fast path. + * Single instanceof per column — no per-cell type dispatch. + */ + private Object[] extractColumnValues(FieldVector vector, Field arrowField, + DataType targetType, int rowCount) { + Object[] values = new Object[rowCount]; + + // ── Primitive int types ────────────────────────────── + if (vector instanceof IntVector) { + IntVector v = (IntVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + if (vector instanceof BigIntVector) { + BigIntVector v = (BigIntVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + if (vector instanceof SmallIntVector) { + SmallIntVector v = (SmallIntVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + if (vector instanceof TinyIntVector) { + TinyIntVector v = (TinyIntVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + if (vector instanceof Float4Vector) { + Float4Vector v = (Float4Vector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + if (vector instanceof Float8Vector) { + Float8Vector v = (Float8Vector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + if (vector instanceof BitVector) { + BitVector v = (BitVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i) == 1; + } + return values; + } + if (vector instanceof DateDayVector) { + DateDayVector v = (DateDayVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + + // ── String / Binary ────────────────────────────────── + if (vector instanceof VarCharVector) { + VarCharVector v = (VarCharVector) vector; + if (targetType instanceof BinaryType || targetType instanceof VarBinaryType) { + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + } else { + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : BinaryString.fromBytes(v.get(i)); + } + } + return values; + } + if (vector instanceof VarBinaryVector) { + VarBinaryVector v = (VarBinaryVector) vector; + for (int i = 0; i < rowCount; i++) { + values[i] = v.isNull(i) ? null : v.get(i); + } + return values; + } + + // ── Timestamp ──────────────────────────────────────── + if (vector instanceof TimeStampMicroVector) { + TimeStampMicroVector v = (TimeStampMicroVector) vector; + NullableTimeStampMicroHolder holder = new NullableTimeStampMicroHolder(); + for (int i = 0; i < rowCount; i++) { + if (v.isNull(i)) { + values[i] = null; + } else { + v.get(i, holder); + values[i] = Timestamp.fromMicros(holder.value); + } + } + return values; + } + + // ── Decimal ────────────────────────────────────────── + if (vector instanceof DecimalVector) { + DecimalVector v = (DecimalVector) vector; + int precision = v.getPrecision(); + int scale = v.getScale(); + org.apache.arrow.memory.ArrowBuf dataBuf = v.getDataBuffer(); + for (int i = 0; i < rowCount; i++) { + if (v.isNull(i)) { + values[i] = null; + } else { + BigDecimal bd = getBigDecimalFromArrowBuf(dataBuf, i, scale, + DecimalVector.TYPE_WIDTH); + values[i] = Decimal.fromBigDecimal(bd, precision, scale); + } + } + return values; + } + + // ── Complex types (Array / Map / Struct) — fallback ── + for (int i = 0; i < rowCount; i++) { + if (vector.isNull(i)) { + values[i] = null; + } else { + values[i] = convertToPaimonType(vector.getObject(i), arrowField, targetType); + } + } + return values; + } + + // ──────────────────────────────────────────────────────────── + // Arrow → Paimon type conversion (used by extractColumnValues fallback) + // ──────────────────────────────────────────────────────────── + + @SuppressWarnings({"unchecked", "rawtypes"}) + private Object convertToPaimonType(Object val, Field arrowField, DataType targetType) { + if (val == null) { + return null; + } + if (targetType instanceof BinaryType || targetType instanceof VarBinaryType) { + if (val instanceof byte[]) { + return val; + } + if (val instanceof BinaryString) { + return ((BinaryString) val).toBytes(); + } + if (val instanceof org.apache.arrow.vector.util.Text) { + return ((org.apache.arrow.vector.util.Text) val).copyBytes(); + } + if (val instanceof String) { + return ((String) val).getBytes(StandardCharsets.UTF_8); + } + return val.toString().getBytes(StandardCharsets.UTF_8); + } + if (val instanceof BinaryString) { + return val; + } + if (val instanceof byte[]) { + return BinaryString.fromBytes((byte[]) val); + } + if (val instanceof org.apache.arrow.vector.util.Text) { + return BinaryString.fromBytes(((org.apache.arrow.vector.util.Text) val).copyBytes()); + } + if (val instanceof org.apache.hadoop.io.Text) { + org.apache.hadoop.io.Text t = (org.apache.hadoop.io.Text) val; + return BinaryString.fromBytes(t.getBytes(), 0, t.getLength()); + } + if (val instanceof CharSequence) { + return BinaryString.fromString(val.toString()); + } + + ArrowType.ArrowTypeID typeID = arrowField != null + ? arrowField.getType().getTypeID() : null; + + if (val instanceof LocalDateTime) { + return Timestamp.fromLocalDateTime((LocalDateTime) val); + } + if (val instanceof Long && typeID == ArrowType.ArrowTypeID.Timestamp) { + return Timestamp.fromMicros((Long) val); + } + if (val instanceof Integer && typeID == ArrowType.ArrowTypeID.Date) { + return val; + } + if (val instanceof java.time.LocalDate) { + return (int) ((java.time.LocalDate) val).toEpochDay(); + } + if (val instanceof BigDecimal) { + BigDecimal bd = (BigDecimal) val; + return Decimal.fromBigDecimal(bd, bd.precision(), bd.scale()); + } + if (targetType instanceof RowType && val instanceof Map) { + return convertStruct((Map) val, (RowType) targetType, arrowField); + } + if (targetType instanceof MapType && val instanceof List) { + return convertMap((List) val, (MapType) targetType, arrowField); + } + if (targetType instanceof ArrayType && val instanceof List) { + return convertArray((List) val, (ArrayType) targetType, arrowField); + } + if (val instanceof byte[]) { + return val; + } + return val; + } + + private GenericRow convertStruct(Map mapVal, RowType rowType, Field arrowField) { + List childFields = rowType.getFields(); + GenericRow structRow = new GenericRow(childFields.size()); + for (int i = 0; i < childFields.size(); i++) { + DataField childField = childFields.get(i); + Object childVal = mapVal.get(childField.name()); + Field childArrowField = findChildField(arrowField, childField.name()); + structRow.setField(i, + convertToPaimonType(childVal, childArrowField, childField.type())); + } + return structRow; + } + + private GenericMap convertMap(List list, MapType mapType, Field arrowField) { + Field keyArrowField = null; + Field valueArrowField = null; + if (arrowField != null && !arrowField.getChildren().isEmpty()) { + Field entries = arrowField.getChildren().get(0); + if (entries.getChildren().size() >= 2) { + keyArrowField = entries.getChildren().get(0); + valueArrowField = entries.getChildren().get(1); + } + } + String keyName = keyArrowField != null ? keyArrowField.getName() : "key"; + String valueName = valueArrowField != null ? valueArrowField.getName() : "value"; + + Map converted = new HashMap<>(); + for (Object element : list) { + if (element instanceof Map) { + Map kv = (Map) element; + Object k = convertToPaimonType(kv.get(keyName), keyArrowField, mapType.getKeyType()); + Object v = convertToPaimonType(kv.get(valueName), valueArrowField, + mapType.getValueType()); + converted.put(k, v); + } + } + return new GenericMap(converted); + } + + private GenericArray convertArray(List list, ArrayType arrayType, Field arrowField) { + Object[] converted = new Object[list.size()]; + Field childArrowField = null; + if (arrowField != null && !arrowField.getChildren().isEmpty()) { + childArrowField = arrowField.getChildren().get(0); + } + for (int i = 0; i < list.size(); i++) { + converted[i] = convertToPaimonType(list.get(i), childArrowField, + arrayType.getElementType()); + } + return new GenericArray(converted); + } + + private static Field findChildField(Field parent, String name) { + if (parent == null || parent.getChildren() == null) { + return null; + } + for (Field f : parent.getChildren()) { + if (f.getName().equals(name)) { + return f; + } + } + return null; + } + + // ──────────────────────────────────────────────────────────── + // Commit & serialization + // ──────────────────────────────────────────────────────────── + + private List doPrepareCommit() throws Exception { + return tableWrite != null && commitIdentifier > 0 + ? tableWrite.prepareCommit(true, commitIdentifier) + : writer.prepareCommit(); + } + + private byte[][] serializeMessages(List messages) throws Exception { + int chunkSize = DEFAULT_COMMIT_CHUNK_SIZE; + List payloads = new ArrayList<>(); + int i = 0; + while (i < messages.size()) { + int end = Math.min(i + chunkSize, messages.size()); + byte[] payload = serializeChunk(messages.subList(i, end)); + if (payload.length > MAX_COMMIT_PAYLOAD_BYTES && chunkSize > 1) { + chunkSize = Math.max(1, chunkSize / 2); + continue; + } + payloads.add(payload); + i = end; + } + return payloads.toArray(new byte[0][]); + } + + private byte[] serializeChunk(List messages) throws Exception { + DataOutputSerializer out = new DataOutputSerializer(1024); + serializer.serializeList(messages, out); + byte[] data = out.getCopyOfBuffer(); + int len = data.length; + int version = serializer.getVersion(); + + byte[] payload = new byte[COMMIT_PAYLOAD_HEADER_BYTES + len]; + // Magic bytes "DPCM" + payload[0] = 'D'; + payload[1] = 'P'; + payload[2] = 'C'; + payload[3] = 'M'; + // Version (big-endian) + payload[4] = (byte) ((version >>> 24) & 0xFF); + payload[5] = (byte) ((version >>> 16) & 0xFF); + payload[6] = (byte) ((version >>> 8) & 0xFF); + payload[7] = (byte) (version & 0xFF); + // Length (big-endian) + payload[8] = (byte) ((len >>> 24) & 0xFF); + payload[9] = (byte) ((len >>> 16) & 0xFF); + payload[10] = (byte) ((len >>> 8) & 0xFF); + payload[11] = (byte) (len & 0xFF); + System.arraycopy(data, 0, payload, COMMIT_PAYLOAD_HEADER_BYTES, len); + return payload; + } + + // ──────────────────────────────────────────────────────────── + // Schema & type resolution + // ──────────────────────────────────────────────────────────── + + private RowType buildWriteType(String[] columnNames) { + if (columnNames == null || columnNames.length == 0) { + throw new IllegalArgumentException( + "PaimonJniWriter requires explicit column names"); + } + List writeFields = new ArrayList<>(columnNames.length); + this.targetTypes = new DataType[columnNames.length]; + for (int i = 0; i < columnNames.length; i++) { + DataField field = paimonFieldMap.get(columnNames[i]); + if (field == null) { + throw new IllegalArgumentException( + "Paimon column '" + columnNames[i] + "' not found in table schema"); + } + writeFields.add(field); + this.targetTypes[i] = field.type(); + } + return new RowType(writeFields); + } + + private DataType[] resolveTargetTypes(List fields) { + if (targetTypes != null && targetTypes.length == fields.size()) { + return targetTypes; + } + DataType[] result = new DataType[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + DataField df = paimonFieldMap.get(fields.get(i).getName()); + if (df == null) { + throw new IllegalArgumentException( + "Paimon column '" + fields.get(i).getName() + "' not found"); + } + result[i] = df.type(); + } + return result; + } + + static boolean isFullRowWrite(RowType tableRowType, RowType writeType) { + return tableRowType.getFieldNames().equals(writeType.getFieldNames()); + } + + // ──────────────────────────────────────────────────────────── + // Resource management + // ──────────────────────────────────────────────────────────── + + private void closeWriterResources(boolean closeAllocator) throws Exception { + try { + if (writer != null) { + writer.close(); + writer = null; + } + tableWrite = null; + if (ioManager != null) { + ioManager.close(); + ioManager = null; + } + memorySegmentPool = null; + if (closeAllocator && allocator != null) { + allocator.close(); + } + } catch (Throwable t) { + throw new RuntimeException("PaimonJniWriter close failed", t); + } + } + + // ──────────────────────────────────────────────────────────── + // Utilities + // ──────────────────────────────────────────────────────────── + + private static BigDecimal getBigDecimalFromArrowBuf( + org.apache.arrow.memory.ArrowBuf byteBuf, + int index, int scale, int byteWidth) { + byte[] value = new byte[byteWidth]; + long startIndex = (long) index * byteWidth; + byteBuf.getBytes(startIndex, value, 0, byteWidth); + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + int stop = byteWidth / 2; + for (int i = 0; i < stop; i++) { + byte temp = value[i]; + int j = (byteWidth - 1) - i; + value[i] = value[j]; + value[j] = temp; + } + } + return new BigDecimal(new BigInteger(value), scale); + } + + private static ByteBuffer getDirectBuffer(long address, int length) throws Exception { + Constructor ctor = directByteBufferConstructor; + if (ctor == null) { + Class cls = Class.forName("java.nio.DirectByteBuffer"); + ctor = cls.getDeclaredConstructor(long.class, int.class); + ctor.setAccessible(true); + directByteBufferConstructor = ctor; + } + return (ByteBuffer) ctor.newInstance(address, length); + } + + private static Catalog createCatalog(Map paimonOpts, + Map hadoopOpts) { + Options options = new Options(); + paimonOpts.forEach(options::set); + + Configuration conf = new Configuration(); + hadoopOpts.forEach(conf::set); + + String hadoopConfDir = options.getString("hadoop-conf-dir", (String) null); + if (hadoopConfDir != null) { + conf.addResource(new Path(hadoopConfDir + "core-site.xml")); + conf.addResource(new Path(hadoopConfDir + "hdfs-site.xml")); + } + String hiveConfDir = options.getString("hive-conf-dir", (String) null); + if (hiveConfDir != null) { + conf.addResource(new Path(hiveConfDir + "hive-site.xml")); + } + paimonOpts.forEach(conf::set); + + CatalogContext ctx = CatalogContext.create(options, conf); + return CatalogFactory.createCatalog(ctx); + } + + private static void configureLogLevels() { + try { + for (String name : new String[] { + "org.apache.paimon.shade.org.apache.parquet", + "org.apache.paimon" + }) { + org.slf4j.Logger targetLogger = org.slf4j.LoggerFactory.getLogger(name); + Class logbackCls = Class.forName("ch.qos.logback.classic.Logger"); + Class levelCls = Class.forName("ch.qos.logback.classic.Level"); + if (logbackCls.isInstance(targetLogger)) { + Object warnLevel = levelCls.getField("WARN").get(null); + logbackCls.getMethod("setLevel", levelCls).invoke(targetLogger, warnLevel); + } + } + } catch (Throwable t) { + LOG.debug("PaimonJniWriter: log level configuration skipped", t); + } + } + + @SuppressWarnings("unused") + private String currentState() { + return "location=" + tableLocation + + ", writerNull=" + (writer == null); + } + + // ──────────────────────────────────────────────────────────── + // Bundle helpers + // ──────────────────────────────────────────────────────────── + + /** Compound key for grouping rows by (partition, bucket). */ + private static final class PartitionBucketKey { + final BinaryRow partition; + final int bucket; + + PartitionBucketKey(BinaryRow partition, int bucket) { + this.partition = partition; + this.bucket = bucket; + } + + @Override + public int hashCode() { + return partition.hashCode() * 31 + bucket; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof PartitionBucketKey)) { + return false; + } + PartitionBucketKey other = (PartitionBucketKey) obj; + return bucket == other.bucket && partition.equals(other.partition); + } + } + + /** BundleRecords backed by a List of pre-built InternalRows. */ + private static final class ListBundleRecords implements BundleRecords { + private final List rows; + + ListBundleRecords(List rows) { + this.rows = rows; + } + + @Override + public long rowCount() { + return rows.size(); + } + + @Override + public Iterator iterator() { + return rows.iterator(); + } + } + + /** InputStream over a direct ByteBuffer (no copy). */ + private static class DirectBufInputStream extends InputStream { + private final ByteBuffer buf; + + DirectBufInputStream(ByteBuffer buf) { + this.buf = buf; + } + + @Override + public int read() { + if (buf.hasRemaining()) { + return buf.get() & 0xFF; + } + return -1; + } + + @Override + public int read(byte[] b, int off, int len) { + if (!buf.hasRemaining()) { + return -1; + } + int n = Math.min(len, buf.remaining()); + buf.get(b, off, n); + return n; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTransaction.java new file mode 100644 index 00000000000000..51f13f93008512 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTransaction.java @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; +import org.apache.doris.thrift.TPaimonCommitMessage; +import org.apache.doris.transaction.Transaction; + +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.table.InnerTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageSerializer; +import org.apache.paimon.table.sink.StreamTableCommit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Paimon transaction. + * + * Lifecycle: + * 1. beginInsert() — record target table + * 2. updateCommitMessages() — called multiple times as BE reports commit data + * 3. commit() — deserialize all CommitMessages, call StreamTableCommit.filterAndCommit() + * 4. rollback() — abort uncommitted data files + * + * CommitMessage wire format: + * BE ← JNI ← Java: byte[] with DPCM header (magic + version + length) + Paimon + * CommitMessageSerializer payload + */ +public class PaimonTransaction implements Transaction { + private static final Logger LOG = LogManager.getLogger(PaimonTransaction.class); + + private static final int COMMIT_HEADER_SIZE = 12; + private static final byte[] COMMIT_MAGIC = new byte[] {'D', 'P', 'C', 'M'}; + + private final PaimonMetadataOps ops; + private PaimonExternalTable table; + private long transactionId = -1L; + private String commitUser = ""; + private boolean committed = false; + + private final List commitPayloads = Lists.newArrayList(); + private final Set commitPayloadSet = new HashSet<>(); + + public PaimonTransaction(PaimonMetadataOps ops) { + this.ops = ops; + } + + // ──────────────────────────────────────────────────────────── + // Transaction lifecycle + // ──────────────────────────────────────────────────────────── + + public void beginInsert(PaimonExternalTable table, Optional insertCtx) { + this.table = table; + } + + public void finishInsert(PaimonExternalTable table, Optional insertCtx) { + } + + @Override + public void commit() throws UserException { + List rawPayloads = snapshotPayloads(); + if (rawPayloads.isEmpty()) { + LOG.info("Skip empty PaimonTransaction commit, txnId={}, table={}", + transactionId, tableName()); + return; + } + if (table == null) { + throw new UserException("Missing paimon table for transaction"); + } + if (transactionId <= 0) { + throw new UserException("Missing transaction id for paimon commit"); + } + + try { + List allMessages = deserializePayloads(rawPayloads); + LOG.info("Commit PaimonTransaction, txnId={}, table={}, payloads={}, messages={}", + transactionId, tableName(), rawPayloads.size(), allMessages.size()); + if (allMessages.isEmpty()) { + throw new RuntimeException( + "Paimon commit messages are empty, payloads=" + rawPayloads.size()); + } + doCommit(allMessages); + synchronized (this) { + committed = true; + } + } catch (Exception e) { + throw new UserException("Failed to commit paimon transaction on FE", e); + } + } + + @Override + public void rollback() { + if (isCommitted()) { + LOG.info("Skip rollback for committed PaimonTransaction, txnId={}, table={}", + transactionId, tableName()); + return; + } + List rawPayloads = snapshotPayloads(); + if (rawPayloads.isEmpty()) { + LOG.info("Skip empty PaimonTransaction rollback, txnId={}, table={}", + transactionId, tableName()); + return; + } + if (table == null) { + LOG.warn("Skip PaimonTransaction rollback, table missing, txnId={}", transactionId); + return; + } + try { + List allMessages = deserializePayloads(rawPayloads); + if (allMessages.isEmpty()) { + LOG.info("Skip PaimonTransaction rollback with empty decoded messages, " + + "txnId={}, table={}", transactionId, tableName()); + return; + } + LOG.info("Rollback PaimonTransaction, txnId={}, table={}, payloads={}, messages={}", + transactionId, tableName(), rawPayloads.size(), allMessages.size()); + doAbort(allMessages); + } catch (Exception e) { + LOG.warn("Failed to rollback PaimonTransaction, txnId={}, table={}", + transactionId, tableName(), e); + } + } + + // ──────────────────────────────────────────────────────────── + // CommitMessage collection (called from Coordinator) + // ──────────────────────────────────────────────────────────── + + public void updateCommitMessages(List messages) { + if (messages == null || messages.isEmpty()) { + return; + } + synchronized (this) { + for (TPaimonCommitMessage msg : messages) { + addPayload(msg); + } + } + } + + private void addPayload(TPaimonCommitMessage message) { + if (message == null || !message.isSetPayload()) { + return; + } + byte[] payload = message.getPayload(); + if (payload == null || payload.length == 0) { + return; + } + String key = Base64.getEncoder().encodeToString(payload); + if (commitPayloadSet.add(key)) { + commitPayloads.add(Arrays.copyOf(payload, payload.length)); + } + } + + // ──────────────────────────────────────────────────────────── + // Setters + // ──────────────────────────────────────────────────────────── + + public void setTransactionId(long transactionId) { + this.transactionId = transactionId; + this.commitUser = "doris_txn_" + transactionId; + } + + public String getCommitUser() { + return commitUser; + } + + public long getTransactionId() { + return transactionId; + } + + boolean isCommitted() { + synchronized (this) { + return committed; + } + } + + int getPayloadCount() { + synchronized (this) { + return commitPayloads.size(); + } + } + + // ──────────────────────────────────────────────────────────── + // Internal commit logic + // ──────────────────────────────────────────────────────────── + + private void doCommit(List msgs) throws Exception { + ops.getCatalog(); + ExecutionAuthenticator authenticator = ops.dorisCatalog.getExecutionAuthenticator(); + authenticator.execute(() -> { + InnerTable paimonTable = getInnerTable(); + StreamTableCommit committer = paimonTable.newCommit(commitUser); + try { + Map> commitMap = new HashMap<>(); + commitMap.put(transactionId, msgs); + committer.filterAndCommit(commitMap); + return null; + } finally { + committer.close(); + } + }); + } + + private void doAbort(List msgs) throws Exception { + ExecutionAuthenticator authenticator = ops.dorisCatalog.getExecutionAuthenticator(); + authenticator.execute(() -> { + InnerTable paimonTable = getInnerTable(); + StreamTableCommit committer = paimonTable.newCommit(commitUser); + try { + committer.abort(msgs); + return null; + } finally { + committer.close(); + } + }); + } + + private InnerTable getInnerTable() { + org.apache.paimon.table.Table paimonTable = + table.getPaimonTable(MvccUtil.getSnapshotFromContext(table)); + if (!(paimonTable instanceof InnerTable)) { + throw new RuntimeException( + "Paimon table does not support commit: " + paimonTable.getClass()); + } + return (InnerTable) paimonTable; + } + + // ──────────────────────────────────────────────────────────── + // Serialization helpers + // ──────────────────────────────────────────────────────────── + + private List snapshotPayloads() { + synchronized (this) { + List copy = new ArrayList<>(commitPayloads.size()); + for (byte[] p : commitPayloads) { + copy.add(Arrays.copyOf(p, p.length)); + } + return copy; + } + } + + static List deserializePayloads(List payloads) throws IOException { + List all = new ArrayList<>(); + for (byte[] payload : payloads) { + all.addAll(deserializePayload(payload)); + } + return all; + } + + static List deserializePayload(byte[] payload) throws IOException { + if (payload == null || payload.length < COMMIT_HEADER_SIZE || !hasMagic(payload)) { + throw new IOException("Invalid paimon commit message payload header"); + } + int version = readInt(payload, 4); + int len = readInt(payload, 8); + if (len < 0 || payload.length != COMMIT_HEADER_SIZE + len) { + throw new IOException("Invalid paimon commit message payload length"); + } + byte[] raw = new byte[len]; + System.arraycopy(payload, COMMIT_HEADER_SIZE, raw, 0, len); + List messages = + new CommitMessageSerializer().deserializeList(version, new DataInputDeserializer(raw)); + if (messages == null) { + throw new IOException("Paimon commit message payload deserialized to null"); + } + return messages; + } + + private static boolean hasMagic(byte[] payload) { + for (int i = 0; i < COMMIT_MAGIC.length; i++) { + if (payload[i] != COMMIT_MAGIC[i]) { + return false; + } + } + return true; + } + + private static int readInt(byte[] payload, int offset) { + return ((payload[offset] & 0xFF) << 24) + | ((payload[offset + 1] & 0xFF) << 16) + | ((payload[offset + 2] & 0xFF) << 8) + | (payload[offset + 3] & 0xFF); + } + + private String tableName() { + return table == null ? "null" : table.getName(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPaimonTableSink.java new file mode 100644 index 00000000000000..9a3a812ef10d30 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPaimonTableSink.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +/** + * Unbound Paimon table sink plan node. + */ +public class UnboundPaimonTableSink + extends UnboundBaseExternalTableSink { + + public UnboundPaimonTableSink(List nameParts, List colNames, + List hints, List partitions, + CHILD_TYPE child) { + this(nameParts, colNames, hints, partitions, DMLCommandType.NONE, + Optional.empty(), Optional.empty(), child); + } + + public UnboundPaimonTableSink(List nameParts, + List colNames, + List hints, + List partitions, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + super(nameParts, PlanType.LOGICAL_UNBOUND_PAIMON_TABLE_SINK, ImmutableList.of(), + groupExpression, logicalProperties, colNames, dmlCommandType, child, + hints, partitions); + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "UnboundPaimonTableSink only accepts one child"); + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.empty(), children.get(0)); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundPaimonTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, logicalProperties, children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 01d9139577578b..26e8cb29eaef87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -69,6 +69,7 @@ import org.apache.doris.datasource.lakesoul.source.LakeSoulScanNode; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; import org.apache.doris.datasource.maxcompute.source.MaxComputeScanNode; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.source.PaimonScanNode; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalTable; import org.apache.doris.datasource.trinoconnector.source.TrinoConnectorScanNode; @@ -153,6 +154,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; @@ -213,6 +215,7 @@ import org.apache.doris.planner.NestedLoopJoinNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.OlapTableSink; +import org.apache.doris.planner.PaimonTableSink; import org.apache.doris.planner.PartitionSortNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNode; @@ -591,6 +594,21 @@ public PlanFragment visitPhysicalIcebergTableSink(PhysicalIcebergTableSink paimonTableSink, + PlanTranslatorContext context) { + PlanFragment rootFragment = paimonTableSink.child().accept(this, context); + rootFragment.setOutputPartition(DataPartition.UNPARTITIONED); + List outputExprs = Lists.newArrayList(); + paimonTableSink.getOutput().stream().map(Slot::getExprId) + .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId))); + PaimonTableSink sink = new PaimonTableSink( + (PaimonExternalTable) paimonTableSink.getTargetTable()); + rootFragment.setSink(sink); + sink.setOutputExprs(outputExprs); + return rootFragment; + } + @Override public PlanFragment visitPhysicalMaxComputeTableSink(PhysicalMaxComputeTableSink mcTableSink, PlanTranslatorContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 1e4f38fd005c80..dd24714420d7a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -40,6 +40,7 @@ public enum RuleType { BINDING_INSERT_BLACKHOLE_SINK(RuleTypeClass.REWRITE), BINDING_INSERT_HIVE_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_ICEBERG_TABLE(RuleTypeClass.REWRITE), + BINDING_INSERT_PAIMON_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_MAX_COMPUTE_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_JDBC_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_CONNECTOR_TABLE(RuleTypeClass.REWRITE), @@ -561,6 +562,7 @@ public enum RuleType { LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_HIVE_TABLE_SINK_TO_PHYSICAL_HIVE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_ICEBERG_TABLE_SINK_TO_PHYSICAL_ICEBERG_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), + LOGICAL_PAIMON_TABLE_SINK_TO_PHYSICAL_PAIMON_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_MAX_COMPUTE_TABLE_SINK_TO_PHYSICAL_MAX_COMPUTE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_ICEBERG_DELETE_SINK_TO_PHYSICAL_ICEBERG_DELETE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_ICEBERG_MERGE_SINK_TO_PHYSICAL_ICEBERG_MERGE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 8c580171b21151..fec75e163b575c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -44,6 +44,8 @@ import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalDatabase; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -54,6 +56,7 @@ import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundMaxComputeTableSink; +import org.apache.doris.nereids.analyzer.UnboundPaimonTableSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTVFTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; @@ -90,6 +93,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; @@ -166,6 +170,8 @@ public List buildRules() { RuleType.BINDING_INSERT_HIVE_TABLE.build(unboundHiveTableSink().thenApply(this::bindHiveTableSink)), RuleType.BINDING_INSERT_ICEBERG_TABLE.build( unboundIcebergTableSink().thenApply(this::bindIcebergTableSink)), + RuleType.BINDING_INSERT_PAIMON_TABLE.build( + unboundPaimonTableSink().thenApply(this::bindPaimonTableSink)), RuleType.BINDING_INSERT_MAX_COMPUTE_TABLE.build( unboundMaxComputeTableSink().thenApply(this::bindMaxComputeTableSink)), RuleType.BINDING_INSERT_CONNECTOR_TABLE.build( @@ -878,6 +884,37 @@ private void validateStaticPartition(UnboundIcebergTableSink sink, IcebergExt } } + private Plan bindPaimonTableSink(MatchingContext> ctx) { + UnboundPaimonTableSink sink = ctx.root; + Pair pair = bind(ctx.cascadesContext, sink); + PaimonExternalDatabase database = pair.first; + PaimonExternalTable table = pair.second; + LogicalPlan child = ((LogicalPlan) sink.child()); + + List bindColumns; + if (sink.getColNames().isEmpty()) { + bindColumns = table.getBaseSchema(true).stream() + .filter(Column::isVisible) + .collect(ImmutableList.toImmutableList()); + } else { + bindColumns = sink.getColNames().stream().map(cn -> { + Column column = table.getColumn(cn); + if (column == null) { + throw new AnalysisException(String.format( + "column %s is not found in table %s", cn, table.getName())); + } + return column; + }).collect(ImmutableList.toImmutableList()); + } + + List output = child.getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + + return new LogicalPaimonTableSink<>(database, table, bindColumns, output, + sink.getDMLCommandType(), Optional.empty(), Optional.empty(), child); + } + private Plan bindMaxComputeTableSink(MatchingContext> ctx) { UnboundMaxComputeTableSink sink = ctx.root; Pair pair = bind(ctx.cascadesContext, sink); @@ -1093,6 +1130,18 @@ private Pair bind(CascadesContext throw new AnalysisException("the target table of insert into is not an iceberg table"); } + private Pair bind(CascadesContext cascadesContext, + UnboundPaimonTableSink sink) { + List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), + sink.getNameParts()); + Pair, TableIf> pair = RelationUtil.getDbAndTable(tableQualifier, + cascadesContext.getConnectContext().getEnv(), Optional.empty()); + if (pair.second instanceof PaimonExternalTable) { + return Pair.of(((PaimonExternalDatabase) pair.first), (PaimonExternalTable) pair.second); + } + throw new AnalysisException("the target table of insert into is not a paimon table"); + } + private Pair bind(CascadesContext cascadesContext, UnboundMaxComputeTableSink sink) { List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPaimonTableSinkToPhysicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPaimonTableSinkToPhysicalPaimonTableSink.java new file mode 100644 index 00000000000000..02438c24628986 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPaimonTableSinkToPhysicalPaimonTableSink.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPaimonTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; + +import java.util.Optional; + +/** + * Implementation rule that converts LogicalPaimonTableSink to PhysicalPaimonTableSink. + */ +public class LogicalPaimonTableSinkToPhysicalPaimonTableSink extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalPaimonTableSink().thenApply(ctx -> { + LogicalPaimonTableSink sink = ctx.root; + return new PhysicalPaimonTableSink<>( + sink.getDatabase(), + sink.getTargetTable(), + sink.getCols(), + sink.getOutputExprs(), + Optional.empty(), + sink.getLogicalProperties(), + null, + null, + sink.child()); + }).toRule(RuleType.LOGICAL_PAIMON_TABLE_SINK_TO_PHYSICAL_PAIMON_TABLE_SINK_RULE); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 2f5dcbb7cfce9f..c0dfbb27313da5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -50,6 +50,7 @@ public enum PlanType { LOGICAL_OLAP_TABLE_SINK, LOGICAL_HIVE_TABLE_SINK, LOGICAL_ICEBERG_TABLE_SINK, + LOGICAL_PAIMON_TABLE_SINK, LOGICAL_MAX_COMPUTE_TABLE_SINK, LOGICAL_ICEBERG_DELETE_SINK, LOGICAL_ICEBERG_MERGE_SINK, @@ -61,6 +62,7 @@ public enum PlanType { LOGICAL_UNBOUND_OLAP_TABLE_SINK, LOGICAL_UNBOUND_HIVE_TABLE_SINK, LOGICAL_UNBOUND_ICEBERG_TABLE_SINK, + LOGICAL_UNBOUND_PAIMON_TABLE_SINK, LOGICAL_UNBOUND_MAX_COMPUTE_TABLE_SINK, LOGICAL_UNBOUND_JDBC_TABLE_SINK, LOGICAL_UNBOUND_CONNECTOR_TABLE_SINK, @@ -125,6 +127,7 @@ public enum PlanType { PHYSICAL_OLAP_TABLE_SINK, PHYSICAL_HIVE_TABLE_SINK, PHYSICAL_ICEBERG_TABLE_SINK, + PHYSICAL_PAIMON_TABLE_SINK, PHYSICAL_MAX_COMPUTE_TABLE_SINK, PHYSICAL_ICEBERG_DELETE_SINK, PHYSICAL_ICEBERG_MERGE_SINK, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 1372cce62e2ab9..9d108ca5e405c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -37,6 +37,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.load.loadv2.LoadJob; import org.apache.doris.load.loadv2.LoadStatistic; @@ -75,6 +76,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.RelationUtil; @@ -565,6 +567,19 @@ ExecutorFactory selectInsertExecutorFactory( emptyInsert, jobId ) ); + } else if (physicalSink instanceof PhysicalPaimonTableSink) { + boolean emptyInsert = childIsEmptyRelation(physicalSink); + PaimonExternalTable paimonTable = (PaimonExternalTable) targetTableIf; + PaimonInsertCommandContext paimonCtx = insertCtx + .map(ctx1 -> (PaimonInsertCommandContext) ctx1) + .orElseGet(PaimonInsertCommandContext::new); + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new PaimonInsertExecutor(ctx, paimonTable, label, planner, + Optional.of(paimonCtx), emptyInsert, jobId) + ); } else if (physicalSink instanceof PhysicalConnectorTableSink) { boolean emptyInsert = childIsEmptyRelation(physicalSink); ExternalTable externalTable = (ExternalTable) targetTableIf; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertCommandContext.java new file mode 100644 index 00000000000000..51abd7d19d678a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertCommandContext.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import java.util.HashMap; +import java.util.Map; + +/** + * Insert command context for Paimon tables. + */ +public class PaimonInsertCommandContext extends BaseExternalTableInsertCommandContext { + private long txnId = 0; + private String commitUser; + private Map staticPartition = new HashMap<>(); + + public long getTxnId() { + return txnId; + } + + public void setTxnId(long txnId) { + this.txnId = txnId; + } + + public String getCommitUser() { + return commitUser; + } + + public void setCommitUser(String commitUser) { + this.commitUser = commitUser; + } + + public Map getStaticPartition() { + return staticPartition; + } + + public void setStaticPartition(Map staticPartition) { + this.staticPartition = staticPartition != null + ? new HashMap<>(staticPartition) : new HashMap<>(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertExecutor.java new file mode 100644 index 00000000000000..eb4ce2f3279e7e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertExecutor.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonTransaction; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.transaction.TransactionType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; + +/** + * Insert executor for Paimon tables. + */ +public class PaimonInsertExecutor extends BaseExternalTableInsertExecutor { + private static final Logger LOG = LogManager.getLogger(PaimonInsertExecutor.class); + + public PaimonInsertExecutor(ConnectContext ctx, PaimonExternalTable table, + String labelName, NereidsPlanner planner, + Optional insertCtx, + boolean emptyInsert, long jobId) { + super(ctx, table, labelName, planner, insertCtx, emptyInsert, jobId); + } + + @Override + protected void beforeExec() throws UserException { + PaimonTransaction transaction = + (PaimonTransaction) transactionManager.getTransaction(txnId); + transaction.beginInsert((PaimonExternalTable) table, insertCtx); + } + + @Override + protected void doBeforeCommit() throws UserException { + PaimonTransaction transaction = + (PaimonTransaction) transactionManager.getTransaction(txnId); + transaction.finishInsert((PaimonExternalTable) table, insertCtx); + } + + @Override + protected TransactionType transactionType() { + return TransactionType.PAIMON; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPaimonTableSink.java new file mode 100644 index 00000000000000..a7335ba5406ac8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPaimonTableSink.java @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.paimon.PaimonExternalDatabase; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Logical Paimon table sink for INSERT INTO paimon_table. + */ +public class LogicalPaimonTableSink extends LogicalTableSink + implements Sink { + + private final PaimonExternalDatabase database; + private final PaimonExternalTable targetTable; + private final DMLCommandType dmlCommandType; + + public LogicalPaimonTableSink(PaimonExternalDatabase database, + PaimonExternalTable targetTable, + List cols, + List outputExprs, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + super(PlanType.LOGICAL_PAIMON_TABLE_SINK, outputExprs, groupExpression, logicalProperties, + cols, child); + this.database = Objects.requireNonNull(database, "database != null"); + this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null"); + this.dmlCommandType = dmlCommandType; + } + + /** Update output expressions based on child output and replace child. */ + public Plan withChildAndUpdateOutput(Plan child) { + List output = child.getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + return AbstractPlan.copyWithSameId(this, () -> + new LogicalPaimonTableSink<>(database, targetTable, cols, output, + dmlCommandType, Optional.empty(), Optional.empty(), child)); + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "LogicalPaimonTableSink only accepts one child"); + return AbstractPlan.copyWithSameId(this, () -> + new LogicalPaimonTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, Optional.empty(), Optional.empty(), children.get(0))); + } + + public LogicalPaimonTableSink withOutputExprs(List outputExprs) { + return AbstractPlan.copyWithSameId(this, () -> + new LogicalPaimonTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, Optional.empty(), Optional.empty(), child())); + } + + public PaimonExternalDatabase getDatabase() { + return database; + } + + public PaimonExternalTable getTargetTable() { + return targetTable; + } + + public DMLCommandType getDmlCommandType() { + return dmlCommandType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + LogicalPaimonTableSink that = (LogicalPaimonTableSink) o; + return dmlCommandType == that.dmlCommandType + && Objects.equals(database, that.database) + && Objects.equals(targetTable, that.targetTable) + && Objects.equals(cols, that.cols); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), database, targetTable, cols, dmlCommandType); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalPaimonTableSink[" + id.asInt() + "]", + "outputExprs", outputExprs, + "database", database.getFullName(), + "targetTable", targetTable.getName(), + "cols", cols, + "dmlCommandType", dmlCommandType); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalPaimonTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return AbstractPlan.copyWithSameId(this, () -> + new LogicalPaimonTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, groupExpression, + Optional.of(getLogicalProperties()), child())); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return AbstractPlan.copyWithSameId(this, () -> + new LogicalPaimonTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, groupExpression, logicalProperties, children.get(0))); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java new file mode 100644 index 00000000000000..f8d53ba3e23ecc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.paimon.PaimonExternalDatabase; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.AbstractPlan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.statistics.Statistics; + +import java.util.List; +import java.util.Optional; + +/** + * Physical Paimon table sink. + */ +public class PhysicalPaimonTableSink + extends PhysicalBaseExternalTableSink { + + public PhysicalPaimonTableSink(PaimonExternalDatabase database, + PaimonExternalTable targetTable, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + CHILD_TYPE child) { + this(database, targetTable, cols, outputExprs, groupExpression, logicalProperties, + PhysicalProperties.GATHER, null, child); + } + + public PhysicalPaimonTableSink(PaimonExternalDatabase database, + PaimonExternalTable targetTable, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + CHILD_TYPE child) { + super(PlanType.PHYSICAL_PAIMON_TABLE_SINK, database, targetTable, cols, outputExprs, + groupExpression, logicalProperties, physicalProperties, statistics, child); + } + + @Override + public Plan withChildren(List children) { + return AbstractPlan.copyWithSameId(this, () -> new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, (PaimonExternalTable) targetTable, + cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, statistics, children.get(0))); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return AbstractPlan.copyWithSameId(this, () -> new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, (PaimonExternalTable) targetTable, + cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, statistics, child())); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return AbstractPlan.copyWithSameId(this, () -> new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, (PaimonExternalTable) targetTable, + cols, outputExprs, groupExpression, + logicalProperties.get(), physicalProperties, statistics, children.get(0))); + } + + @Override + public PhysicalPaimonTableSink withPhysicalPropertiesAndStats( + PhysicalProperties physicalProperties, Statistics stats) { + return AbstractPlan.copyWithSameId(this, () -> new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, (PaimonExternalTable) targetTable, + cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, stats, child())); + } + + @Override + public PhysicalProperties getRequirePhysicalProperties() { + // Partition and bucket routing is handled by the Paimon SDK internally. + // Doris does not need to shuffle data by partition/bucket keys. + return PhysicalProperties.SINK_RANDOM_PARTITIONED; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalPaimonTableSink(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java index dcc6f715c9e3c8..a09bec81b9c997 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundMaxComputeTableSink; +import org.apache.doris.nereids.analyzer.UnboundPaimonTableSink; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTVFTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; @@ -37,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; @@ -51,6 +53,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFTableSink; @@ -85,6 +88,10 @@ default R visitUnboundIcebergTableSink(UnboundIcebergTableSink u return visitLogicalSink(unboundTableSink, context); } + default R visitUnboundPaimonTableSink(UnboundPaimonTableSink unboundTableSink, C context) { + return visitLogicalSink(unboundTableSink, context); + } + default R visitUnboundConnectorTableSink(UnboundConnectorTableSink unboundTableSink, C context) { return visitLogicalSink(unboundTableSink, context); } @@ -133,6 +140,10 @@ default R visitLogicalIcebergTableSink(LogicalIcebergTableSink i return visitLogicalTableSink(icebergTableSink, context); } + default R visitLogicalPaimonTableSink(LogicalPaimonTableSink paimonTableSink, C context) { + return visitLogicalTableSink(paimonTableSink, context); + } + default R visitLogicalMaxComputeTableSink(LogicalMaxComputeTableSink mcTableSink, C context) { return visitLogicalTableSink(mcTableSink, context); } @@ -197,6 +208,10 @@ default R visitPhysicalIcebergTableSink(PhysicalIcebergTableSink return visitPhysicalTableSink(icebergTableSink, context); } + default R visitPhysicalPaimonTableSink(PhysicalPaimonTableSink paimonTableSink, C context) { + return visitPhysicalTableSink(paimonTableSink, context); + } + default R visitPhysicalMaxComputeTableSink(PhysicalMaxComputeTableSink mcTableSink, C context) { return visitPhysicalTableSink(mcTableSink, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java new file mode 100644 index 00000000000000..0fb6d392a09fbf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.Column; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; +import org.apache.doris.nereids.trees.plans.commands.insert.PaimonInsertCommandContext; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TPaimonTableSink; +import org.apache.doris.thrift.TPaimonWriteBackendType; +import org.apache.doris.thrift.TPaimonWriteMode; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.utils.InstantiationUtil; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Paimon table sink. + * + * Generates TPaimonTableSink payload consumed by BE, including table location, + * Paimon options, Hadoop config, write mode, and sink column names. + * + * v1: single-writer architecture; partition/bucket routing delegated to SDK. + */ +public class PaimonTableSink extends BaseExternalTableDataSink { + private static final Logger LOG = LogManager.getLogger(PaimonTableSink.class); + private static final String OUTPUT_COLUMN_NAMES_KEY = "doris.output_column_names"; + private static final String COLUMN_NAME_SEPARATOR = ""; + private static final Base64.Encoder BASE64_ENCODER = + java.util.Base64.getUrlEncoder().withoutPadding(); + + private final PaimonExternalTable targetTable; + private List outputExprs; + private List cols; + + private static final HashSet supportedTypes = new HashSet() {{ + add(TFileFormatType.FORMAT_ORC); + add(TFileFormatType.FORMAT_PARQUET); + }}; + + public PaimonTableSink(PaimonExternalTable targetTable) { + super(); + this.targetTable = targetTable; + } + + public void setCols(List cols) { + this.cols = cols; + } + + public void setOutputExprs(List outputExprs) { + this.outputExprs = outputExprs; + } + + @Override + protected Set supportedFileFormatTypes() { + return supportedTypes; + } + + @Override + public String getExplainString(String prefix, TExplainLevel explainLevel) { + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append(prefix).append("PAIMON TABLE SINK\n"); + if (explainLevel == TExplainLevel.BRIEF) { + return strBuilder.toString(); + } + strBuilder.append(prefix).append(" table: ").append(targetTable.getName()).append("\n"); + return strBuilder.toString(); + } + + @Override + public void bindDataSink(Optional insertCtx) throws AnalysisException { + TPaimonTableSink tSink = new TPaimonTableSink(); + + tSink.setDbName(targetTable.getDbName()); + tSink.setTbName(targetTable.getName()); + + Map hadoopConfig = new HashMap<>( + targetTable.getCatalog().getCatalogProperty().getHadoopProperties()); + Map paimonOptions = new HashMap<>(); + + String warehouse = ((PaimonExternalCatalog) targetTable.getCatalog()) + .getPaimonOptionsMap().get(CatalogOptions.WAREHOUSE.key()); + String defaultFs = resolveDefaultFsName(warehouse); + if (defaultFs != null && !defaultFs.isEmpty()) { + String current = hadoopConfig.get("fs.defaultFS"); + if (current == null || current.isEmpty() || current.startsWith("file:/")) { + hadoopConfig.put("fs.defaultFS", defaultFs); + } + } + + // Transaction context: commit identifier and user + if (insertCtx.isPresent() && insertCtx.get() instanceof PaimonInsertCommandContext) { + PaimonInsertCommandContext ctx = + (PaimonInsertCommandContext) insertCtx.get(); + if (ctx.getTxnId() > 0) { + paimonOptions.put("doris.commit_identifier", String.valueOf(ctx.getTxnId())); + } + if (ctx.getCommitUser() != null && !ctx.getCommitUser().isEmpty()) { + paimonOptions.put("doris.commit_user", ctx.getCommitUser()); + } + } + + // Column names (BE-side column name preservation) + paimonOptions.put(OUTPUT_COLUMN_NAMES_KEY, String.join(COLUMN_NAME_SEPARATOR, outputColumnNames())); + + // Hadoop user + String hadoopUser = hadoopConfig.get("hadoop.username"); + if (hadoopUser == null || hadoopUser.isEmpty()) { + hadoopUser = hadoopConfig.get("hadoop.user.name"); + } + if (hadoopUser == null || hadoopUser.isEmpty()) { + hadoopUser = "hadoop"; + } + hadoopConfig.put("hadoop.user.name", hadoopUser); + + // Table location + String tableLocation = null; + org.apache.paimon.table.Table paimonTable = + targetTable.getPaimonTable(MvccUtil.getSnapshotFromContext(targetTable)); + if (paimonTable instanceof org.apache.paimon.table.FileStoreTable) { + tableLocation = ((org.apache.paimon.table.FileStoreTable) paimonTable).location().toString(); + } + if (tableLocation != null && !tableLocation.isEmpty()) { + tSink.setTableLocation(tableLocation); + } + + // Serialize table for BE-side fast loading + if (paimonTable != null) { + tSink.setSerializedTable(encodeObjectToString(paimonTable)); + } + + // Serialize table for BE-side fast loading + tSink.setBackendType(TPaimonWriteBackendType.JNI); + if (insertCtx.isPresent() && insertCtx.get() instanceof PaimonInsertCommandContext) { + PaimonInsertCommandContext ctx = + (PaimonInsertCommandContext) insertCtx.get(); + if (ctx.isOverwrite()) { + tSink.setWriteMode(TPaimonWriteMode.OVERWRITE); + paimonOptions.put("doris.write_mode", "OVERWRITE"); + if (ctx.getStaticPartition() != null && !ctx.getStaticPartition().isEmpty()) { + tSink.setStaticPartition(new HashMap<>(ctx.getStaticPartition())); + for (Map.Entry e : ctx.getStaticPartition().entrySet()) { + paimonOptions.put("doris.static_partition." + e.getKey(), e.getValue()); + } + } + } else { + tSink.setWriteMode(TPaimonWriteMode.APPEND); + } + } else { + tSink.setWriteMode(TPaimonWriteMode.APPEND); + } + + tSink.setPaimonOptions(paimonOptions); + tSink.setHadoopConfig(hadoopConfig); + + List columnNames = new ArrayList<>(); + for (Column col : cols) { + columnNames.add(col.getName()); + } + tSink.setColumnNames(columnNames); + + tDataSink = new TDataSink(TDataSinkType.PAIMON_TABLE_SINK); + tDataSink.setPaimonTableSink(tSink); + } + + private List outputColumnNames() throws AnalysisException { + List fullSchema = targetTable.getFullSchema(); + if (fullSchema.size() != outputExprs.size()) { + throw new AnalysisException("Paimon sink output column size mismatch, schema=" + + fullSchema.size() + ", exprs=" + outputExprs.size()); + } + List names = new ArrayList<>(fullSchema.size()); + for (Column col : fullSchema) { + names.add(col.getName()); + } + return names; + } + + private static String encodeObjectToString(Object obj) { + try { + byte[] bytes = InstantiationUtil.serializeObject(obj); + return new String(BASE64_ENCODER.encode(bytes), + java.nio.charset.StandardCharsets.UTF_8); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize Paimon table", e); + } + } + + private static String resolveDefaultFsName(String warehouse) { + if (warehouse == null || warehouse.isEmpty()) { + return null; + } + try { + java.net.URI uri = java.net.URI.create(warehouse); + String scheme = uri.getScheme(); + String authority = uri.getAuthority(); + if (scheme != null && !scheme.isEmpty() && authority != null && !authority.isEmpty()) { + return scheme + "://" + authority; + } + } catch (Exception e) { + LOG.warn("paimon: invalid warehouse uri {}", warehouse); + } + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 74e0128e8dad38..3b4a34e8f68416 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -41,6 +41,7 @@ import org.apache.doris.datasource.hive.HMSTransaction; import org.apache.doris.datasource.iceberg.IcebergTransaction; import org.apache.doris.datasource.maxcompute.MCTransaction; +import org.apache.doris.datasource.paimon.PaimonTransaction; import org.apache.doris.load.loadv2.LoadJob; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.MysqlCommand; @@ -2644,6 +2645,11 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { ((MCTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) .updateMCCommitData(params.getMcCommitDatas()); } + if (params.isSetPaimonCommitMessages()) { + ((PaimonTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(txnId)) + .updateCommitMessages(params.getPaimonCommitMessages()); + } if (ctx.done) { if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java index 38ba2ebbc79501..f2d6a5f185354f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.hive.HMSTransaction; import org.apache.doris.datasource.iceberg.IcebergTransaction; import org.apache.doris.datasource.maxcompute.MCTransaction; +import org.apache.doris.datasource.paimon.PaimonTransaction; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.AbstractJobProcessor; import org.apache.doris.qe.CoordinatorContext; @@ -240,6 +241,11 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF ((MCTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) .updateMCCommitData(params.getMcCommitDatas()); } + if (params.isSetPaimonCommitMessages()) { + ((PaimonTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(txnId)) + .updateCommitMessages(params.getPaimonCommitMessages()); + } if (fragmentTask.isDone()) { if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/PaimonTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/PaimonTransactionManager.java new file mode 100644 index 00000000000000..e01dc674cc1bd5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/PaimonTransactionManager.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.transaction; + +import org.apache.doris.datasource.paimon.PaimonMetadataOps; +import org.apache.doris.datasource.paimon.PaimonTransaction; + +/** + * Transaction manager for Paimon external tables. + */ +public class PaimonTransactionManager extends AbstractExternalTransactionManager { + + public PaimonTransactionManager(PaimonMetadataOps ops) { + super(ops); + } + + @Override + PaimonTransaction createTransaction() { + return new PaimonTransaction((PaimonMetadataOps) ops); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java index 9a5584a0601874..1801b65b7f9837 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java @@ -20,6 +20,7 @@ import org.apache.doris.datasource.hive.HiveMetadataOps; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonMetadataOps; import org.apache.doris.fs.SpiSwitchingFileSystem; import java.util.concurrent.Executor; @@ -38,4 +39,8 @@ public static TransactionManager createIcebergTransactionManager(IcebergMetadata public static TransactionManager createMCTransactionManager(MaxComputeExternalCatalog catalog) { return new MCTransactionManager(catalog); } + + public static TransactionManager createPaimonTransactionManager(PaimonMetadataOps ops) { + return new PaimonTransactionManager(ops); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java index 83e092c0ed7136..d6a587eaa2effe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java @@ -22,5 +22,6 @@ public enum TransactionType { HMS, ICEBERG, JDBC, - MAXCOMPUTE + MAXCOMPUTE, + PAIMON } diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 3a45208ea9bf11..3f6eeb438c909c 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -46,6 +46,7 @@ enum TDataSinkType { MAXCOMPUTE_TABLE_SINK = 18, ICEBERG_DELETE_SINK = 19, ICEBERG_MERGE_SINK = 20, + PAIMON_TABLE_SINK = 21, } enum TResultSinkType { @@ -618,6 +619,33 @@ struct TMaxComputeTableSink { 18: optional i64 txn_id // FE external transaction ID for runtime block_id allocation } +enum TPaimonWriteBackendType { + JNI = 0, + FFI = 1, +} + +enum TPaimonWriteMode { + APPEND = 0, + OVERWRITE = 1, +} + +struct TPaimonCommitMessage { + 1: optional binary payload // Paimon native CommitMessageSerializer bytes (DPCM-framed) +} + +struct TPaimonTableSink { + 1: optional string db_name + 2: optional string tb_name + 3: optional string table_location + 4: optional map paimon_options + 5: optional map hadoop_config + 6: optional list column_names + 7: optional string serialized_table // serialized Paimon Table object (base64) + 8: optional TPaimonWriteBackendType backend_type + 9: optional TPaimonWriteMode write_mode + 10: optional map static_partition // static partition spec for INSERT OVERWRITE +} + struct TDataSink { 1: required TDataSinkType type 2: optional TDataStreamSink stream_sink @@ -638,4 +666,5 @@ struct TDataSink { 18: optional TMaxComputeTableSink max_compute_table_sink 19: optional TIcebergDeleteSink iceberg_delete_sink 20: optional TIcebergMergeSink iceberg_merge_sink + 21: optional TPaimonTableSink paimon_table_sink } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index c1026ee37972eb..7617fee0b4ec9e 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -333,6 +333,8 @@ struct TReportExecStatusParams { 32: optional list mc_commit_datas 33: optional string first_error_msg + + 34: optional list paimon_commit_messages } struct TFeResult { diff --git a/regression-test/suites/paimon_write/test_paimon_write_append_only.groovy b/regression-test/suites/paimon_write/test_paimon_write_append_only.groovy new file mode 100644 index 00000000000000..aa36e094247ee5 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_append_only.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_append_only", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_ao_catalog" + String dbName = "test_pw_ao_db" + + // Tables are created via Spark because Doris does not yet support + // Paimon DDL (CREATE TABLE ... engine=paimon). + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_append; + CREATE TABLE paimon.${dbName}.t_append ( + id INT, name STRING, score DOUBLE + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_part; + CREATE TABLE paimon.${dbName}.t_append_part ( + id INT, name STRING, score DOUBLE, region STRING + ) USING paimon + PARTITIONED BY (region) + TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_empty; + CREATE TABLE paimon.${dbName}.t_append_empty ( + id INT, name STRING + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // FT-001: Append-only table — basic INSERT + sql """INSERT INTO t_append VALUES (1, 'alice', 95.5)""" + sql """INSERT INTO t_append VALUES (2, 'bob', 87.0), (3, 'charlie', 92.3)""" + order_qt_ao_basic """SELECT * FROM t_append ORDER BY id""" + + sql """INSERT INTO t_append VALUES (4, 'diana', 88.0), (5, 'eve', 91.0)""" + def cnt = sql """SELECT COUNT(*) FROM t_append""" + assertEquals(5, cnt[0][0] as int) + + // Cross-validate with Spark + def sparkCnt = spark_paimon """SELECT COUNT(*) FROM paimon.${dbName}.t_append""" + assertEquals(5, sparkCnt[0][0] as int) + + // FT-002: Partitioned append-only + sql """INSERT INTO t_append_part VALUES (1, 'alice', 95.5, 'east'), (2, 'bob', 87.0, 'west')""" + sql """INSERT INTO t_append_part VALUES (3, 'charlie', 92.3, 'east'), (4, 'diana', 88.0, 'north')""" + order_qt_ao_part """SELECT * FROM t_append_part ORDER BY id""" + + def eastCnt = sql """SELECT COUNT(*) FROM t_append_part WHERE region = 'east'""" + assertEquals(2, eastCnt[0][0] as int) + + // FT-014: Empty INSERT — should succeed with 0 rows + sql """INSERT INTO t_append_empty SELECT 1, 'test' WHERE 1 = 0""" + def emptyCnt = sql """SELECT COUNT(*) FROM t_append_empty""" + assertEquals(0, emptyCnt[0][0] as int) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_complex_types.groovy b/regression-test/suites/paimon_write/test_paimon_write_complex_types.groovy new file mode 100644 index 00000000000000..6704727afe3f31 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_complex_types.groovy @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_complex_types", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_cx_catalog" + String dbName = "test_pw_cx_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_array; + CREATE TABLE paimon.${dbName}.t_array ( + id INT, + c_array_int ARRAY, + c_array_string ARRAY, + c_array_double ARRAY + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_map; + CREATE TABLE paimon.${dbName}.t_map ( + id INT, + c_map_str_int MAP, + c_map_int_str MAP + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_struct; + CREATE TABLE paimon.${dbName}.t_struct ( + id INT, + c_struct STRUCT + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_nested; + CREATE TABLE paimon.${dbName}.t_nested ( + id INT, + c_map_arr MAP> + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // FT-028: ARRAY types — normal array, empty array, NULL array + sql """INSERT INTO t_array VALUES + (1, [1, 2, 3], ['a', 'b', 'c'], [1.1, 2.2]), + (2, [], [], []), + (3, [10, NULL, 30], ['x', NULL, 'z'], [NULL, 2.0]), + (4, NULL, NULL, NULL) + """ + def arrCnt = sql """SELECT COUNT(*) FROM t_array""" + assertEquals(4, arrCnt[0][0] as int) + order_qt_cx_array """SELECT id, c_array_int, c_array_string, c_array_double FROM t_array ORDER BY id""" + + // FT-029: MAP types — normal map, empty map, NULL value + sql """INSERT INTO t_map VALUES + (1, map('math', 90, 'eng', 95), map(1, 'one', 2, 'two')), + (2, map(), map()), + (3, map('science', NULL), map(3, NULL)), + (4, NULL, NULL) + """ + def mapCnt = sql """SELECT COUNT(*) FROM t_map""" + assertEquals(4, mapCnt[0][0] as int) + order_qt_cx_map """SELECT id, c_map_str_int, c_map_int_str FROM t_map ORDER BY id""" + + // FT-030: STRUCT types + sql """INSERT INTO t_struct VALUES + (1, named_struct('name', 'alice', 'age', 30)), + (2, named_struct('name', NULL, 'age', NULL)), + (3, NULL) + """ + order_qt_cx_struct """SELECT id, c_struct FROM t_struct ORDER BY id""" + + // Nested: MAP> + sql """INSERT INTO t_nested VALUES + (1, map('group1', [1, 2], 'group2', [3, 4, 5])), + (2, map('empty', [])), + (3, NULL) + """ + order_qt_cx_nested """SELECT id, c_map_arr FROM t_nested ORDER BY id""" + + // Cross-validate ARRAY with Spark + def sparkArr = spark_paimon """SELECT id, c_array_int FROM paimon.${dbName}.t_array ORDER BY id""" + def dorisArr = sql """SELECT id, c_array_int FROM t_array ORDER BY id""" + assertEquals(sparkArr[0], dorisArr[0]) + assertEquals(sparkArr[1], dorisArr[1]) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_edge_cases.groovy b/regression-test/suites/paimon_write/test_paimon_write_edge_cases.groovy new file mode 100644 index 00000000000000..0d129fa5a8ffa0 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_edge_cases.groovy @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_edge_cases", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_edge_catalog" + String dbName = "test_pw_edge_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_str; + CREATE TABLE paimon.${dbName}.t_edge_str ( + id INT, c_string STRING, c_varchar VARCHAR(10) + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_numeric; + CREATE TABLE paimon.${dbName}.t_edge_numeric ( + id INT, + c_tiny TINYINT, + c_small SMALLINT, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_bool; + CREATE TABLE paimon.${dbName}.t_edge_bool ( + id INT, c_bool BOOLEAN + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_pk_null; + CREATE TABLE paimon.${dbName}.t_edge_pk_null ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ('primary-key' = 'id', 'bucket' = '1', 'bucket-key' = 'id'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_mixed_write; + CREATE TABLE paimon.${dbName}.t_mixed_write ( + id INT, name STRING, score DOUBLE + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // FT-042: Empty string and boundary VARCHAR + sql """INSERT INTO t_edge_str VALUES + (1, '', ''), + (2, 'hello world', 'short_str'), + (3, 'x', 'abcdefghij'), + (4, 'very long string over 100 chars: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890', 'max10chars') + """ + def strCnt = sql """SELECT COUNT(*) FROM t_edge_str""" + assertEquals(4, strCnt[0][0] as int) + order_qt_edge_str """SELECT id, c_varchar FROM t_edge_str ORDER BY id""" + + // FT-021: Numeric boundary values (INT_MIN, INT_MAX, etc.) + sql """INSERT INTO t_edge_numeric VALUES + (1, CAST(127 AS TINYINT), CAST(32767 AS SMALLINT), 2147483647, + CAST(9223372036854775807 AS BIGINT), + CAST(3.4028235E38 AS FLOAT), CAST(1.7976931348623157E308 AS DOUBLE)), + (2, CAST(-128 AS TINYINT), CAST(-32768 AS SMALLINT), -2147483648, + CAST(-9223372036854775808 AS BIGINT), + CAST(-3.4028235E38 AS FLOAT), CAST(-1.7976931348623157E308 AS DOUBLE)), + (3, CAST(0 AS TINYINT), CAST(0 AS SMALLINT), 0, + CAST(0 AS BIGINT), + CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE)) + """ + order_qt_edge_numeric """SELECT id, c_tiny, c_small, c_int, c_bigint FROM t_edge_numeric ORDER BY id""" + + // BOOLEAN with NULL and both true/false + sql """INSERT INTO t_edge_bool VALUES (1, true), (2, false), (3, NULL)""" + order_qt_edge_bool """SELECT id, c_bool FROM t_edge_bool ORDER BY id""" + + // FT-041: PK table — insert NULL values, then update with non-NULL + sql """INSERT INTO t_edge_pk_null VALUES (1, 'first'), (2, NULL)""" + def before = sql """SELECT id, name FROM t_edge_pk_null ORDER BY id""" + assertEquals(2, before.size()) + + sql """INSERT INTO t_edge_pk_null VALUES (2, 'updated'), (3, 'third')""" + def after = sql """SELECT COUNT(*) FROM t_edge_pk_null""" + assertEquals(3, after[0][0] as int) + order_qt_edge_pk_null """SELECT id, name FROM t_edge_pk_null ORDER BY id""" + + // Mixed INSERT patterns: VALUES, then SELECT from self, then single-row VALUES + sql """INSERT INTO t_mixed_write VALUES (1, 'a', 10.0), (2, 'b', 20.0)""" + sql """INSERT INTO t_mixed_write VALUES (3, 'c', 30.0)""" + sql """INSERT INTO t_mixed_write SELECT id + 3, concat(name, '_copy'), score + 30.0 FROM t_mixed_write""" + def mixedCnt = sql """SELECT COUNT(*) FROM t_mixed_write""" + assertEquals(6, mixedCnt[0][0] as int) + order_qt_edge_mixed """SELECT id, name, score FROM t_mixed_write ORDER BY id""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_pk.groovy b/regression-test/suites/paimon_write/test_paimon_write_pk.groovy new file mode 100644 index 00000000000000..55377a6c6c65ac --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_pk.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_pk", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_pk_catalog" + String dbName = "test_pw_pk_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_dedup; + CREATE TABLE paimon.${dbName}.t_pk_dedup ( + id INT, name STRING, score DOUBLE, ts BIGINT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_bucket4; + CREATE TABLE paimon.${dbName}.t_pk_bucket4 ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '4', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_composite; + CREATE TABLE paimon.${dbName}.t_pk_composite ( + user_id INT, event_time BIGINT, event_type STRING, value DOUBLE + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'user_id,event_time', + 'bucket' = '2', + 'bucket-key' = 'user_id' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + // Prepare an internal OLAP source table for INSERT INTO ... SELECT + sql """drop table if exists internal.${dbName}.t_source""" + sql """ + CREATE TABLE internal.${dbName}.t_source ( + id INT, name STRING, score DOUBLE, ts BIGINT + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num'='1'); + """ + sql """INSERT INTO internal.${dbName}.t_source VALUES + (1, 'alice', 95.5, 1000), + (1, 'alice_updated', 99.0, 2000), + (2, 'bob', 87.0, 1000), + (3, 'charlie', 92.3, 1000), + (3, 'charlie_v2', 88.0, 1500), + (4, 'diana', 91.0, 1000), + (5, 'eve', 85.0, 1000) + """ + + try { + // FT-004: PK table, deduplicate — duplicate keys merged by Paimon SDK + sql """INSERT INTO t_pk_dedup SELECT id, name, score, ts FROM internal.${dbName}.t_source""" + def cnt = sql """SELECT COUNT(*) FROM t_pk_dedup""" + assertEquals(5, cnt[0][0] as int) + order_qt_pk_dedup """SELECT id, name, score FROM t_pk_dedup ORDER BY id""" + + // Cross-validate with Spark + def sparkCnt = spark_paimon """SELECT COUNT(*) FROM paimon.${dbName}.t_pk_dedup""" + assertEquals(5, sparkCnt[0][0] as int) + + // FT-003: Fixed bucket table with multiple buckets + for (int i = 0; i < 20; i++) { + sql """INSERT INTO t_pk_bucket4 VALUES (${i}, 'row${i}')""" + } + def bucketRows = sql """SELECT COUNT(*) FROM t_pk_bucket4""" + assertEquals(20, bucketRows[0][0] as int) + order_qt_pk_bucket """SELECT id, name FROM t_pk_bucket4 ORDER BY id""" + + // PK table with composite primary key + sql """INSERT INTO t_pk_composite VALUES + (1, 100, 'click', 1.0), + (1, 200, 'view', 2.0), + (2, 100, 'click', 3.0), + (1, 100, 'click_updated', 99.0) + """ + def compCnt = sql """SELECT COUNT(*) FROM t_pk_composite""" + // (1,100) duplicated → 3 unique PKs: (1,100), (1,200), (2,100) + assertEquals(3, compCnt[0][0] as int) + order_qt_pk_composite """SELECT user_id, event_time, event_type, value FROM t_pk_composite ORDER BY user_id, event_time""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_transaction.groovy b/regression-test/suites/paimon_write/test_paimon_write_transaction.groovy new file mode 100644 index 00000000000000..e7eaa8cbd6e1fb --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_transaction.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_transaction", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_txn_catalog" + String dbName = "test_pw_txn_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_commit; + CREATE TABLE paimon.${dbName}.t_commit ( + id INT, name STRING + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_commit_batch; + CREATE TABLE paimon.${dbName}.t_commit_batch ( + id INT, val DOUBLE + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite; + CREATE TABLE paimon.${dbName}.t_overwrite ( + id INT, name STRING + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_multi; + CREATE TABLE paimon.${dbName}.t_multi ( + id INT, name STRING, score DOUBLE + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // FT-010: Basic commit — INSERT INTO, commit, read back + sql """INSERT INTO t_commit VALUES (1, 'alice'), (2, 'bob')""" + def r1 = sql """SELECT COUNT(*) FROM t_commit""" + assertEquals(2, r1[0][0] as int) + + sql """INSERT INTO t_commit VALUES (3, 'charlie'), (4, 'diana')""" + def r2 = sql """SELECT COUNT(*) FROM t_commit""" + assertEquals(4, r2[0][0] as int) + order_qt_txn_commit """SELECT id, name FROM t_commit ORDER BY id""" + + // Cross-validate with Spark + def sparkCnt = spark_paimon """SELECT COUNT(*) FROM paimon.${dbName}.t_commit""" + assertEquals(4, sparkCnt[0][0] as int) + + // FT-011: Batch INSERT — 10 rows, then self-copy via SELECT + sql """INSERT INTO t_commit_batch VALUES + (1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0), (5, 5.0), + (6, 6.0), (7, 7.0), (8, 8.0), (9, 9.0), (10, 10.0)""" + sql """INSERT INTO t_commit_batch SELECT id + 10, val + 10.0 FROM t_commit_batch""" + def batchCnt = sql """SELECT COUNT(*) FROM t_commit_batch""" + assertEquals(20, batchCnt[0][0] as int) + order_qt_txn_batch """SELECT id, val FROM t_commit_batch ORDER BY id""" + + // INSERT OVERWRITE — replace all data + sql """INSERT INTO t_overwrite VALUES (1, 'old1'), (2, 'old2'), (3, 'old3')""" + assertEquals(3, sql("""SELECT COUNT(*) FROM t_overwrite""")[0][0] as int) + + sql """INSERT OVERWRITE TABLE t_overwrite VALUES (10, 'new1'), (20, 'new2')""" + assertEquals(2, sql("""SELECT COUNT(*) FROM t_overwrite""")[0][0] as int) + order_qt_txn_overwrite """SELECT id, name FROM t_overwrite ORDER BY id""" + + // Multi-row VALUES — verify all 20 rows committed + sql """ + INSERT INTO t_multi VALUES + (1, 'a', 10.0), (2, 'b', 20.0), (3, 'c', 30.0), (4, 'd', 40.0), (5, 'e', 50.0), + (6, 'f', 60.0), (7, 'g', 70.0), (8, 'h', 80.0), (9, 'i', 90.0), (10, 'j', 100.0), + (11, 'k', 110.0),(12, 'l', 120.0),(13, 'm', 130.0),(14, 'n', 140.0),(15, 'o', 150.0), + (16, 'p', 160.0),(17, 'q', 170.0),(18, 'r', 180.0),(19, 's', 190.0),(20, 't', 200.0) + """ + assertEquals(20, sql("""SELECT COUNT(*) FROM t_multi""")[0][0] as int) + order_qt_txn_multi """SELECT id, name, score FROM t_multi ORDER BY id""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_types.groovy b/regression-test/suites/paimon_write/test_paimon_write_types.groovy new file mode 100644 index 00000000000000..a51496ccc686e0 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_types.groovy @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_types", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_types_catalog" + String dbName = "test_pw_types_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types; + CREATE TABLE paimon.${dbName}.t_types ( + c_boolean BOOLEAN, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE, + c_decimal DECIMAL(10,2), + c_string STRING, + c_varchar VARCHAR(100), + c_date DATE, + c_datetime TIMESTAMP + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_null; + CREATE TABLE paimon.${dbName}.t_types_null ( + id INT, + c_int INT, + c_string STRING, + c_double DOUBLE, + c_boolean BOOLEAN + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_decimal; + CREATE TABLE paimon.${dbName}.t_types_decimal ( + id INT, + d2 DECIMAL(2,1), + d10 DECIMAL(10,2), + d18 DECIMAL(18,6), + d38 DECIMAL(38,10) + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_dt; + CREATE TABLE paimon.${dbName}.t_types_dt ( + d DATE, dt TIMESTAMP + ) USING paimon TBLPROPERTIES ('bucket' = '1'); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + // FT-020~027: Basic types with boundary values + sql """ + INSERT INTO t_types VALUES + (true, 1, 100, CAST(1.5 AS FLOAT), CAST(2.71828 AS DOUBLE), + CAST(99.99 AS DECIMAL(10,2)), 'hello', 'short', + DATE '2024-01-15', TIMESTAMP '2024-01-15 10:30:00'), + (false, 2147483647, 9223372036854775807, + CAST(3.4E38 AS FLOAT), CAST(1.7E308 AS DOUBLE), + CAST(0.00 AS DECIMAL(10,2)), '', '', + DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), + (false, -2147483648, -9223372036854775808, + CAST(-3.4E38 AS FLOAT), CAST(-1.7E308 AS DOUBLE), + CAST(-1.50 AS DECIMAL(10,2)), 'long_string_1234567890', 'max_varchar', + DATE '2099-12-31', TIMESTAMP '2099-12-31 23:59:59'), + (true, 0, 0, CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE), + CAST(12345678.90 AS DECIMAL(10,2)), 'hello', 'fixed_len', + DATE '2024-06-15', TIMESTAMP '2024-06-15 12:00:00.123456') + """ + order_qt_types_basic """SELECT * FROM t_types ORDER BY c_int""" + + // FT-040: NULL handling + sql """INSERT INTO t_types_null VALUES (1, 100, 'data', 1.5, true)""" + sql """INSERT INTO t_types_null VALUES (2, NULL, NULL, NULL, NULL)""" + sql """INSERT INTO t_types_null VALUES (3, NULL, 'partial', 2.0, false)""" + def nullCnt = sql """SELECT COUNT(*) FROM t_types_null""" + assertEquals(3, nullCnt[0][0] as int) + order_qt_types_null """SELECT id, c_int, c_string, c_double, c_boolean FROM t_types_null ORDER BY id""" + + // FT-043: Decimal precision + sql """ + INSERT INTO t_types_decimal VALUES + (1, CAST(1.5 AS DECIMAL(2,1)), CAST(12345678.90 AS DECIMAL(10,2)), + CAST(123456789012.123456 AS DECIMAL(18,6)), + CAST(1234567890123456789012345678.1234567890 AS DECIMAL(38,10))), + (2, CAST(-1.5 AS DECIMAL(2,1)), CAST(-0.01 AS DECIMAL(10,2)), + CAST(-1.000001 AS DECIMAL(18,6)), + CAST(0.0000000001 AS DECIMAL(38,10))), + (3, CAST(0.0 AS DECIMAL(2,1)), CAST(0.00 AS DECIMAL(10,2)), + CAST(0.000000 AS DECIMAL(18,6)), + CAST(0.0000000000 AS DECIMAL(38,10))) + """ + order_qt_types_decimal """SELECT id, d2, d10, d18, d38 FROM t_types_decimal ORDER BY id""" + + // DATE / DATETIME boundary + sql """ + INSERT INTO t_types_dt VALUES + (DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), + (DATE '2024-06-15', TIMESTAMP '2024-06-15 12:00:00'), + (DATE '2099-12-31', TIMESTAMP '2099-12-31 23:59:59.999999') + """ + order_qt_types_dt """SELECT d, dt FROM t_types_dt ORDER BY d""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +}