diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 87d35865dda7c1..5a65c81bebe632 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1552,6 +1552,11 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); // 1GB /** Iceberg sink configurations **/ DEFINE_mInt64(iceberg_sink_max_file_size, "1073741824"); // 1GB +/** Paimon sink configurations **/ +DEFINE_mInt64(paimon_jni_writer_memory_pool_limit_bytes, "536870912"); // 512MB +DEFINE_Validator(paimon_jni_writer_memory_pool_limit_bytes, + [](int64_t bytes) -> bool { return bytes > 0; }); + // URI scheme to Doris file type mappings used by paimon-cpp DorisFileSystem. // Each entry uses the format "=", and file_type must be one of: // local, hdfs, s3, http, broker. diff --git a/be/src/common/config.h b/be/src/common/config.h index 037674826db1ce..f143751088b9b6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1641,6 +1641,10 @@ DECLARE_mInt64(hive_sink_max_file_size); /** Iceberg sink configurations **/ DECLARE_mInt64(iceberg_sink_max_file_size); +/** Paimon sink configurations **/ +// Hard upper bound for Doris-managed Paimon write-buffer memory per JNI writer. +DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes); + /** Paimon file system configurations **/ DECLARE_Strings(paimon_file_system_scheme_mappings); diff --git a/be/src/exec/operator/operator.cpp b/be/src/exec/operator/operator.cpp index 34fa48dd06b71f..b6bbd64f96b4f8 100644 --- a/be/src/exec/operator/operator.cpp +++ b/be/src/exec/operator/operator.cpp @@ -63,6 +63,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -802,6 +803,7 @@ DECLARE_OPERATOR(OlapTableSinkV2LocalState) DECLARE_OPERATOR(HiveTableSinkLocalState) DECLARE_OPERATOR(TVFTableSinkLocalState) DECLARE_OPERATOR(IcebergTableSinkLocalState) +DECLARE_OPERATOR(PaimonTableSinkLocalState) DECLARE_OPERATOR(SpillIcebergTableSinkLocalState) DECLARE_OPERATOR(IcebergDeleteSinkLocalState) DECLARE_OPERATOR(IcebergMergeSinkLocalState) @@ -929,6 +931,7 @@ template class AsyncWriterSink; template class AsyncWriterSink; template class AsyncWriterSink; +template class AsyncWriterSink; #ifdef BE_TEST template class OperatorX; diff --git a/be/src/exec/operator/paimon_table_sink_operator.cpp b/be/src/exec/operator/paimon_table_sink_operator.cpp new file mode 100644 index 00000000000000..c1386be558ec62 --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.cpp @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/operator/paimon_table_sink_operator.h" + +#include "common/logging.h" + +namespace doris { + +Status PaimonTableSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info) { + return Base::init(state, info); +} + +Status PaimonTableSinkOperatorX::sink_impl(RuntimeState* state, Block* in_block, bool eos) { + auto& local_state = get_local_state(state); + SCOPED_TIMER(local_state.exec_time_counter()); + COUNTER_UPDATE(local_state.rows_input_counter(), static_cast(in_block->rows())); + + // Delegate to AsyncWriterSink → PaimonTableWriter for this pipeline instance. + // Each pipeline instance has its own writer session; partition and bucket + // routing is handled internally by the Paimon SDK inside IPaimonWriter::write(). + return local_state.sink(state, in_block, eos); +} + +} // namespace doris diff --git a/be/src/exec/operator/paimon_table_sink_operator.h b/be/src/exec/operator/paimon_table_sink_operator.h new file mode 100644 index 00000000000000..0bed36a27da75e --- /dev/null +++ b/be/src/exec/operator/paimon_table_sink_operator.h @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/operator/operator.h" +#include "exec/sink/writer/paimon/paimon_table_writer.h" +#include "runtime/runtime_state.h" + +namespace doris { + +/// Paimon table sink operator — simple pass-through to AsyncWriterSink. +/// +/// Each pipeline instance (LocalState) owns one PaimonTableWriter, which in +/// turn owns one IPaimonWriteBackend + IPaimonWriter. Pipeline parallelism +/// determines the number of concurrent Paimon writer sessions per table. +/// +/// Partition and bucket routing is performed internally by the Paimon SDK +/// (Java via JNI, or Rust via FFI). Doris does not compute partition values +/// or bucket ids; it passes complete Blocks through the backend to the SDK, +/// where each row is routed via getPartition(row) + getBucket(row). +/// +/// This mirrors Iceberg's approach: IcebergTableSinkOperatorX delegates to +/// AsyncWriterSink, with partition routing inside +/// VIcebergTableWriter::write(). +class PaimonTableSinkOperatorX; + +class PaimonTableSinkLocalState final + : public AsyncWriterSink { +public: + using Base = AsyncWriterSink; + using Parent = PaimonTableSinkOperatorX; + ENABLE_FACTORY_CREATOR(PaimonTableSinkLocalState); + PaimonTableSinkLocalState(DataSinkOperatorXBase* parent, RuntimeState* state) + : Base(parent, state) {} + Status init(RuntimeState* state, LocalSinkStateInfo& info) override; + Status open(RuntimeState* state) override { + SCOPED_TIMER(exec_time_counter()); + SCOPED_TIMER(_open_timer); + return Base::open(state); + } + + friend class PaimonTableSinkOperatorX; +}; + +class PaimonTableSinkOperatorX final : public DataSinkOperatorX { +public: + using Base = DataSinkOperatorX; + PaimonTableSinkOperatorX(ObjectPool* pool, int operator_id, const RowDescriptor& row_desc, + const std::vector& t_output_expr) + : Base(operator_id, 0, 0), + _row_desc(row_desc), + _t_output_expr(t_output_expr), + _pool(pool) {} + + Status init(const TDataSink& thrift_sink) override { + RETURN_IF_ERROR(Base::init(thrift_sink)); + DCHECK(thrift_sink.__isset.paimon_table_sink); + RETURN_IF_ERROR(VExpr::create_expr_trees(_t_output_expr, _output_vexpr_ctxs)); + return Status::OK(); + } + + Status prepare(RuntimeState* state) override { + RETURN_IF_ERROR(Base::prepare(state)); + RETURN_IF_ERROR(VExpr::prepare(_output_vexpr_ctxs, state, _row_desc)); + return VExpr::open(_output_vexpr_ctxs, state); + } + + Status sink_impl(RuntimeState* state, Block* in_block, bool eos) override; + +private: + friend class PaimonTableSinkLocalState; + template + requires(std::is_base_of_v) + friend class AsyncWriterSink; + + const RowDescriptor& _row_desc; + VExprContextSPtrs _output_vexpr_ctxs; + const std::vector& _t_output_expr; + ObjectPool* _pool = nullptr; +}; + +} // namespace doris diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index ef58e664462c25..4226bf51b17436 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -86,6 +86,7 @@ #include "exec/operator/olap_scan_operator.h" #include "exec/operator/olap_table_sink_operator.h" #include "exec/operator/olap_table_sink_v2_operator.h" +#include "exec/operator/paimon_table_sink_operator.h" #include "exec/operator/partition_sort_sink_operator.h" #include "exec/operator/partition_sort_source_operator.h" #include "exec/operator/partitioned_aggregation_sink_operator.h" @@ -1157,6 +1158,14 @@ Status PipelineFragmentContext::_create_data_sink(ObjectPool* pool, const TDataS output_exprs); break; } + case TDataSinkType::PAIMON_TABLE_SINK: { + if (!thrift_sink.__isset.paimon_table_sink) { + return Status::InternalError("Missing paimon table sink."); + } + _sink = std::make_shared(pool, next_sink_operator_id(), row_desc, + output_exprs); + break; + } case TDataSinkType::JDBC_TABLE_SINK: { if (!thrift_sink.__isset.jdbc_table_sink) { return Status::InternalError("Missing data jdbc sink."); @@ -2175,6 +2184,20 @@ void PipelineFragmentContext::_coordinator_callback(const ReportStatusRequest& r } } + if (auto pcm = req.runtime_state->paimon_commit_messages(); !pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), pcm.begin(), + pcm.end()); + } else if (!req.runtime_states.empty()) { + for (auto* rs : req.runtime_states) { + if (auto rs_pcm = rs->paimon_commit_messages(); !rs_pcm.empty()) { + params.__isset.paimon_commit_messages = true; + params.paimon_commit_messages.insert(params.paimon_commit_messages.end(), + rs_pcm.begin(), rs_pcm.end()); + } + } + } + req.runtime_state->get_unreported_errors(&(params.error_log)); params.__isset.error_log = (!params.error_log.empty()); diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp new file mode 100644 index 00000000000000..a5abfcdc15c41c --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.cpp @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" + +namespace doris { + +Status FfiPaimonWriteBackend::open(const TPaimonTableSink&, RuntimeState*, RuntimeProfile*) { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + +Status FfiPaimonWriteBackend::create_writer(std::unique_ptr*) { + return Status::NotSupported("Paimon Rust FFI writer is not implemented"); +} + +Status FfiPaimonWriteBackend::close() { + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h new file mode 100644 index 00000000000000..be833d53b79bcd --- /dev/null +++ b/be/src/exec/sink/writer/paimon/ffi_paimon_write_backend.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +/// Placeholder for the future paimon-rust writer implementation. Keeping this +/// backend in the factory makes the integration boundary explicit without +/// introducing a BE commit contract that the Rust writer will not own. +class FfiPaimonWriteBackend final : public IPaimonWriteBackend { +public: + Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) override; + Status create_writer(std::unique_ptr* writer) override; + Status close() override; + PaimonBackendType type() const override { return PaimonBackendType::FFI; } +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp new file mode 100644 index 00000000000000..5f6a7930da6d24 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp @@ -0,0 +1,487 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/logging.h" +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" +#include "format/arrow/arrow_block_convertor.h" +#include "format/arrow/arrow_row_batch.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "util/jni-util.h" +#include "util/pretty_printer.h" +#include "util/string_util.h" + +namespace doris { + +namespace { +constexpr std::string_view PAIMON_JNI_WRITER_IO_TMP_DIR = "paimon_jni_writer_io_tmp"; + +std::atomic& paimon_jni_close_failed() { + static auto* failed = new std::atomic(false); + return *failed; +} + +std::mutex& retained_memory_managers_mutex() { + static auto* mutex = new std::mutex(); + return *mutex; +} + +std::vector>& retained_memory_managers() { + static auto* managers = new std::vector>(); + return *managers; +} + +void retain_memory_after_failed_close(std::unique_ptr manager) { + paimon_jni_close_failed().store(true, std::memory_order_release); + if (manager == nullptr) { + return; + } + std::lock_guard lock(retained_memory_managers_mutex()); + retained_memory_managers().emplace_back(std::move(manager)); +} +} // namespace + +// ──────────────────────────────────────────────────────────── +// JNI helpers — JVM attachment and class loading +// ──────────────────────────────────────────────────────────── + +static constexpr const char* PAIMON_JNI_WRITER_CLASS = "org/apache/doris/paimon/PaimonJniWriter"; +static constexpr const char* SCANNER_LOADER_CLASS = + "org/apache/doris/common/classloader/ScannerLoader"; + +/// Attach the current native thread to the JVM if not already attached, +/// and return a valid JNIEnv pointer. +static Status _get_jni_env(JNIEnv** env) { + JavaVM* jvm = nullptr; + jsize n_vms = 0; + jint result = JNI_GetCreatedJavaVMs(&jvm, 1, &n_vms); + if (result != JNI_OK || n_vms == 0) { + return Status::InternalError("Failed to get created JavaVM"); + } + result = jvm->GetEnv(reinterpret_cast(env), JNI_VERSION_1_8); + if (result == JNI_EDETACHED) { + result = jvm->AttachCurrentThread(reinterpret_cast(env), nullptr); + if (result != JNI_OK) { + return Status::InternalError("Failed to attach current thread to JVM"); + } + } else if (result != JNI_OK) { + return Status::InternalError("Failed to get JNIEnv"); + } + return Status::OK(); +} + +JniPaimonWriteBackend::~JniPaimonWriteBackend() { + Status st = close(); + if (!st.ok()) { + LOG(WARNING) << "Failed to close Paimon JNI backend during destruction: " << st.to_string(); + } +} + +Status JniPaimonWriteBackend::close() { + if (_jni_writer_obj == nullptr && _jni_writer_cls == nullptr) { + _memory_manager.reset(); + _opened = false; + return Status::OK(); + } + + JNIEnv* env = nullptr; + Status env_status = _get_jni_env(&env); + if (!env_status.ok()) { + bool java_users_may_exist = _jni_writer_obj != nullptr; + // JNI global references cannot be released without an environment. + // Deliberately abandon the handles so the Java writer remains alive. + _jni_writer_obj = nullptr; + _jni_writer_cls = nullptr; + if (java_users_may_exist) { + retain_memory_after_failed_close(std::move(_memory_manager)); + } else { + _memory_manager.reset(); + } + _opened = false; + return env_status; + } + + Status close_status = Status::OK(); + if (_jni_writer_obj != nullptr) { + _refresh_memory_profile(); + if (_close_id == nullptr) { + close_status = Status::InternalError("PaimonJniWriter.close method is unavailable"); + } else { + env->CallVoidMethod(_jni_writer_obj, _close_id); + close_status = _check_jni_exception(env, "close PaimonJniWriter"); + } + env->DeleteGlobalRef(_jni_writer_obj); + _jni_writer_obj = nullptr; + } + if (_jni_writer_cls != nullptr) { + env->DeleteGlobalRef(_jni_writer_cls); + _jni_writer_cls = nullptr; + } + + if (close_status.ok()) { + _memory_manager.reset(); + } else { + if (_memory_manager != nullptr) { + LOG(WARNING) + << "Retaining Paimon JNI native memory after an unconfirmed Java close: limit=" + << PrettyPrinter::print_bytes(_memory_manager->memory_limit()) << ", peak=" + << PrettyPrinter::print_bytes(_memory_manager->native_peak_allocated_bytes()); + } + // Paimon may still have asynchronous flush or compaction tasks using + // MemorySegments backed by these pages. Retain ownership until process + // exit and reject new writers below. Retention is therefore limited to + // writers which were already open when the first close failure occurred. + retain_memory_after_failed_close(std::move(_memory_manager)); + } + _opened = false; + return close_status; +} + +Status JniPaimonWriteBackend::_check_jni_exception(JNIEnv* env, const std::string& method_name) { + if (env->ExceptionCheck()) { + Status st = + Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in " + method_name + ": "); + LOG(WARNING) << st.to_string(); + return st; + } + return Status::OK(); +} + +Status JniPaimonWriteBackend::_load_writer_class(JNIEnv* env, jclass* writer_class) { + jclass loader_class = env->FindClass(SCANNER_LOADER_CLASS); + RETURN_IF_ERROR(_check_jni_exception(env, "find ScannerLoader")); + + jmethodID loader_constructor = env->GetMethodID(loader_class, "", "()V"); + jmethodID get_loaded_class = env->GetMethodID(loader_class, "getLoadedClass", + "(Ljava/lang/String;)Ljava/lang/Class;"); + RETURN_IF_ERROR(_check_jni_exception(env, "resolve ScannerLoader methods")); + + jobject loader = env->NewObject(loader_class, loader_constructor); + jstring class_name = env->NewStringUTF(PAIMON_JNI_WRITER_CLASS); + auto* loaded_class = + static_cast(env->CallObjectMethod(loader, get_loaded_class, class_name)); + RETURN_IF_ERROR(_check_jni_exception(env, "load PaimonJniWriter")); + + *writer_class = loaded_class; + env->DeleteLocalRef(class_name); + env->DeleteLocalRef(loader); + env->DeleteLocalRef(loader_class); + return Status::OK(); +} + +static jobject _to_java_options(JNIEnv* env, const std::map& options) { + jclass map_cls = env->FindClass("java/util/HashMap"); + jmethodID map_ctor = env->GetMethodID(map_cls, "", "()V"); + jmethodID put_method = env->GetMethodID( + map_cls, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + + jobject map_obj = env->NewObject(map_cls, map_ctor); + for (const auto& kv : options) { + jstring key = env->NewStringUTF(kv.first.c_str()); + jstring val = env->NewStringUTF(kv.second.c_str()); + env->CallObjectMethod(map_obj, put_method, key, val); + env->DeleteLocalRef(key); + env->DeleteLocalRef(val); + } + env->DeleteLocalRef(map_cls); + return map_obj; +} + +Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) { + if (paimon_jni_close_failed().load(std::memory_order_acquire)) { + return Status::InternalError( + "Paimon JNI writes are disabled on this BE because a previous Java writer close " + "failed; restart the BE to reclaim retained native memory safely"); + } + _sink = sink; + DORIS_CHECK(sink.__isset.column_names); + DORIS_CHECK(sink.__isset.write_mode); + DORIS_CHECK(sink.__isset.serialized_table); + DORIS_CHECK(!sink.serialized_table.empty()); + DORIS_CHECK(sink.__isset.transaction_id); + DORIS_CHECK(sink.transaction_id > 0); + DORIS_CHECK(sink.__isset.commit_user); + DORIS_CHECK(!sink.commit_user.empty()); + DORIS_CHECK(profile != nullptr); + + RETURN_IF_ERROR(PaimonJniMemoryManager::create(state, &_memory_manager)); + RuntimeProfile* jni_profile = profile->create_child("JniPaimonWriteBackend", true, true); + _native_page_memory_limit = ADD_COUNTER(jni_profile, "NativePageMemoryLimit", TUnit::BYTES); + _native_page_memory_peak = ADD_COUNTER(jni_profile, "NativePageMemoryPeak", TUnit::BYTES); + + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + + // Step 1: Load PaimonJniWriter class through ScannerLoader (Paimon jars are + // not on the default application classpath, so FindClass won't work). + jclass local_cls = nullptr; + RETURN_IF_ERROR(_load_writer_class(env, &local_cls)); + _jni_writer_cls = static_cast(env->NewGlobalRef(local_cls)); + env->DeleteLocalRef(local_cls); + RETURN_IF_ERROR(PaimonJniMemoryManager::register_natives(env, _jni_writer_cls)); + + // Step 2: Cache JNI method IDs for write, prepareCommit, abort, close. + jmethodID open_id = env->GetMethodID( + _jni_writer_cls, "open", + "(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;JLjava/lang/String;ZLjava/lang/" + "String;Ljava/lang/String;JJ)V"); + _write_id = env->GetMethodID(_jni_writer_cls, "write", "(Ljava/nio/ByteBuffer;)V"); + _prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B"); + _abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V"); + _close_id = env->GetMethodID(_jni_writer_cls, "close", "()V"); + RETURN_IF_ERROR(_check_jni_exception(env, "GetMethodID")); + + // Step 3: Create the Java PaimonJniWriter instance. + jmethodID ctor_id = env->GetMethodID(_jni_writer_cls, "", "()V"); + jobject local_obj = env->NewObject(_jni_writer_cls, ctor_id); + RETURN_IF_ERROR(_check_jni_exception(env, "NewObject")); + _jni_writer_obj = env->NewGlobalRef(local_obj); + env->DeleteLocalRef(local_obj); + + // Step 4: Build Java arguments and call PaimonJniWriter.open(). + const std::map empty_config; + jstring j_serialized_table = env->NewStringUTF(sink.serialized_table.c_str()); + jobject j_hadoop_config = + _to_java_options(env, sink.__isset.hadoop_config ? sink.hadoop_config : empty_config); + jstring j_commit_user = env->NewStringUTF(sink.commit_user.c_str()); + jstring j_time_zone = env->NewStringUTF(state->timezone().c_str()); + std::vector spill_directories; + for (const auto& store_path : state->exec_env()->store_paths()) { + spill_directories.push_back(store_path.path + "/" + + std::string(PAIMON_JNI_WRITER_IO_TMP_DIR)); + } + DORIS_CHECK(!spill_directories.empty()); + jstring j_spill_directories = env->NewStringUTF(join(spill_directories, ":").c_str()); + + jclass string_cls = env->FindClass("java/lang/String"); + jobjectArray j_cols = + env->NewObjectArray(static_cast(sink.column_names.size()), string_cls, nullptr); + for (size_t i = 0; i < sink.column_names.size(); ++i) { + jstring str = env->NewStringUTF(sink.column_names[i].c_str()); + env->SetObjectArrayElement(j_cols, static_cast(i), str); + env->DeleteLocalRef(str); + } + + env->CallVoidMethod(_jni_writer_obj, open_id, j_serialized_table, j_hadoop_config, j_cols, + static_cast(sink.transaction_id), j_commit_user, + static_cast(sink.write_mode == TPaimonWriteMode::OVERWRITE), + j_time_zone, j_spill_directories, + static_cast(_memory_manager->memory_limit()), + reinterpret_cast(_memory_manager.get())); + Status st = _check_jni_exception(env, "open"); + + env->DeleteLocalRef(j_serialized_table); + env->DeleteLocalRef(j_hadoop_config); + env->DeleteLocalRef(j_commit_user); + env->DeleteLocalRef(j_time_zone); + env->DeleteLocalRef(j_spill_directories); + env->DeleteLocalRef(j_cols); + env->DeleteLocalRef(string_cls); + + if (st.ok()) { + _opened = true; + _refresh_memory_profile(); + LOG(INFO) << "Paimon JNI writer memory limit: " + << PrettyPrinter::print_bytes(_memory_manager->memory_limit()) + << ", local_sink_count=" << std::max(1, state->num_local_sink()); + } + return st; +} + +// Writer creation stays non-const because the backend interface also supports future stateful FFI +// implementations. +Status JniPaimonWriteBackend::create_writer( // NOLINT(readability-make-member-function-const) + std::unique_ptr* writer) { + DORIS_CHECK(_opened); + *writer = std::make_unique(_jni_writer_obj, _write_id, _prepare_commit_id, + _abort_id, std::make_unique>(), + _sink); + return Status::OK(); +} + +JniPaimonWriter::JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, + jmethodID prepare_commit_id, jmethodID abort_id, + std::unique_ptr> arrow_pool, + TPaimonTableSink sink) + : _jni_writer_obj(jni_writer_obj), + _write_id(write_id), + _prepare_commit_id(prepare_commit_id), + _abort_id(abort_id), + _arrow_pool(std::move(arrow_pool)), + _sink(std::move(sink)) {} + +Status JniPaimonWriter::_write_projected_block(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + // Use Thrift column_names as the authoritative schema source for both + // Arrow schema construction and Java-side write type derivation. + DORIS_CHECK(_sink.__isset.column_names); + DORIS_CHECK_EQ(_sink.column_names.size(), block.columns()); + for (size_t i = 0; i < _sink.column_names.size(); ++i) { + block.get_by_position(i).name = _sink.column_names[i]; + } + + // Pipeline: Doris Block → Arrow Schema → Arrow RecordBatch → IPC Stream → JNI direct buffer + // + // Step 1: Build Arrow schema from the projected Block. + // Paimon write timestamps are transported as civil-time fields. The Java writer uses the + // pinned Paimon target type to preserve NTZ values or convert LTZ values with the session zone. + std::shared_ptr arrow_schema; + RETURN_IF_ERROR(get_arrow_schema_from_block(block, &arrow_schema, "")); + + // Step 2: Convert Doris Block columns to an Arrow RecordBatch. + std::shared_ptr record_batch; + RETURN_IF_ERROR(convert_to_arrow_batch(block, arrow_schema, _arrow_pool.get(), &record_batch, + state->timezone_obj())); + + // Step 3: Serialize the RecordBatch to Arrow IPC Stream format in memory. + auto out_stream_res = arrow::io::BufferOutputStream::Create(4096, _arrow_pool.get()); + if (!out_stream_res.ok()) { + return Status::InternalError("Arrow BufferOutputStream create failed: {}", + out_stream_res.status().ToString()); + } + auto out_stream = *out_stream_res; + + auto writer_res = arrow::ipc::MakeStreamWriter(out_stream, arrow_schema); + if (!writer_res.ok()) { + return Status::InternalError("Arrow StreamWriter create failed: {}", + writer_res.status().ToString()); + } + auto ipc_writer = *writer_res; + if (!ipc_writer->WriteRecordBatch(*record_batch).ok()) { + return Status::InternalError("Arrow WriteRecordBatch failed"); + } + if (!ipc_writer->Close().ok()) { + return Status::InternalError("Arrow StreamWriter close failed"); + } + + auto buffer_res = out_stream->Finish(); + if (!buffer_res.ok()) { + return Status::InternalError("Arrow output stream finish failed: {}", + buffer_res.status().ToString()); + } + std::shared_ptr buffer = *buffer_res; + + // Step 4: Wrap the IPC buffer in a JNI direct ByteBuffer (zero-copy) and + // call PaimonJniWriter.write(ByteBuffer). Java side reads the Arrow IPC + // stream via ArrowStreamReader. + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + + jobject direct_buffer = + env->NewDirectByteBuffer(buffer->mutable_data(), static_cast(buffer->size())); + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception in NewDirectByteBuffer for PaimonJniWriter::write: ")); + + env->CallVoidMethod(_jni_writer_obj, _write_id, direct_buffer); + Status write_status = + Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in JniPaimonWriter::write: "); + env->DeleteLocalRef(direct_buffer); + return write_status; +} + +Status JniPaimonWriter::write(RuntimeState* state, Block& block) { + return _write_projected_block(state, block); +} + +Status JniPaimonWriter::prepare_commit(std::vector& messages) { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + + // Call PaimonJniWriter.prepareCommit() which returns byte[][] — + // each element is a DPCM-framed serialized CommitMessage chunk produced + // by PaimonCommitCodec.encode(). + jobject j_payloads_obj = env->CallObjectMethod(_jni_writer_obj, _prepare_commit_id); + Status st = Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in prepareCommit: "); + if (!st.ok()) { + return st; + } + + if (j_payloads_obj == nullptr) { + return Status::InternalError("PaimonJniWriter.prepareCommit returned null"); + } + + // Unpack the byte[][] into TPaimonCommitMessage structs for FE transport. + auto* j_payloads = static_cast(j_payloads_obj); + jsize num_payloads = env->GetArrayLength(j_payloads); + + for (jsize i = 0; i < num_payloads; ++i) { + auto j_bytes = static_cast(env->GetObjectArrayElement(j_payloads, i)); + if (j_bytes == nullptr) { + env->DeleteLocalRef(j_payloads); + return Status::InternalError("PaimonJniWriter.prepareCommit returned a null payload"); + } + jsize len = env->GetArrayLength(j_bytes); + if (len == 0) { + env->DeleteLocalRef(j_bytes); + env->DeleteLocalRef(j_payloads); + return Status::InternalError("PaimonJniWriter.prepareCommit returned an empty payload"); + } + jbyte* bytes = env->GetByteArrayElements(j_bytes, nullptr); + if (bytes == nullptr) { + env->DeleteLocalRef(j_bytes); + env->DeleteLocalRef(j_payloads); + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, false, "JNI exception while reading Paimon commit payload: ")); + return Status::InternalError("Failed to read Paimon commit payload"); + } + std::string payload(reinterpret_cast(bytes), static_cast(len)); + TPaimonCommitMessage msg; + msg.__set_payload(payload); + messages.emplace_back(std::move(msg)); + env->ReleaseByteArrayElements(j_bytes, bytes, JNI_ABORT); + env->DeleteLocalRef(j_bytes); + } + env->DeleteLocalRef(j_payloads); + return Status::OK(); +} + +Status JniPaimonWriter::abort() { + JNIEnv* env = nullptr; + RETURN_IF_ERROR(_get_jni_env(&env)); + env->CallVoidMethod(_jni_writer_obj, _abort_id); + return Jni::Env::GetJniExceptionMsg(env, true, "JNI exception in abort: "); +} + +void JniPaimonWriteBackend::_refresh_memory_profile() { + if (_memory_manager == nullptr) { + return; + } + COUNTER_SET(_native_page_memory_limit, _memory_manager->memory_limit()); + COUNTER_SET(_native_page_memory_peak, _memory_manager->native_peak_allocated_bytes()); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h new file mode 100644 index 00000000000000..ae36bdf37a3582 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/status.h" +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "format/parquet/arrow_memory_pool.h" +#include "runtime/runtime_profile.h" + +namespace doris { + +class RuntimeState; + +/// JNI backend that owns the Java PaimonJniWriter object and its JNI method +/// handles. Creates lightweight JniPaimonWriter adapters that share this +/// backend's JVM connection. +/// +/// Each JniPaimonWriteBackend corresponds to one Java PaimonJniWriter +/// instance; the JniPaimonWriter adapters are thin wrappers that delegate +/// write/prepare_commit/abort calls through the cached JNI method IDs. JNI-only +/// memory ownership and Profile counters stay here and are not part of the +/// common backend contract. +class JniPaimonWriteBackend final : public IPaimonWriteBackend { +public: + ~JniPaimonWriteBackend() override; + + Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) override; + Status create_writer(std::unique_ptr* writer) override; + Status close() override; + PaimonBackendType type() const override { return PaimonBackendType::JNI; } + +private: + Status _check_jni_exception(JNIEnv* env, const std::string& method_name); + Status _load_writer_class(JNIEnv* env, jclass* writer_class); + void _refresh_memory_profile(); + + // JNI global references — live for the duration of this backend. + jclass _jni_writer_cls = nullptr; + jobject _jni_writer_obj = nullptr; + + // Cached JNI method IDs for the PaimonJniWriter Java methods. + jmethodID _write_id = nullptr; + jmethodID _prepare_commit_id = nullptr; + jmethodID _abort_id = nullptr; + jmethodID _close_id = nullptr; + + TPaimonTableSink _sink; + std::unique_ptr _memory_manager; + RuntimeProfile::Counter* _native_page_memory_limit = nullptr; + RuntimeProfile::Counter* _native_page_memory_peak = nullptr; + bool _opened = false; +}; + +/// Lightweight C++ adapter that delegates to the shared JNI backend. +/// +/// Owns the Arrow memory pool used for Block → Arrow IPC conversion. +/// Each JniPaimonWriter is created by JniPaimonWriteBackend::create_writer() +/// and shares the backend's JNI method IDs and Java writer object reference. +class JniPaimonWriter final : public IPaimonWriter { +public: + JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, jmethodID prepare_commit_id, + jmethodID abort_id, std::unique_ptr> arrow_pool, + TPaimonTableSink sink); + + Status write(RuntimeState* state, Block& block) override; + Status prepare_commit(std::vector& messages) override; + Status abort() override; + +private: + /// Convert Block → Arrow RecordBatch → IPC Stream, then pass to Java via JNI direct buffer. + Status _write_projected_block(RuntimeState* state, Block& block); + + // Shared JNI state (owned by JniPaimonWriteBackend, not this adapter). + jobject _jni_writer_obj; + jmethodID _write_id; + jmethodID _prepare_commit_id; + jmethodID _abort_id; + + // Arrow resources owned by this writer adapter. + std::unique_ptr> _arrow_pool; + TPaimonTableSink _sink; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp new file mode 100644 index 00000000000000..63eae9904e8c73 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.cpp @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_jni_memory_manager.h" + +#include +#include +#include +#include + +#include "common/check.h" +#include "common/config.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/allocator.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "runtime/query_context.h" +#include "runtime/runtime_state.h" +#include "runtime/thread_context.h" +#include "util/defer_op.h" +#include "util/jni-util.h" +#include "util/pretty_printer.h" + +namespace doris { + +class PaimonJniMemoryManager::Impl { +public: + Impl(std::shared_ptr resource_context, int64_t memory_limit) + : _resource_context(std::move(resource_context)), _memory_limit(memory_limit) { + DORIS_CHECK(_resource_context != nullptr); + DORIS_CHECK(_memory_limit > 0); + } + + ~Impl() { + // Java may retain direct buffers until its writer is closed. Release + // every outstanding page here as the final native ownership boundary. + try { + release_all_pages(); + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to release Paimon JNI native memory: " << e.what(); + } catch (...) { + LOG(WARNING) << "Failed to release Paimon JNI native memory: unknown exception"; + } + } + + jobject allocate_page(JNIEnv* env, jint bytes) { + if (bytes <= 0) { + throw Exception(Status::InvalidArgument( + "Paimon JNI memory page size must be positive, actual={}", bytes)); + } + + // Reserve the writer-local budget before entering the allocator. This + // prevents concurrent JNI callbacks from transiently allocating past + // the configured cap and only discovering it after query accounting + // or the system allocator has already rejected the request. + { + std::lock_guard lock(_mutex); + if (bytes > _memory_limit - _native_allocated_bytes - _native_reserved_bytes) { + throw Exception(Status::Error( + "Paimon JNI write buffer exceeded its {} native memory limit", + PrettyPrinter::print_bytes(_memory_limit))); + } + _native_reserved_bytes += bytes; + } + bool reservation_committed = false; + Defer rollback_reservation {[&]() { + if (!reservation_committed) { + std::lock_guard lock(_mutex); + _native_reserved_bytes -= bytes; + } + }}; + + // Allocate and account while attached to the query's resource + // context. The callback can run on a JVM-created thread, so merely + // relying on the calling BE thread's context would bypass query + // memory accounting. + void* address = with_resource_context([&]() { + enable_thread_catch_bad_alloc++; + Defer restore_bad_alloc_catch {[&]() { enable_thread_catch_bad_alloc--; }}; + void* allocated = _allocator.alloc(static_cast(bytes)); + try { + std::lock_guard lock(_mutex); + _allocations.emplace_back(allocated, static_cast(bytes)); + _native_reserved_bytes -= bytes; + _native_allocated_bytes += bytes; + _native_peak_allocated_bytes = + std::max(_native_peak_allocated_bytes, _native_allocated_bytes); + reservation_committed = true; + } catch (...) { + _allocator.free(allocated, static_cast(bytes)); + throw; + } + return allocated; + }); + + // NewDirectByteBuffer does not copy memory; Paimon will read/write the + // page directly. If JNI rejects the address, undo the native + // allocation and its accounting entry before returning. + jobject buffer = env->NewDirectByteBuffer(address, bytes); + if (buffer == nullptr || env->ExceptionCheck()) { + remove_and_free_page(address, static_cast(bytes)); + return nullptr; + } + return buffer; + } + + int64_t memory_limit() const { return _memory_limit; } + + int64_t native_peak_allocated_bytes() const { + std::lock_guard lock(_mutex); + return _native_peak_allocated_bytes; + } + +private: + template + auto with_resource_context(Function&& function) + -> decltype(std::forward(function)()) { + // JNI normally re-enters on an attached async-writer thread. Attach + // Java-created threads explicitly too, so every allocation/free is + // charged to the query rather than to an unrelated thread context. + if (!pthread_context_ptr_init && bthread_self() == 0) { + SCOPED_ATTACH_TASK(_resource_context); + return std::forward(function)(); + } + if (thread_context()->is_attach_task()) { + SCOPED_SWITCH_RESOURCE_CONTEXT(_resource_context); + return std::forward(function)(); + } + SCOPED_ATTACH_TASK(_resource_context); + return std::forward(function)(); + } + + void release_all_pages() { + // Detach ownership from the bookkeeping vector under the lock, then + // free outside the lock. Allocator/free may invoke code that takes + // unrelated locks and must not block page accounting readers. + std::vector> allocations; + { + std::lock_guard lock(_mutex); + allocations.swap(_allocations); + _native_allocated_bytes = 0; + } + if (allocations.empty()) { + return; + } + + with_resource_context([&]() { + for (const auto& [address, bytes] : allocations) { + _allocator.free(address, bytes); + } + std::vector>().swap(allocations); + }); + } + + void remove_and_free_page(void* address, size_t bytes) { + // Roll back a page whose Java direct-buffer wrapper could not be + // created. The address is removed under the same lock used by the + // normal accounting path, while the potentially expensive free is + // performed after releasing it. + { + std::lock_guard lock(_mutex); + auto it = std::find_if( + _allocations.begin(), _allocations.end(), + [&](const auto& allocation) { return allocation.first == address; }); + if (it != _allocations.end()) { + _allocations.erase(it); + _native_allocated_bytes -= bytes; + } + } + with_resource_context([&]() { _allocator.free(address, bytes); }); + } + + // Query resource context used for all native allocator operations. + std::shared_ptr _resource_context; + // Immutable per-writer cap, calculated by PaimonJniMemoryManager::create. + const int64_t _memory_limit; + // Doris allocator used instead of JVM/Arrow allocation so native pages are + // visible to Doris' memory accounting and allocator hooks. + Allocator _allocator; + // Protects the allocation list and both usage counters. JNI callbacks and + // Java close/finalizer paths may arrive concurrently. + mutable std::mutex _mutex; + // Every entry is (native address, size) and remains here until released. + std::vector> _allocations; + // Bytes reserved by callbacks which have passed the local limit check but + // have not yet completed their allocator call. + int64_t _native_reserved_bytes = 0; + // Committed and high-water native page usage, respectively. + int64_t _native_allocated_bytes = 0; + int64_t _native_peak_allocated_bytes = 0; +}; + +namespace { + +jobject allocate_paimon_memory_page(JNIEnv* env, jclass, jlong manager_handle, jint bytes) { + // This is called from PaimonJniWriter's Java memory pool. The handle is + // the native manager address passed when the writer is opened; ownership + // stays with the C++ writer/backend, so this callback must never delete it. + auto* manager = reinterpret_cast(manager_handle); + if (manager == nullptr) { + jclass exception_class = env->FindClass("java/lang/IllegalStateException"); + env->ThrowNew(exception_class, "Paimon JNI memory manager is null"); + env->DeleteLocalRef(exception_class); + return nullptr; + } + try { + return manager->allocate_page(env, bytes); + } catch (const std::exception& e) { + jclass exception_class = env->FindClass("java/lang/OutOfMemoryError"); + env->ThrowNew(exception_class, e.what()); + env->DeleteLocalRef(exception_class); + return nullptr; + } +} + +} // namespace + +PaimonJniMemoryManager::PaimonJniMemoryManager(std::unique_ptr impl) + : _impl(std::move(impl)) {} + +PaimonJniMemoryManager::~PaimonJniMemoryManager() = default; + +Status PaimonJniMemoryManager::create(RuntimeState* state, + std::unique_ptr* manager) { + DORIS_CHECK(state != nullptr); + DORIS_CHECK(manager != nullptr); + if (state->query_mem_tracker() == nullptr) { + return Status::InternalError( + "Paimon JNI writer cannot size its write buffer without a query tracker"); + } + if (state->get_query_ctx() == nullptr) { + return Status::InternalError( + "Paimon JNI writer cannot allocate native memory without QueryContext"); + } + + // A query can create multiple local sink instances. Divide its budget + // before applying the configured cap so one writer cannot consume the + // entire query allowance. + const int64_t writer_count = std::max(1, state->num_local_sink()); + const int64_t query_limit = state->query_mem_tracker()->limit(); + const int64_t query_share = query_limit > 0 ? query_limit / writer_count : query_limit; + const int64_t configured_memory_limit = config::paimon_jni_writer_memory_pool_limit_bytes; + const int64_t memory_limit = query_share > 0 ? std::min(query_share, configured_memory_limit) + : configured_memory_limit; + if (memory_limit <= 0) { + return Status::Error( + "Paimon JNI writer has insufficient memory budget: query_limit={}, " + "local_sink_count={}, write_buffer_limit={}", + PrettyPrinter::print_bytes(query_limit), writer_count, + PrettyPrinter::print_bytes(memory_limit)); + } + + // ResourceContext is retained by Impl for the manager's whole lifetime; + // this is what keeps asynchronous JNI callbacks associated with the query. + auto impl = std::make_unique(state->get_query_ctx()->resource_ctx(), memory_limit); + *manager = std::unique_ptr(new PaimonJniMemoryManager(std::move(impl))); + return Status::OK(); +} + +Status PaimonJniMemoryManager::register_natives(JNIEnv* env, jclass writer_class) { + // Keep the JNI surface minimal: Java asks native code only for a page; + // all ownership, limits, and cleanup stay in PaimonJniMemoryManager. + static char allocate_name[] = "allocatePaimonMemoryPage"; + static char allocate_signature[] = "(JI)Ljava/nio/ByteBuffer;"; + static ::JNINativeMethod methods[] = { + {allocate_name, allocate_signature, + reinterpret_cast(&allocate_paimon_memory_page)}, + }; + if (env->RegisterNatives(writer_class, methods, + static_cast(sizeof(methods) / sizeof(methods[0]))) != JNI_OK) { + RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg( + env, true, "JNI exception registering Paimon memory native methods: ")); + return Status::JniError("Failed to register Paimon memory native methods"); + } + return Status::OK(); +} + +jobject PaimonJniMemoryManager::allocate_page(JNIEnv* env, jint bytes) { + return _impl->allocate_page(env, bytes); +} + +int64_t PaimonJniMemoryManager::memory_limit() const { + return _impl->memory_limit(); +} + +int64_t PaimonJniMemoryManager::native_peak_allocated_bytes() const { + return _impl->native_peak_allocated_bytes(); +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h new file mode 100644 index 00000000000000..0d4818cd6dce42 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_jni_memory_manager.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" + +namespace doris { + +class RuntimeState; + +/// Owns the Doris-side native memory used by one Java Paimon writer. +/// +/// Paimon's sort/merge buffers are Java objects, but their page storage is +/// requested through a JNI callback. This manager is the bridge for that +/// callback: it allocates each page with Doris' allocator, exposes the page as +/// a direct ByteBuffer, tracks it until the writer is closed, and releases all +/// pages in its destructor. The native writer/backend therefore keeps this +/// manager alive for at least as long as the Java writer can access its +/// callback handle. +/// +/// The limit is a per-writer budget. It is derived from the query memory +/// limit and the number of local sink instances, then capped by the global +/// Paimon JNI configuration. The manager accounts only for pages allocated +/// by this callback; Java heap and other Paimon-managed memory remain under +/// their respective runtimes. +class PaimonJniMemoryManager { +public: + ~PaimonJniMemoryManager(); + + /// Construct a manager whose budget is sized from the query context. + /// + /// The query must provide both a memory tracker and QueryContext. The + /// latter supplies the ResourceContext used whenever allocation/freeing + /// crosses into a JNI-created or asynchronous thread. + static Status create(RuntimeState* state, std::unique_ptr* manager); + /// Register the static JNI callback used by PaimonJniWriter. + static Status register_natives(JNIEnv* env, jclass writer_class); + + /// Allocate one native page and return it as a direct ByteBuffer. + /// + /// On failure this method leaves no accounting entry behind and reports + /// the error through the JNI environment. The returned buffer remains + /// valid until the manager is destroyed (or allocation of that page is + /// rolled back because NewDirectByteBuffer failed). + jobject allocate_page(JNIEnv* env, jint bytes); + + /// Return the immutable per-writer native page budget in bytes. + int64_t memory_limit() const; + + /// Return the high-water mark of native pages allocated by this manager. + int64_t native_peak_allocated_bytes() const; + +private: + class Impl; + + explicit PaimonJniMemoryManager(std::unique_ptr impl); + + std::unique_ptr _impl; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp new file mode 100644 index 00000000000000..95ddca671097bd --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.cpp @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_table_writer.h" + +#include "common/check.h" +#include "common/logging.h" +#include "core/block/block.h" +#include "runtime/runtime_state.h" + +namespace doris { + +PaimonTableWriter::PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs, + std::shared_ptr dep, + std::shared_ptr fin_dep) + : AsyncResultWriter(output_exprs, std::move(dep), std::move(fin_dep)), + _t_sink(std::move(t_sink)) { + DCHECK(_t_sink.__isset.paimon_table_sink); +} + +Status PaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) { + _state = state; + _operator_profile = profile; + + // Register profile counters + _written_rows_counter = ADD_COUNTER(_operator_profile, "WrittenRows", TUnit::UNIT); + _written_bytes_counter = ADD_COUNTER(_operator_profile, "WrittenBytes", TUnit::BYTES); + _send_data_timer = ADD_TIMER(_operator_profile, "SendDataTime"); + _project_timer = ADD_CHILD_TIMER(_operator_profile, "ProjectTime", "SendDataTime"); + _file_store_write_timer = + ADD_CHILD_TIMER(_operator_profile, "FileStoreWriteTime", "SendDataTime"); + _open_timer = ADD_TIMER(_operator_profile, "OpenTime"); + _close_timer = ADD_TIMER(_operator_profile, "CloseTime"); + _prepare_commit_timer = ADD_TIMER(_operator_profile, "PrepareCommitTime"); + _commit_payload_count = ADD_COUNTER(_operator_profile, "CommitPayloadCount", TUnit::UNIT); + _commit_payload_bytes_counter = + ADD_COUNTER(_operator_profile, "CommitPayloadBytes", TUnit::BYTES); + + SCOPED_TIMER(_open_timer); + + // Step 1: Create the backend (JNI or FFI) based on the sink configuration. + RETURN_IF_ERROR(PaimonWriteBackendFactory::create(_t_sink.paimon_table_sink, &_backend)); + DCHECK(_backend); + // Step 2: Open the backend — for JNI this loads the Java class and calls PaimonJniWriter.open(). + RETURN_IF_ERROR(_backend->open(_t_sink.paimon_table_sink, state, profile)); + // Step 3: Create a lightweight writer adapter that delegates to the opened backend. + RETURN_IF_ERROR(_backend->create_writer(&_writer)); + DCHECK(_writer); + + LOG(INFO) << "PaimonTableWriter opened: backend=" << static_cast(_backend->type()) + << ", writer_scope=local_state"; + return Status::OK(); +} + +Status PaimonTableWriter::write(RuntimeState* state, Block& block) { + if (block.rows() == 0) { + return Status::OK(); + } + + SCOPED_TIMER(_send_data_timer); + + // Step 1: Apply output expressions to produce the columns selected by FE. + Block output_block; + { + SCOPED_TIMER(_project_timer); + RETURN_IF_ERROR(_projection_block(block, &output_block)); + } + + COUNTER_UPDATE(_written_rows_counter, block.rows()); + COUNTER_UPDATE(_written_bytes_counter, block.bytes()); + _state->update_num_rows_load_total(block.rows()); + _state->update_num_bytes_load_total(block.bytes()); + + // Step 2: Delegate to the backend writer (JNI or FFI). For the JNI path + // this converts Block → Arrow IPC → direct buffer → Java PaimonJniWriter. + DCHECK(_writer); + { + SCOPED_TIMER(_file_store_write_timer); + RETURN_IF_ERROR(_writer->write(_state, output_block)); + } + _written_rows += block.rows(); + return Status::OK(); +} + +Status PaimonTableWriter::close(Status status) { + SCOPED_TIMER(_close_timer); + + // Prepare messages first, but do not publish them until the backend confirms + // that every SDK user has stopped and its native backing memory is safe to release. + std::vector messages; + if (status.ok()) { + DCHECK(_writer); + { + SCOPED_TIMER(_prepare_commit_timer); + Status prep_st = _writer->prepare_commit(messages); + if (!prep_st.ok()) { + status = prep_st; + } + } + } + + // If prepare_commit failed or the incoming status was already an error, + // abort the writer to clean up uncommitted data files. + if (!status.ok()) { + LOG(WARNING) << "Paimon writer closing with error: " << status.to_string(); + if (_writer) { + Status abort_st = _writer->abort(); + if (!abort_st.ok()) { + LOG(WARNING) << "Paimon writer abort failed: " << abort_st.to_string(); + } + } + } + + // The adapter only owns Arrow conversion resources. Release it before closing + // the backend, whose Java close is the authoritative SDK shutdown boundary. + _writer.reset(); + + if (_backend) { + Status close_st = _backend->close(); + if (!close_st.ok()) { + if (status.ok()) { + status = close_st; + } else { + LOG(WARNING) << "Paimon backend close also failed: " << close_st.to_string(); + } + } + } + + // Only a fully prepared and cleanly stopped writer may contribute payloads + // to the FE transaction. A Java close failure therefore aborts the Doris + // transaction instead of allowing it to commit potentially unsafe output. + if (status.ok()) { + COUNTER_UPDATE(_commit_payload_count, static_cast(messages.size())); + for (const auto& msg : messages) { + DORIS_CHECK(msg.__isset.payload); + COUNTER_UPDATE(_commit_payload_bytes_counter, static_cast(msg.payload.size())); + } + if (!messages.empty()) { + _state->add_paimon_commit_messages(messages); + LOG(INFO) << "Paimon writer closed: " << messages.size() + << " commit messages, total rows=" << _written_rows; + } + } + + _backend.reset(); + return status; +} + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_table_writer.h b/be/src/exec/sink/writer/paimon/paimon_table_writer.h new file mode 100644 index 00000000000000..cbfc3209e80285 --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_table_writer.h @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" +#include "exec/sink/writer/async_result_writer.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" +#include "exprs/vexpr_fwd.h" +#include "runtime/runtime_profile.h" + +namespace doris { + +class RuntimeState; + +/// Each PaimonTableSinkLocalState owns one PaimonTableWriter, which in turn +/// owns one IPaimonWriteBackend and one IPaimonWriter. Pipeline parallelism +/// therefore determines the number of independent Paimon writer sessions; +/// each writer session delegates partition and bucket routing to the Paimon +/// SDK (Java via JNI, or Rust via FFI in the future). +/// +/// Doris does NOT compute partition values or bucket ids — it passes complete +/// Blocks through the selected backend (JNI/FFI) to the Paimon SDK, which +/// internally computes partition values, bucket ids, and routes rows to the +/// correct file writers. +/// +/// Architecture: +/// PaimonTableSinkOperatorX +/// │ sink_impl() → AsyncWriterSink::sink() (no routing) +/// ▼ +/// PaimonTableWriter (one per LocalState / pipeline instance) +/// │ owns IPaimonWriteBackend (JNI or FFI) +/// │ └─ create_writer() → IPaimonWriter +/// │ write() +/// │ → JNI backend: Block → Arrow IPC → Java Paimon SDK +/// │ → FFI backend: Block → Rust writer (future) +/// │ → selected SDK owns row normalization, routing, buffering, +/// │ file writing, and compaction +/// ▼ +/// close() → prepareCommit() → CommitMessage[] +/// +/// Commit flow (BE only prepares messages; FE is the commit coordinator): +/// close() → writer->prepare_commit() +/// → collect TPaimonCommitMessage[] (DPCM-framed serialized messages) +/// → RuntimeState::add_paimon_commit_messages() +/// → RPC to FE Coordinator → PaimonTransaction +class PaimonTableWriter final : public AsyncResultWriter { +public: + PaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs, + std::shared_ptr dep, std::shared_ptr fin_dep); + + ~PaimonTableWriter() override = default; + + Status open(RuntimeState* state, RuntimeProfile* profile) override; + + Status write(RuntimeState* state, Block& block) override; + + Status close(Status status) override; + +private: + TDataSink _t_sink; + RuntimeState* _state = nullptr; + + // Backend owns the JNI/FFI connection and creates the writer adapter. + // Both are scoped to this PaimonTableWriter (one per LocalState). + std::unique_ptr _backend; + std::unique_ptr _writer; + + // Profile counters + RuntimeProfile::Counter* _written_rows_counter = nullptr; + RuntimeProfile::Counter* _written_bytes_counter = nullptr; + RuntimeProfile::Counter* _send_data_timer = nullptr; + RuntimeProfile::Counter* _project_timer = nullptr; + RuntimeProfile::Counter* _file_store_write_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _prepare_commit_timer = nullptr; + RuntimeProfile::Counter* _commit_payload_count = nullptr; + RuntimeProfile::Counter* _commit_payload_bytes_counter = nullptr; +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend.h b/be/src/exec/sink/writer/paimon/paimon_write_backend.h new file mode 100644 index 00000000000000..d28b204a4c1aac --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend.h @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" +#include "core/block/block.h" + +namespace doris { + +class RuntimeState; +class RuntimeProfile; + +enum class PaimonBackendType { + JNI, // Java via JNI (PaimonJniWriter) + FFI, // Rust via FFI (placeholder, not yet implemented) +}; + +/// Writer contract implemented by one SDK writer adapter. Each +/// PaimonTableWriter owns one IPaimonWriter, which delegates to the +/// underlying Paimon SDK (Java JNI or Rust FFI). Partition and bucket +/// routing happens inside the selected SDK backend. +/// +/// Lifecycle: created by IPaimonWriteBackend::create_writer() after the +/// backend is opened; used for the duration of one pipeline instance. +class IPaimonWriter { +public: + virtual ~IPaimonWriter() = default; + + /// Write a projected Block to the Paimon SDK. + /// For the JNI path: Block → Arrow IPC → direct buffer → Java. + virtual Status write(RuntimeState* state, Block& block) = 0; + + /// Flush all buffered data, close files, and collect serialized commit + /// messages (DPCM-framed). Called once at EOS. + virtual Status prepare_commit(std::vector& messages) = 0; + + /// Discard written data files on error. Called when write or prepare_commit fails. + virtual Status abort() = 0; +}; + +/// Backend boundary for creating writers via JNI (Java) or FFI (Rust). +/// +/// The backend owns the connection/session to the external runtime: +/// - JNI: owns the JVM class reference, method IDs, and the Java writer object. +/// - FFI: (future) owns the Rust FFI handle. +/// +/// Each backend creates one or more IPaimonWriter adapters that share the +/// same underlying connection. Snapshot commit is deliberately excluded from +/// this boundary: BE only prepares commit messages (byte payloads), while FE +/// PaimonTransaction is the single commit coordinator. +class IPaimonWriteBackend { +public: + virtual ~IPaimonWriteBackend() = default; + + /// Initialize the backend connection. For JNI this loads the writer class, + /// creates the Java object, and calls PaimonJniWriter.open(). + virtual Status open(const TPaimonTableSink& sink, RuntimeState* state, + RuntimeProfile* profile) = 0; + + /// Create a lightweight writer adapter that delegates to this backend. + virtual Status create_writer(std::unique_ptr* writer) = 0; + + /// Stop all SDK users and release backend resources. + /// + /// A successful return is the ownership boundary after which native memory + /// backing SDK buffers can be reclaimed safely. Callers must not publish + /// prepared commit messages until this succeeds. + virtual Status close() = 0; + + virtual PaimonBackendType type() const = 0; +}; + +/// Factory that selects and creates the appropriate write backend. +/// +/// Backend selection is based on TPaimonTableSink.backend_type: +/// - Default (unset or JNI): JniPaimonWriteBackend +/// - FFI: FfiPaimonWriteBackend (placeholder for future Rust writer) +class PaimonWriteBackendFactory { +public: + /// Create a backend instance based on the sink configuration. + static Status create(const TPaimonTableSink& sink, + std::unique_ptr* backend); + + /// Determine which backend type to use for the given sink. + static PaimonBackendType select_backend_type(const TPaimonTableSink& sink); +}; + +} // namespace doris diff --git a/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp new file mode 100644 index 00000000000000..087228abbe5d2b --- /dev/null +++ b/be/src/exec/sink/writer/paimon/paimon_write_backend_factory.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/ffi_paimon_write_backend.h" +#include "exec/sink/writer/paimon/jni_paimon_write_backend.h" +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +namespace doris { + +Status PaimonWriteBackendFactory::create(const TPaimonTableSink& sink, + std::unique_ptr* backend) { + switch (select_backend_type(sink)) { + case PaimonBackendType::JNI: + *backend = std::make_unique(); + return Status::OK(); + case PaimonBackendType::FFI: + *backend = std::make_unique(); + return Status::OK(); + } + return Status::InternalError("Unknown Paimon write backend"); +} + +PaimonBackendType PaimonWriteBackendFactory::select_backend_type(const TPaimonTableSink& sink) { + if (sink.__isset.backend_type && sink.backend_type == TPaimonWriteBackendType::FFI) { + return PaimonBackendType::FFI; + } + return PaimonBackendType::JNI; +} + +} // namespace doris diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index fc764085c8698a..d378c2f4426025 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -552,6 +552,17 @@ class RuntimeState { _mc_commit_datas.emplace_back(mc_commit_data); } + std::vector paimon_commit_messages() const { + std::lock_guard lock(_paimon_commit_messages_mutex); + return _paimon_commit_messages; + } + + void add_paimon_commit_messages(const std::vector& commit_messages) { + std::lock_guard lock(_paimon_commit_messages_mutex); + _paimon_commit_messages.insert(_paimon_commit_messages.end(), commit_messages.begin(), + commit_messages.end()); + } + // local runtime filter mgr, the runtime filter do not have remote target or // not need local merge should regist here. the instance exec finish, the local // runtime filter mgr can release the memory of local runtime filter @@ -982,6 +993,9 @@ class RuntimeState { mutable std::mutex _mc_commit_datas_mutex; std::vector _mc_commit_datas; + mutable std::mutex _paimon_commit_messages_mutex; + std::vector _paimon_commit_messages; + std::vector> _op_id_to_local_state; std::unique_ptr _sink_local_state; diff --git a/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp new file mode 100644 index 00000000000000..35be9c884d727d --- /dev/null +++ b/be/test/exec/sink/writer/paimon/paimon_write_backend_test.cpp @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/sink/writer/paimon/paimon_write_backend.h" + +#include + +namespace doris { + +TEST(PaimonWriteBackendFactoryTest, SelectBackendType) { + TPaimonTableSink sink; + EXPECT_EQ(PaimonBackendType::JNI, PaimonWriteBackendFactory::select_backend_type(sink)); + + sink.__set_backend_type(TPaimonWriteBackendType::FFI); + EXPECT_EQ(PaimonBackendType::FFI, PaimonWriteBackendFactory::select_backend_type(sink)); +} + +} // namespace doris diff --git a/build.sh b/build.sh index e0c327b6d0c1aa..69af6f090f0394 100755 --- a/build.sh +++ b/build.sh @@ -604,7 +604,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then modules+=("be-java-extensions/java-common") modules+=("be-java-extensions/java-udf") modules+=("be-java-extensions/jdbc-scanner") - modules+=("be-java-extensions/paimon-scanner") + modules+=("be-java-extensions/paimon-connector") modules+=("be-java-extensions/trino-connector-scanner") modules+=("be-java-extensions/max-compute-connector") modules+=("be-java-extensions/avro-scanner") @@ -971,7 +971,7 @@ EOF extensions_modules=("java-udf") extensions_modules+=("jdbc-scanner") extensions_modules+=("hadoop-hudi-scanner") - extensions_modules+=("paimon-scanner") + extensions_modules+=("paimon-connector") extensions_modules+=("trino-connector-scanner") extensions_modules+=("max-compute-connector") extensions_modules+=("avro-scanner") diff --git a/fe/be-java-extensions/paimon-scanner/pom.xml b/fe/be-java-extensions/paimon-connector/pom.xml similarity index 93% rename from fe/be-java-extensions/paimon-scanner/pom.xml rename to fe/be-java-extensions/paimon-connector/pom.xml index 60ff39c2681b11..4ca2f561312eda 100644 --- a/fe/be-java-extensions/paimon-scanner/pom.xml +++ b/fe/be-java-extensions/paimon-connector/pom.xml @@ -27,7 +27,7 @@ under the License. 4.0.0 - paimon-scanner + paimon-connector 8 @@ -61,10 +61,15 @@ under the License. paimon-format + + org.apache.arrow + arrow-vector + + - paimon-scanner + paimon-connector ${project.basedir}/target/ diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/DorisMemorySegmentPool.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/DorisMemorySegmentPool.java new file mode 100644 index 00000000000000..e917eaaf2b393c --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/DorisMemorySegmentPool.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.memory.AbstractMemorySegmentPool; +import org.apache.paimon.memory.MemorySegment; + +import java.nio.ByteBuffer; + +/** + * Paimon write-buffer pool backed by memory allocated and tracked by Doris BE. + * + *

This class only adapts the Paimon page interface to the BE allocator. The native memory + * manager owns every returned page and releases them after the Java writer has closed. + */ +final class DorisMemorySegmentPool extends AbstractMemorySegmentPool { + private final long nativeMemoryManager; + + DorisMemorySegmentPool(long maxMemory, int pageSize, long nativeMemoryManager) { + super(maxMemory, pageSize); + if (nativeMemoryManager == 0) { + throw new IllegalArgumentException("Doris native memory manager must not be null"); + } + if (maxMemory < pageSize) { + throw new IllegalArgumentException( + "Doris-managed Paimon memory pool must contain at least one page: maxMemory=" + + maxMemory + ", pageSize=" + pageSize); + } + this.nativeMemoryManager = nativeMemoryManager; + } + + @Override + protected MemorySegment allocateMemory() { + ByteBuffer buffer = + PaimonJniWriter.allocatePaimonMemoryPage(nativeMemoryManager, pageSize); + if (buffer == null) { + throw new OutOfMemoryError( + "Doris failed to allocate a native Paimon memory page of " + pageSize + " bytes"); + } + return MemorySegment.wrapOffHeapMemory(buffer); + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/GlobalIndexAssigner.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/GlobalIndexAssigner.java new file mode 100644 index 00000000000000..16d33c6e4678ce --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/GlobalIndexAssigner.java @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.crosspartition.BucketAssigner; +import org.apache.paimon.crosspartition.ExistingProcessor; +import org.apache.paimon.crosspartition.IndexBootstrap; +import org.apache.paimon.crosspartition.KeyPartPartitionKeyExtractor; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.PartitionKeyExtractor; +import org.apache.paimon.table.sink.RowPartitionAllPrimaryKeyExtractor; +import org.apache.paimon.utils.IDMapping; +import org.apache.paimon.utils.PositiveIntInt; +import org.apache.paimon.utils.ProjectToRowFunction; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * Assigns buckets for a key-dynamic table with a process-local global-key index. + * + *

Doris gathers key-dynamic writes into one writer, so a single in-memory index can preserve + * Paimon's cross-partition merge semantics without a native state backend. + */ +final class GlobalIndexAssigner implements AutoCloseable { + private final FileStoreTable table; + // TODO: After resolving rocksdbjni allocator compatibility with the Doris BE jemalloc hook, + // use a RocksDB-backed on-disk index to bound Java heap usage for large tables. + private final Map keyIndex = new HashMap<>(); + + private int bucketIndex; + private int targetBucketRowNumber; + private int assignId; + private int numAssigners; + private boolean bootstrapping; + private BiConsumer collector; + private PartitionKeyExtractor extractor; + private PartitionKeyExtractor bootstrapExtractor; + private IDMapping partitionMapping; + private BucketAssigner bucketAssigner; + private ExistingProcessor existingProcessor; + + GlobalIndexAssigner(FileStoreTable table) { + this.table = table; + } + + void open( + int numAssigners, + int assignId, + BiConsumer collector) { + this.numAssigners = numAssigners; + this.assignId = assignId; + this.collector = collector; + + CoreOptions coreOptions = table.coreOptions(); + this.bucketIndex = IndexBootstrap.bootstrapType(table.schema()).getFieldCount() - 1; + this.targetBucketRowNumber = + checkedTargetBucketRowNumber(coreOptions.dynamicBucketTargetRowNum()); + this.extractor = new RowPartitionAllPrimaryKeyExtractor(table.schema()); + this.bootstrapExtractor = new KeyPartPartitionKeyExtractor(table.schema()); + this.partitionMapping = new IDMapping<>(BinaryRow::copy); + this.bucketAssigner = new BucketAssigner(); + this.existingProcessor = + ExistingProcessor.create( + coreOptions.mergeEngine(), + new ProjectToRowFunction(table.rowType(), table.partitionKeys()), + bucketAssigner, + this::collect); + this.bootstrapping = true; + } + + static int checkedTargetBucketRowNumber(long targetBucketRowNumber) { + if (targetBucketRowNumber <= 0 || targetBucketRowNumber > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "Paimon dynamic-bucket.target-row-num must be between 1 and " + + Integer.MAX_VALUE + ", actual=" + targetBucketRowNumber); + } + return (int) targetBucketRowNumber; + } + + void bootstrapKey(InternalRow value) { + if (!bootstrapping) { + throw new IllegalStateException("Paimon global index bootstrap has finished"); + } + + BinaryRow partition = bootstrapExtractor.partition(value); + BinaryRow key = bootstrapExtractor.trimmedPrimaryKey(value); + int partitionId = partitionMapping.index(partition); + int bucket = value.getInt(bucketIndex); + bucketAssigner.bootstrapBucket(partition, bucket); + PositiveIntInt previous = + keyIndex.putIfAbsent(key.copy(), new PositiveIntInt(partitionId, bucket)); + if (previous != null) { + throw new IllegalStateException( + "Duplicate primary key found while bootstrapping a key-dynamic Paimon table; " + + "the table only supports a single writer"); + } + } + + void finishBootstrap() { + bootstrapping = false; + } + + void processInput(InternalRow value) throws Exception { + if (bootstrapping) { + throw new IllegalStateException("Paimon global index bootstrap is not finished"); + } + + BinaryRow partition = extractor.partition(value); + BinaryRow key = extractor.trimmedPrimaryKey(value); + int partitionId = partitionMapping.index(partition); + PositiveIntInt partitionBucket = keyIndex.get(key); + if (partitionBucket == null) { + processNewRecord(partition, partitionId, key, value); + return; + } + + int previousPartitionId = partitionBucket.i1(); + int previousBucket = partitionBucket.i2(); + if (previousPartitionId == partitionId) { + collect(value, previousBucket); + return; + } + + BinaryRow previousPartition = partitionMapping.get(previousPartitionId); + if (existingProcessor.processExists(value, previousPartition, previousBucket)) { + processNewRecord(partition, partitionId, key, value); + } + } + + private void processNewRecord( + BinaryRow partition, int partitionId, BinaryRow key, InternalRow value) { + int bucket = + bucketAssigner.assignBucket( + partition, this::isAssignedBucket, targetBucketRowNumber); + keyIndex.put(key.copy(), new PositiveIntInt(partitionId, bucket)); + collect(value, bucket); + } + + private boolean isAssignedBucket(int bucket) { + return Math.abs(bucket % numAssigners) == assignId; + } + + private void collect(InternalRow value, int bucket) { + collector.accept(value, bucket); + } + + @Override + public void close() { + keyIndex.clear(); + collector = null; + extractor = null; + bootstrapExtractor = null; + partitionMapping = null; + bucketAssigner = null; + existingProcessor = null; + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonArrowConverter.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonArrowConverter.java new file mode 100644 index 00000000000000..9502f8bb0e2bd2 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonArrowConverter.java @@ -0,0 +1,409 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampVector; +import org.apache.arrow.vector.TinyIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.MapVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.data.variant.GenericVariant; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.VarBinaryType; +import org.apache.paimon.types.VariantType; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Converts Arrow columns into Paimon internal values without owning writer state. */ +final class PaimonArrowConverter { + + private final ZoneId sessionTimeZone; + + PaimonArrowConverter(ZoneId sessionTimeZone) { + this.sessionTimeZone = sessionTimeZone; + } + + RowReader rows(VectorSchemaRoot root, DataType[] targetTypes) { + List fields = root.getSchema().getFields(); + List vectors = root.getFieldVectors(); + if (fields.size() != targetTypes.length) { + throw new IllegalArgumentException("Arrow column count does not match Paimon write type"); + } + return new RowReader(fields, vectors, targetTypes); + } + + /** Bound view of one Arrow batch which converts only the requested row. */ + final class RowReader { + private final List fields; + private final List vectors; + private final DataType[] targetTypes; + + private RowReader( + List fields, List vectors, DataType[] targetTypes) { + this.fields = fields; + this.vectors = vectors; + this.targetTypes = targetTypes; + } + + Object[] values(int rowIndex) { + Object[] values = new Object[vectors.size()]; + for (int column = 0; column < vectors.size(); column++) { + values[column] = convertVectorValue( + vectors.get(column), rowIndex, fields.get(column), targetTypes[column]); + } + return values; + } + } + + private Object convertVectorValue( + FieldVector vector, int index, Field arrowField, DataType targetType) { + if (vector.isNull(index)) { + return null; + } + if (vector instanceof StructVector && targetType instanceof RowType) { + return convertStructVector((StructVector) vector, index, (RowType) targetType); + } + if (vector instanceof MapVector && targetType instanceof MapType) { + return convertMapVector((MapVector) vector, index, (MapType) targetType); + } + if (vector instanceof ListVector && targetType instanceof ArrayType) { + return convertArrayVector((ListVector) vector, index, (ArrayType) targetType); + } + if (vector instanceof IntVector) { + return ((IntVector) vector).get(index); + } + if (vector instanceof BigIntVector) { + return ((BigIntVector) vector).get(index); + } + if (vector instanceof SmallIntVector) { + return ((SmallIntVector) vector).get(index); + } + if (vector instanceof TinyIntVector) { + return ((TinyIntVector) vector).get(index); + } + if (vector instanceof Float4Vector) { + return ((Float4Vector) vector).get(index); + } + if (vector instanceof Float8Vector) { + return ((Float8Vector) vector).get(index); + } + if (vector instanceof BitVector) { + return ((BitVector) vector).get(index) == 1; + } + if (vector instanceof DateDayVector) { + return ((DateDayVector) vector).get(index); + } + if (vector instanceof VarCharVector) { + byte[] value = ((VarCharVector) vector).get(index); + return convertText(value, targetType); + } + if (vector instanceof VarBinaryVector) { + return ((VarBinaryVector) vector).get(index); + } + if (vector instanceof TimeStampVector) { + ArrowType.Timestamp timestampType = (ArrowType.Timestamp) arrowField.getType(); + return toPaimonTimestamp( + arrowTimestampToMicros(((TimeStampVector) vector).get(index), timestampType), + timestampType, targetType); + } + if (vector instanceof DecimalVector) { + DecimalVector decimalVector = (DecimalVector) vector; + int precision = decimalVector.getPrecision(); + int scale = decimalVector.getScale(); + BigDecimal decimal = getBigDecimalFromArrowBuf( + decimalVector.getDataBuffer(), index, scale, DecimalVector.TYPE_WIDTH); + return Decimal.fromBigDecimal(decimal, precision, scale); + } + return convertToPaimonType(vector.getObject(index), arrowField, targetType); + } + + private Object convertToPaimonType(Object value, Field arrowField, DataType targetType) { + if (value == null) { + return null; + } + if (targetType instanceof VariantType) { + if (value instanceof byte[]) { + return toVariant((byte[]) value); + } + if (value instanceof BinaryString) { + return toVariant(((BinaryString) value).toBytes()); + } + if (value instanceof org.apache.arrow.vector.util.Text) { + return toVariant(((org.apache.arrow.vector.util.Text) value).copyBytes()); + } + if (value instanceof org.apache.hadoop.io.Text) { + org.apache.hadoop.io.Text text = (org.apache.hadoop.io.Text) value; + return GenericVariant.fromJson(text.toString()); + } + if (value instanceof CharSequence) { + return GenericVariant.fromJson(value.toString()); + } + throw new IllegalArgumentException( + "Paimon VARIANT requires Arrow UTF-8 JSON, but got " + + value.getClass().getName()); + } + if (targetType instanceof BinaryType || targetType instanceof VarBinaryType) { + if (value instanceof byte[]) { + return value; + } + if (value instanceof BinaryString) { + return ((BinaryString) value).toBytes(); + } + if (value instanceof org.apache.arrow.vector.util.Text) { + return ((org.apache.arrow.vector.util.Text) value).copyBytes(); + } + if (value instanceof String) { + return ((String) value).getBytes(StandardCharsets.UTF_8); + } + return value.toString().getBytes(StandardCharsets.UTF_8); + } + if (value instanceof BinaryString) { + return value; + } + if (value instanceof byte[]) { + return BinaryString.fromBytes((byte[]) value); + } + if (value instanceof org.apache.arrow.vector.util.Text) { + return BinaryString.fromBytes(((org.apache.arrow.vector.util.Text) value).copyBytes()); + } + if (value instanceof org.apache.hadoop.io.Text) { + org.apache.hadoop.io.Text text = (org.apache.hadoop.io.Text) value; + return BinaryString.fromBytes(text.getBytes(), 0, text.getLength()); + } + if (value instanceof CharSequence) { + return BinaryString.fromString(value.toString()); + } + + ArrowType.ArrowTypeID typeId = arrowField == null + ? null : arrowField.getType().getTypeID(); + if (value instanceof LocalDateTime) { + return toPaimonTimestamp((LocalDateTime) value, targetType); + } + if (value instanceof Long && typeId == ArrowType.ArrowTypeID.Timestamp) { + ArrowType.Timestamp timestampType = (ArrowType.Timestamp) arrowField.getType(); + return toPaimonTimestamp( + arrowTimestampToMicros((Long) value, timestampType), timestampType, targetType); + } + if (value instanceof Integer && typeId == ArrowType.ArrowTypeID.Date) { + return value; + } + if (value instanceof java.time.LocalDate) { + return (int) ((java.time.LocalDate) value).toEpochDay(); + } + if (value instanceof BigDecimal) { + BigDecimal decimal = (BigDecimal) value; + return Decimal.fromBigDecimal(decimal, decimal.precision(), decimal.scale()); + } + return value; + } + + static Object convertText(byte[] value, DataType targetType) { + if (targetType instanceof VariantType) { + return toVariant(value); + } + if (targetType instanceof BinaryType || targetType instanceof VarBinaryType) { + return value; + } + return BinaryString.fromBytes(value); + } + + private static GenericVariant toVariant(byte[] json) { + return GenericVariant.fromJson(new String(json, StandardCharsets.UTF_8)); + } + + private GenericRow convertStructVector( + StructVector vector, int index, RowType rowType) { + List childFields = rowType.getFields(); + List childVectors = vector.getChildrenFromFields(); + validateStructVectors(childFields, childVectors); + GenericRow row = new GenericRow(childFields.size()); + for (int i = 0; i < childFields.size(); i++) { + DataField childField = childFields.get(i); + FieldVector childVector = childVectors.get(i); + row.setField(i, convertVectorValue( + childVector, index, childVector.getField(), childField.type())); + } + return row; + } + + private static void validateStructVectors( + List childFields, List childVectors) { + if (childVectors.size() != childFields.size()) { + throw structFieldCountMismatch(childVectors.size(), childFields.size()); + } + for (int i = 0; i < childFields.size(); i++) { + validateStructField(i, childFields.get(i), childVectors.get(i).getName()); + } + } + + static void validateStructSchema(RowType rowType, List arrowFieldNames) { + List childFields = rowType.getFields(); + if (arrowFieldNames.size() != childFields.size()) { + throw structFieldCountMismatch(arrowFieldNames.size(), childFields.size()); + } + for (int i = 0; i < childFields.size(); i++) { + validateStructField(i, childFields.get(i), arrowFieldNames.get(i)); + } + } + + private static IllegalArgumentException structFieldCountMismatch( + int arrowFieldCount, int paimonFieldCount) { + return new IllegalArgumentException( + "Arrow struct field count does not match Paimon row type: arrow=" + + arrowFieldCount + ", paimon=" + paimonFieldCount); + } + + private static void validateStructField( + int position, DataField paimonField, String arrowFieldName) { + if (!arrowFieldName.equalsIgnoreCase(paimonField.name())) { + throw new IllegalArgumentException( + "Arrow struct field at position " + position + " does not match Paimon field " + + paimonField.name() + ": " + arrowFieldName); + } + } + + private GenericMap convertMapVector( + MapVector vector, int index, MapType mapType) { + StructVector entries = (StructVector) vector.getDataVector(); + List entryVectors = entries.getChildrenFromFields(); + if (entryVectors.size() < 2) { + throw new IllegalArgumentException("Arrow map must contain key and value vectors"); + } + FieldVector keyVector = entryVectors.get(0); + FieldVector valueVector = entryVectors.get(1); + int start = vector.getElementStartIndex(index); + int end = vector.getElementEndIndex(index); + Map converted = new HashMap<>(); + for (int entryIndex = start; entryIndex < end; entryIndex++) { + converted.put( + convertVectorValue( + keyVector, entryIndex, keyVector.getField(), mapType.getKeyType()), + convertVectorValue( + valueVector, entryIndex, valueVector.getField(), + mapType.getValueType())); + } + return new GenericMap(converted); + } + + private GenericArray convertArrayVector( + ListVector vector, int index, ArrayType arrayType) { + FieldVector elementVector = vector.getDataVector(); + int start = vector.getElementStartIndex(index); + int end = vector.getElementEndIndex(index); + Object[] converted = new Object[end - start]; + for (int elementIndex = start; elementIndex < end; elementIndex++) { + converted[elementIndex - start] = convertVectorValue( + elementVector, elementIndex, elementVector.getField(), + arrayType.getElementType()); + } + return new GenericArray(converted); + } + + private static BigDecimal getBigDecimalFromArrowBuf( + org.apache.arrow.memory.ArrowBuf buffer, int index, int scale, int byteWidth) { + byte[] value = new byte[byteWidth]; + buffer.getBytes((long) index * byteWidth, value, 0, byteWidth); + if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { + for (int i = 0; i < byteWidth / 2; i++) { + byte temporary = value[i]; + int opposite = byteWidth - 1 - i; + value[i] = value[opposite]; + value[opposite] = temporary; + } + } + return new BigDecimal(new BigInteger(value), scale); + } + + private static long arrowTimestampToMicros( + long value, ArrowType.Timestamp timestampType) { + switch (timestampType.getUnit()) { + case SECOND: + return Math.multiplyExact(value, 1_000_000L); + case MILLISECOND: + return Math.multiplyExact(value, 1_000L); + case MICROSECOND: + return value; + case NANOSECOND: + return Math.floorDiv(value, 1_000L); + default: + throw new IllegalArgumentException( + "Unsupported Arrow timestamp unit: " + timestampType.getUnit()); + } + } + + Timestamp toPaimonTimestamp(long micros, ArrowType.Timestamp arrowType, + DataType targetType) { + String arrowTimeZone = arrowType.getTimezone(); + if (arrowTimeZone != null && !arrowTimeZone.isEmpty()) { + throw new IllegalArgumentException( + "Paimon write timestamp must use a timezone-free Arrow type"); + } + long epochSecond = Math.floorDiv(micros, 1_000_000L); + long microsOfSecond = Math.floorMod(micros, 1_000_000L); + LocalDateTime civilTime = LocalDateTime.ofEpochSecond( + epochSecond, (int) microsOfSecond * 1_000, ZoneOffset.UTC); + return toPaimonTimestamp(civilTime, targetType); + } + + Timestamp toPaimonTimestamp(LocalDateTime civilTime, DataType targetType) { + if (targetType instanceof LocalZonedTimestampType) { + return Timestamp.fromInstant(civilTime.atZone(sessionTimeZone).toInstant()); + } + if (targetType instanceof TimestampType) { + return Timestamp.fromLocalDateTime(civilTime); + } + throw new IllegalArgumentException( + "Arrow timestamp cannot be written to Paimon type " + targetType); + } +} diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java rename to fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonCommitCodec.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonCommitCodec.java new file mode 100644 index 00000000000000..16bbe6eb8bf748 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonCommitCodec.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.io.DataOutputViewStreamWrapper; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageSerializer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Encodes Paimon commit messages into the DPCM (Doris-Paimon Commit Message) payload + * format forwarded to FE. + * + *

DPCM framing format

+ * Each payload is framed as: + * 
+ *   ┌──────────┬─────────────┬────────────┬──────────────────────┐
+ *   │ Magic (4)│ Version (4) │ Length (4) │ Serialized Messages  │
+ *   │  "DPCM"  │  big-endian │ big-endian │     (varies)         │
+ *   └──────────┴─────────────┴────────────┴──────────────────────┘
+ *
+ * + *

Messages are serialized using Paimon's {@link CommitMessageSerializer} and + * split into chunks if the serialized payload exceeds {@link #MAX_PAYLOAD_BYTES} + * (8 MiB). Chunk size starts at {@link #DEFAULT_CHUNK_SIZE} (512 messages) and + * is halved adaptively until each chunk fits within the size limit. + */ +final class PaimonCommitCodec { + static final int HEADER_BYTES = 12; + /** Maximum framed payload size per chunk (8 MiB). */ + static final int MAX_PAYLOAD_BYTES = 8 * 1024 * 1024; + /** Starting number of commit messages per chunk. */ + static final int DEFAULT_CHUNK_SIZE = 512; + + private final CommitMessageSerializer serializer = new CommitMessageSerializer(); + private final int maxPayloadBytes; + private final int defaultChunkSize; + + PaimonCommitCodec() { + this(MAX_PAYLOAD_BYTES, DEFAULT_CHUNK_SIZE); + } + + PaimonCommitCodec(int maxPayloadBytes, int defaultChunkSize) { + if (maxPayloadBytes <= HEADER_BYTES || defaultChunkSize <= 0) { + throw new IllegalArgumentException("Invalid Paimon commit payload limits"); + } + this.maxPayloadBytes = maxPayloadBytes; + this.defaultChunkSize = defaultChunkSize; + } + + /** + * Encode commit messages into DPCM-framed byte chunks. + * + * @param messages Paimon commit messages from {@code prepareCommit()} + * @return byte[][] where each element is a complete DPCM-framed chunk + */ + byte[][] encode(List messages) throws Exception { + if (messages.isEmpty()) { + return new byte[0][]; + } + + // Adaptive chunking uses a size-limited output. An oversized attempt + // therefore stops before allocating beyond one chunk's budget. + int chunkSize = defaultChunkSize; + List payloads = new ArrayList<>(); + int offset = 0; + while (offset < messages.size()) { + int end = Math.min(offset + chunkSize, messages.size()); + byte[] payload; + try { + payload = encodeChunk(messages.subList(offset, end)); + } catch (PayloadTooLargeException e) { + if (chunkSize > 1) { + chunkSize = Math.max(1, chunkSize / 2); + continue; + } + throw new IOException("A single Paimon commit message exceeds the " + + maxPayloadBytes + " byte framed payload limit", e); + } + payloads.add(payload); + offset = end; + } + return payloads.toArray(new byte[0][]); + } + + /** Serialize one chunk of messages and wrap it in a DPCM frame. */ + private byte[] encodeChunk(List messages) throws Exception { + BoundedOutputStream output = new BoundedOutputStream(maxPayloadBytes); + output.write(new byte[HEADER_BYTES]); + serializer.serializeList(messages, new DataOutputViewStreamWrapper(output)); + + byte[] payload = output.toByteArray(); + payload[0] = 'D'; + payload[1] = 'P'; + payload[2] = 'C'; + payload[3] = 'M'; + writeInt(payload, 4, serializer.getVersion()); + writeInt(payload, 8, payload.length - HEADER_BYTES); + return payload; + } + + /** + * Wrap serialized data in a DPCM frame: 4-byte magic "DPCM", 4-byte version + * (big-endian), 4-byte data length (big-endian), followed by the data. + */ + static byte[] frame(byte[] data, int version) { + byte[] payload = new byte[HEADER_BYTES + data.length]; + payload[0] = 'D'; + payload[1] = 'P'; + payload[2] = 'C'; + payload[3] = 'M'; + writeInt(payload, 4, version); + writeInt(payload, 8, data.length); + System.arraycopy(data, 0, payload, HEADER_BYTES, data.length); + return payload; + } + + /** Write a 32-bit integer in big-endian byte order. */ + private static void writeInt(byte[] output, int offset, int value) { + output[offset] = (byte) ((value >>> 24) & 0xFF); + output[offset + 1] = (byte) ((value >>> 16) & 0xFF); + output[offset + 2] = (byte) ((value >>> 8) & 0xFF); + output[offset + 3] = (byte) (value & 0xFF); + } + + private static final class PayloadTooLargeException extends IOException { + private PayloadTooLargeException(int limit) { + super("Paimon commit serialization exceeds " + limit + " bytes"); + } + } + + /** Output stream which fails before Paimon serialization can exceed one framed chunk. */ + private static final class BoundedOutputStream extends OutputStream { + private final ByteArrayOutputStream output; + private final int limit; + + private BoundedOutputStream(int limit) { + this.output = new ByteArrayOutputStream(Math.min(1024, limit)); + this.limit = limit; + } + + private void reserve(int bytes) throws IOException { + if (bytes < 0 || bytes > limit - output.size()) { + throw new PayloadTooLargeException(limit); + } + } + + @Override + public void write(int value) throws IOException { + reserve(1); + output.write(value); + } + + @Override + public void write(byte[] value, int offset, int length) throws IOException { + reserve(length); + output.write(value, offset, length); + } + + private byte[] toByteArray() { + return output.toByteArray(); + } + } +} diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJdbcDriverUtils.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJdbcDriverUtils.java similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJdbcDriverUtils.java rename to fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJdbcDriverUtils.java diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java similarity index 97% rename from fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java rename to fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 3279aa3df746af..d68dbfe9eca316 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -24,11 +24,13 @@ import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache; import com.google.common.base.Preconditions; +import org.apache.paimon.CoreOptions; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.IOManagerImpl; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.Split; @@ -49,6 +51,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -568,6 +571,14 @@ static Optional parseDataSizeBytes(String value) { private void initTable() { Preconditions.checkState(params.containsKey("serialized_table")); table = PaimonUtils.deserialize(params.get("serialized_table")); + // The serialized table may pin an older data snapshot while carrying the latest schema + // after a schema change. Applying a normal copy would time travel to that snapshot's + // schema again and make renamed or newly added columns disappear. + Map readOptions = Collections.singletonMap( + CoreOptions.READ_BATCH_SIZE.key(), String.valueOf(batchSize)); + table = table instanceof FileStoreTable + ? ((FileStoreTable) table).copyWithoutTimeTravel(readOptions) + : table.copy(readOptions); paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType()); if (LOG.isDebugEnabled()) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java new file mode 100644 index 00000000000000..21c5682f638576 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java @@ -0,0 +1,660 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.doris.common.classloader.ThreadClassLoaderContext; +import org.apache.doris.common.security.authentication.PreExecutionAuthenticator; +import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.crosspartition.IndexBootstrap; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.disk.IOManagerImpl; +import org.apache.paimon.index.BucketAssigner; +import org.apache.paimon.index.HashBucketAssigner; +import org.apache.paimon.index.SimpleHashBucketAssigner; +import org.apache.paimon.memory.MemoryPoolFactory; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.InnerTableCommit; +import org.apache.paimon.table.sink.PartitionKeyExtractor; +import org.apache.paimon.table.sink.RowPartitionKeyExtractor; +import org.apache.paimon.table.sink.SinkRecord; +import org.apache.paimon.table.sink.TableWriteImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * JNI entry point for Paimon write operations. + * + *

Called from C++ ({@code JniPaimonWriter}) via JNI. One instance per BE pipeline + * fragment (one per {@code PaimonTableWriter}). Data path: + * + * 

+ *   C++ Block → Arrow IPC Stream → JNI direct ByteBuffer
+ *   → PaimonJniWriter.write(directBuffer)
+ *   → ArrowStreamReader → VectorSchemaRoot
+ *   → PaimonArrowConverter (row-at-a-time typed extraction)
+ *   → PaimonWriteSchema.tableRow() (canonical table-schema order)
+ *   → Paimon SDK bucket assignment and table write
+ *
+ * + *

Commit path: + * + * 

+ *   PaimonTableWriter::close() → JNI → PaimonJniWriter.prepareCommit()
+ *   → TableWriteImpl.prepareCommit()
+ *   → PaimonCommitCodec.encode() → DPCM-framed byte[][]
+ *   → C++ collects TPaimonCommitMessage[] → RPC to FE → PaimonTransaction
+ *
+ */ +public class PaimonJniWriter { + private static final Logger LOG = LoggerFactory.getLogger(PaimonJniWriter.class); + + private final ClassLoader classLoader; + private final PaimonCommitCodec commitCodec = new PaimonCommitCodec(); + + private BufferAllocator allocator; + private PreExecutionAuthenticator preExecutionAuthenticator; + private PaimonArrowConverter arrowConverter; + + private PaimonWriteSchema writeSchema; + private FileStoreTable table; + private TableWriteImpl writer; + private IOManager ioManager; + private long commitIdentifier; + private String commitUser; + private BucketMode bucketMode; + private BucketAssigner hashBucketAssigner; + private PartitionKeyExtractor dynamicBucketExtractor; + private GlobalIndexAssigner globalIndexAssigner; + private boolean fullCompactionChangelog; + private final Set fullCompactionBuckets = new HashSet<>(); + private List preparedCommitMessages = Collections.emptyList(); + private boolean sdkCloseFailed; + + public PaimonJniWriter() { + // TODO: Charge ArrowStreamReader's decoded vectors to the same native manager budget + // used by DorisMemorySegmentPool. A standalone finite RootAllocator would bound Arrow + // itself but would still allow Arrow vectors plus Paimon pages to exceed the advertised + // per-writer/query limit, so this requires shared reserve/release accounting across JNI. + this.allocator = new RootAllocator(Long.MAX_VALUE); + this.classLoader = this.getClass().getClassLoader(); + } + + // ──────────────────────────────────────────────────────────── + // JNI entry points (called from C++) + // ──────────────────────────────────────────────────────────── + + /** + * Initialize the writer. Called once per BE pipeline fragment via JNI. + * + *

This method: + *

    + *
  1. Deserializes the target Paimon {@link FileStoreTable} selected by FE.
    2. + *
  2. Creates a {@link PaimonWriteSchema} which normalizes Doris input + * columns to the table-schema row layout.
    3. + *
  3. Opens one Paimon SDK writer session.
    4. + *
+ * + * @param serializedTable serialized Paimon table selected by FE + * @param hadoopConfig filesystem and authentication configuration + * @param columnNames output column names in the order produced by BE + * @param transactionId Doris external transaction identifier + * @param commitUser Paimon commit user shared with the FE committer + * @param overwrite whether this is an overwrite write + * @param timeZone normalized Doris session timezone used for Paimon LTZ values + * @param spillDirectories Doris storage-root scoped directories for Paimon write-buffer spill + * @param memoryPoolLimitBytes maximum Doris-managed Paimon write-buffer memory + * @param nativeMemoryManager opaque BE manager used to allocate tracked native pages + */ + public void open(String serializedTable, Map hadoopConfig, + String[] columnNames, long transactionId, String commitUser, + boolean overwrite, String timeZone, String spillDirectories, + long memoryPoolLimitBytes, long nativeMemoryManager) throws Exception { + try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) { + if (memoryPoolLimitBytes <= 0) { + throw new IllegalArgumentException( + "PaimonJniWriter requires a positive memory pool limit"); + } + if (nativeMemoryManager == 0) { + throw new IllegalArgumentException( + "PaimonJniWriter requires a native memory manager"); + } + this.preExecutionAuthenticator = PreExecutionAuthenticatorCache.getAuthenticator(hadoopConfig); + this.arrowConverter = new PaimonArrowConverter(ZoneId.of(timeZone)); + preExecutionAuthenticator.execute(() -> { + try { + FileStoreTable table = PaimonUtils.deserialize(serializedTable); + LOG.info("PaimonJniWriter opening: table={}, columns={}", + table.fullName(), columnNames != null ? columnNames.length : 0); + this.commitIdentifier = transactionId; + this.table = table; + this.commitUser = commitUser; + this.bucketMode = table.bucketMode(); + + CoreOptions coreOptions = CoreOptions.fromMap(table.options()); + this.writeSchema = PaimonWriteSchema.create(table.rowType(), columnNames); + validateWriteColumnsForMergeEngine(columnNames.length, coreOptions); + this.fullCompactionChangelog = + !coreOptions.writeOnly() + && coreOptions.changelogProducer() + == CoreOptions.ChangelogProducer.FULL_COMPACTION; + openFileStoreWriter( + table, + commitUser, + overwrite, + spillDirectories, + coreOptions, + memoryPoolLimitBytes, + nativeMemoryManager); + return null; + } catch (Throwable t) { + try { + closeResources(); + } catch (Throwable closeFailure) { + t.addSuppressed(closeFailure); + } + throw new RuntimeException("PaimonJniWriter open failed", t); + } + }); + } + } + + /** + * Write a batch of rows from an Arrow IPC Stream buffer. + * + *

Called from C++ {@code JniPaimonWriter::_write_projected_block()} + * once per Block. The buffer is a zero-copy direct view of the native + * Arrow IPC Stream bytes. Rows are deserialized, normalized to table-schema + * order, and handed to Paimon's writer and bucket assigner APIs. The SDK + * owns partition/bucket semantics, buffering, spill, and file rolling. + * + * @param directBuffer direct view of the native Arrow IPC Stream bytes (no copy) + */ + public void write(ByteBuffer directBuffer) throws Exception { + try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) { + preExecutionAuthenticator.execute(() -> { + try { + try (ArrowStreamReader reader = new ArrowStreamReader( + new DirectBufInputStream(directBuffer), allocator)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + while (reader.loadNextBatch()) { + writeBatch(root); + } + } + return null; + } catch (Throwable t) { + throw new RuntimeException( + "PaimonJniWriter write failed: bytes=" + directBuffer.capacity(), t); + } + }); + } + } + + /** + * Prepare commit: flush all in-memory data, close files, and serialize commit + * messages for the FE coordinator. + * + *

Flushes and collects Paimon {@link CommitMessage}s, then encodes them via + * {@link PaimonCommitCodec} into DPCM-framed byte chunks that are forwarded to + * FE through the BE. + * + * @return byte[][] each element is a DPCM-framed serialized CommitMessage chunk + */ + public byte[][] prepareCommit() throws Exception { + try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) { + return preExecutionAuthenticator.execute(() -> { + try { + List messages = prepareCommitMessages(); + if (messages.isEmpty()) { + LOG.info("PaimonJniWriter prepareCommit: empty"); + return new byte[0][]; + } + LOG.info("PaimonJniWriter prepareCommit: {} messages", messages.size()); + return commitCodec.encode(messages); + } catch (Throwable t) { + throw new RuntimeException("PaimonJniWriter prepareCommit failed", t); + } + }); + } + } + + /** + * Abort: discard all written data files and close the SDK writer. + * Called from C++ when write or prepareCommit fails. + */ + public void abort() throws Exception { + try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) { + try { + if (preExecutionAuthenticator != null) { + preExecutionAuthenticator.execute(() -> { + abortWriter(); + return null; + }); + } else { + abortWriter(); + } + } catch (Exception e) { + LOG.error("PaimonJniWriter abort failed", e); + throw e; + } + } + } + + /** + * Close: release all resources. + */ + public void close() throws Exception { + try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) { + try { + if (preExecutionAuthenticator != null) { + preExecutionAuthenticator.execute(() -> { + closeResources(); + return null; + }); + } else { + closeResources(); + } + } catch (Exception e) { + LOG.warn("PaimonJniWriter close error", e); + throw e; + } + } + } + + // ──────────────────────────────────────────────────────────── + // Initialization helpers + // ──────────────────────────────────────────────────────────── + + private void openFileStoreWriter(FileStoreTable table, String commitUser, boolean overwrite, + String spillDirectories, CoreOptions coreOptions, long memoryPoolLimitBytes, + long nativeMemoryManager) throws Exception { + writer = table.newWrite(commitUser); + if (overwrite) { + writer.withIgnorePreviousFiles(true); + } + openMemoryResources( + coreOptions, spillDirectories, memoryPoolLimitBytes, nativeMemoryManager); + openDynamicBucketAssigner(table, commitUser, overwrite, coreOptions); + } + + private void validateWriteColumnsForMergeEngine(int writeColumnCount, CoreOptions coreOptions) { + if (writeColumnCount == table.rowType().getFieldCount() || table.primaryKeys().isEmpty()) { + return; + } + + CoreOptions.MergeEngine mergeEngine = coreOptions.mergeEngine(); + if (mergeEngine != CoreOptions.MergeEngine.PARTIAL_UPDATE) { + throw new UnsupportedOperationException( + "Paimon primary-key partial-column write requires " + + "merge-engine=partial-update, but table uses merge-engine=" + + mergeEngine); + } + } + + private void openMemoryResources( + CoreOptions coreOptions, + String spillDirectories, + long memoryPoolLimitBytes, + long nativeMemoryManager) throws Exception { + int pageSize = coreOptions.pageSize(); + long effectivePoolLimit = Math.min(coreOptions.writeBufferSize(), memoryPoolLimitBytes); + DorisMemorySegmentPool memorySegmentPool = + new DorisMemorySegmentPool(effectivePoolLimit, pageSize, nativeMemoryManager); + MemoryPoolFactory memoryPoolFactory = new MemoryPoolFactory(memorySegmentPool); + writer.withMemoryPoolFactory(memoryPoolFactory); + LOG.info("Paimon writer uses Doris-managed memory pool: limit={} bytes, pageSize={}", + memoryPoolFactory.totalBufferSize(), pageSize); + + if (!coreOptions.writeBufferSpillable()) { + return; + } + + String[] splitDirectories = IOManagerImpl.splitPaths(spillDirectories); + for (String directory : splitDirectories) { + Files.createDirectories(Paths.get(directory)); + } + ioManager = IOManager.create(splitDirectories); + writer.withIOManager(ioManager); + LOG.info("Paimon writer spill enabled: dirs={}", spillDirectories); + } + + private void openDynamicBucketAssigner(FileStoreTable table, String commitUser, + boolean overwrite, CoreOptions coreOptions) throws Exception { + switch (bucketMode) { + case HASH_DYNAMIC: + openHashDynamicBucketAssigner(table, commitUser, overwrite, coreOptions); + break; + case KEY_DYNAMIC: + openKeyDynamicBucketAssigner(table); + break; + default: + // Fixed, unaware and postpone modes route through TableWrite.write(row). + break; + } + } + + private void openHashDynamicBucketAssigner(FileStoreTable table, String commitUser, + boolean overwrite, CoreOptions coreOptions) { + dynamicBucketExtractor = new RowPartitionKeyExtractor(table.schema()); + if (overwrite) { + hashBucketAssigner = + new SimpleHashBucketAssigner( + 1, + 0, + coreOptions.dynamicBucketTargetRowNum(), + coreOptions.dynamicBucketMaxBuckets()); + return; + } + + hashBucketAssigner = + new HashBucketAssigner( + table.snapshotManager(), + commitUser, + table.store().newIndexFileHandler(), + 1, + 1, + 0, + coreOptions.dynamicBucketTargetRowNum(), + coreOptions.dynamicBucketMaxBuckets()); + } + + private void openKeyDynamicBucketAssigner(FileStoreTable table) throws Exception { + globalIndexAssigner = new GlobalIndexAssigner(table); + globalIndexAssigner.open(1, 0, this::writeAssignedRow); + new IndexBootstrap(table).bootstrap( + 1, 0, this::bootstrapGlobalIndexKey); + globalIndexAssigner.finishBootstrap(); + } + + // ──────────────────────────────────────────────────────────── + // Data writing + // ──────────────────────────────────────────────────────────── + + private void writeBatch(VectorSchemaRoot root) throws Exception { + int rowCount = root.getRowCount(); + if (rowCount == 0) { + return; + } + // Convert and write one row at a time. Keeping only one row of boxed values + // avoids retaining a second, Object[][] representation of the full Arrow batch. + PaimonArrowConverter.RowReader rows = + arrowConverter.rows(root, writeSchema.targetTypes()); + for (int r = 0; r < rowCount; r++) { + InternalRow row = writeSchema.tableRow(rows.values(r)); + switch (bucketMode) { + case HASH_DYNAMIC: + writeHashDynamicRow(row); + break; + case KEY_DYNAMIC: + globalIndexAssigner.processInput(row); + break; + default: + writeRow(row); + break; + } + } + } + + private void writeHashDynamicRow(InternalRow row) throws Exception { + int bucket = + hashBucketAssigner.assign( + dynamicBucketExtractor.partition(row), + dynamicBucketExtractor.trimmedPrimaryKey(row).hashCode()); + writeRow(row, bucket); + } + + private void writeAssignedRow(InternalRow row, Integer bucket) { + try { + writeRow(row, bucket); + } catch (Exception e) { + throw new RuntimeException("Failed to write Paimon key-dynamic bucket row", e); + } + } + + private void bootstrapGlobalIndexKey(InternalRow row) { + try { + globalIndexAssigner.bootstrapKey(row); + } catch (Exception e) { + throw new RuntimeException("Failed to bootstrap Paimon key-dynamic index", e); + } + } + + private void writeRow(InternalRow row) throws Exception { + if (!fullCompactionChangelog) { + writer.write(row); + return; + } + + trackFullCompactionBucket(writer.writeAndReturn(row)); + } + + private void writeRow(InternalRow row, int bucket) throws Exception { + if (!fullCompactionChangelog) { + writer.write(row, bucket); + return; + } + + trackFullCompactionBucket(writer.writeAndReturn(row, bucket)); + } + + private void trackFullCompactionBucket(SinkRecord sinkRecord) { + if (sinkRecord == null) { + return; + } + fullCompactionBuckets.add( + new PartitionBucket( + sinkRecord.partition().copy(), sinkRecord.bucket())); + } + + // ──────────────────────────────────────────────────────────── + // Resource management + // ──────────────────────────────────────────────────────────── + + private void closeResources() throws Exception { + try { + closeWriter(); + } finally { + writeSchema = null; + arrowConverter = null; + if (allocator != null) { + allocator.close(); + allocator = null; + } + } + } + + private List prepareCommitMessages() throws Exception { + if (writer == null) { + throw new IllegalStateException("Paimon writer is not open"); + } + prepareDynamicBucketCommit(); + submitFullCompaction(); + List messages = commitIdentifier > 0 + ? writer.prepareCommit(true, commitIdentifier) + : writer.prepareCommit(); + preparedCommitMessages = new ArrayList<>(messages); + return messages; + } + + private void prepareDynamicBucketCommit() throws Exception { + if (hashBucketAssigner != null) { + hashBucketAssigner.prepareCommit(commitIdentifier); + } + } + + private void submitFullCompaction() throws Exception { + if (!fullCompactionChangelog || fullCompactionBuckets.isEmpty()) { + return; + } + LOG.info("PaimonJniWriter submitting full compaction for {} buckets", + fullCompactionBuckets.size()); + Iterator iterator = fullCompactionBuckets.iterator(); + while (iterator.hasNext()) { + PartitionBucket partitionBucket = iterator.next(); + writer.compact(partitionBucket.partition, partitionBucket.bucket, true); + iterator.remove(); + } + } + + private void closeWriter() throws Exception { + if (sdkCloseFailed) { + throw new IllegalStateException( + "A previous Paimon SDK close failed; native memory cannot be released safely"); + } + Exception failure = closeResource(writer, null); + failure = closeResource(globalIndexAssigner, failure); + failure = closeResource(ioManager, failure); + clearWriterState(); + if (failure != null) { + sdkCloseFailed = true; + throw failure; + } + } + + private static Exception closeResource(AutoCloseable resource, Exception previousFailure) { + if (resource == null) { + return previousFailure; + } + try { + resource.close(); + } catch (Exception closeFailure) { + if (previousFailure == null) { + return closeFailure; + } + previousFailure.addSuppressed(closeFailure); + } + return previousFailure; + } + + private void clearWriterState() { + writer = null; + table = null; + commitIdentifier = 0; + commitUser = null; + bucketMode = null; + hashBucketAssigner = null; + dynamicBucketExtractor = null; + globalIndexAssigner = null; + ioManager = null; + fullCompactionChangelog = false; + fullCompactionBuckets.clear(); + preparedCommitMessages = Collections.emptyList(); + } + + private void abortWriter() throws Exception { + try { + List messages = preparedCommitMessages; + if (messages.isEmpty() && writer != null) { + messages = prepareCommitMessages(); + } + if (!messages.isEmpty()) { + InnerTableCommit committer = table.newCommit(commitUser); + try { + committer.abort(messages); + } finally { + committer.close(); + } + } + } finally { + closeWriter(); + } + } + + // ──────────────────────────────────────────────────────────── + // Utilities + // ──────────────────────────────────────────────────────────── + + static native ByteBuffer allocatePaimonMemoryPage(long nativeMemoryManager, int bytes); + + private static class PartitionBucket { + private final BinaryRow partition; + private final int bucket; + + private PartitionBucket(BinaryRow partition, int bucket) { + this.partition = partition; + this.bucket = bucket; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof PartitionBucket)) { + return false; + } + PartitionBucket that = (PartitionBucket) other; + return bucket == that.bucket && partition.equals(that.partition); + } + + @Override + public int hashCode() { + return Objects.hash(partition, bucket); + } + } + + /** InputStream over a direct ByteBuffer (no copy). */ + private static class DirectBufInputStream extends InputStream { + private final ByteBuffer buf; + + DirectBufInputStream(ByteBuffer buf) { + this.buf = buf; + } + + @Override + public int read() { + if (buf.hasRemaining()) { + return buf.get() & 0xFF; + } + return -1; + } + + @Override + public int read(byte[] b, int off, int len) { + if (!buf.hasRemaining()) { + return -1; + } + int n = Math.min(len, buf.remaining()); + buf.get(b, off, n); + return n; + } + } +} diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java rename to fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonUtils.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonUtils.java similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonUtils.java rename to fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonUtils.java diff --git a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java new file mode 100644 index 00000000000000..127d96adb9e3a7 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.DefaultValueUtils; + +import java.util.Arrays; + +/** + * Immutable mapping from Doris input columns to a Paimon table row. + * + *

The input may contain a subset of table columns in a different order. This class + * resolves their types and table positions once, then converts each input row to the + * table-schema layout expected by the Paimon writer. + */ +final class PaimonWriteSchema { + private final DataType[] targetTypes; + /** Maps Doris input-column position → Paimon table-schema position. */ + private final int[] tableFieldIndexes; + /** Paimon defaults for table fields omitted from the Doris input. */ + private final int[] omittedDefaultFieldIndexes; + private final Object[] omittedDefaultValues; + private final int tableFieldCount; + + private PaimonWriteSchema(DataType[] targetTypes, int[] tableFieldIndexes, + int[] omittedDefaultFieldIndexes, Object[] omittedDefaultValues, int tableFieldCount) { + this.targetTypes = targetTypes; + this.tableFieldIndexes = tableFieldIndexes; + this.omittedDefaultFieldIndexes = omittedDefaultFieldIndexes; + this.omittedDefaultValues = omittedDefaultValues; + this.tableFieldCount = tableFieldCount; + } + + /** + * Create the write schema by resolving {@code columnNames} against the + * Paimon table schema. + * + * @param tableType full Paimon table row type (all columns in table order) + * @param columnNames output column names from BE (in Doris output order) + * @return immutable schema metadata for this writer session + * @throws IllegalArgumentException if any column name is not found in the table schema + */ + static PaimonWriteSchema create(RowType tableType, String[] columnNames) { + if (columnNames == null || columnNames.length == 0) { + throw new IllegalArgumentException( + "PaimonJniWriter requires explicit column names"); + } + + DataType[] targetTypes = new DataType[columnNames.length]; + int[] tableFieldIndexes = new int[columnNames.length]; + boolean[] specifiedFields = new boolean[tableType.getFieldCount()]; + for (int i = 0; i < columnNames.length; i++) { + int tableIndex = tableType.getFieldIndex(columnNames[i]); + if (tableIndex < 0) { + throw new IllegalArgumentException( + "Paimon column '" + columnNames[i] + "' not found in table schema"); + } + if (specifiedFields[tableIndex]) { + throw new IllegalArgumentException( + "Duplicate Paimon write column '" + columnNames[i] + "'"); + } + specifiedFields[tableIndex] = true; + DataField field = tableType.getFields().get(tableIndex); + targetTypes[i] = field.type(); + tableFieldIndexes[i] = tableIndex; + } + + int[] omittedDefaultFieldIndexes = new int[tableType.getFieldCount()]; + Object[] omittedDefaultValues = new Object[tableType.getFieldCount()]; + int omittedDefaultCount = 0; + for (int tableIndex = 0; tableIndex < tableType.getFieldCount(); tableIndex++) { + DataField field = tableType.getFields().get(tableIndex); + if (specifiedFields[tableIndex] || field.defaultValue() == null) { + continue; + } + omittedDefaultFieldIndexes[omittedDefaultCount] = tableIndex; + omittedDefaultValues[omittedDefaultCount] = + DefaultValueUtils.convertDefaultValue(field.type(), field.defaultValue()); + omittedDefaultCount++; + } + + return new PaimonWriteSchema( + targetTypes, + tableFieldIndexes, + Arrays.copyOf(omittedDefaultFieldIndexes, omittedDefaultCount), + Arrays.copyOf(omittedDefaultValues, omittedDefaultCount), + tableType.getFieldCount()); + } + + /** Paimon {@link DataType}s for each write column, in write order. */ + DataType[] targetTypes() { + return targetTypes; + } + + /** Expand one input row to the full Paimon table-schema layout. */ + GenericRow tableRow(Object[] columnValues) { + if (columnValues.length != tableFieldIndexes.length) { + throw new IllegalArgumentException( + "Paimon input value count does not match write schema"); + } + GenericRow row = new GenericRow(tableFieldCount); + for (int i = 0; i < omittedDefaultFieldIndexes.length; i++) { + row.setField(omittedDefaultFieldIndexes[i], omittedDefaultValues[i]); + } + for (int i = 0; i < tableFieldIndexes.length; i++) { + // Actual Doris input is applied last so an explicit NULL remains distinct + // from an omitted field and retains Paimon's writer-side semantics. + row.setField(tableFieldIndexes[i], columnValues[i]); + } + return row; + } +} diff --git a/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml b/fe/be-java-extensions/paimon-connector/src/main/resources/package.xml similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml rename to fe/be-java-extensions/paimon-connector/src/main/resources/package.xml diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/GlobalIndexAssignerTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/GlobalIndexAssignerTest.java new file mode 100644 index 00000000000000..8f4427a8ffe4d1 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/GlobalIndexAssignerTest.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class GlobalIndexAssignerTest { + + @Test + void testCheckedTargetBucketRowNumber() { + Assertions.assertEquals(1, GlobalIndexAssigner.checkedTargetBucketRowNumber(1)); + Assertions.assertEquals( + Integer.MAX_VALUE, + GlobalIndexAssigner.checkedTargetBucketRowNumber(Integer.MAX_VALUE)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> GlobalIndexAssigner.checkedTargetBucketRowNumber(0)); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> GlobalIndexAssigner.checkedTargetBucketRowNumber( + (long) Integer.MAX_VALUE + 1)); + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowConverterTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowConverterTest.java new file mode 100644 index 00000000000000..4c75e9b27a41c2 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowConverterTest.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.data.variant.GenericVariant; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.VariantType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.Arrays; + +public class PaimonArrowConverterTest { + + @Test + public void testTimestampWithoutTimeZonePreservesDstGapWallClock() { + LocalDateTime wallClock = LocalDateTime.parse("2024-03-10T02:30:00.123456"); + long micros = wallClock.toEpochSecond(ZoneOffset.UTC) * 1_000_000L + + wallClock.getNano() / 1_000L; + ArrowType.Timestamp arrowType = new ArrowType.Timestamp( + TimeUnit.MICROSECOND, null); + + PaimonArrowConverter converter = new PaimonArrowConverter( + ZoneId.of("America/Los_Angeles")); + Timestamp result = converter.toPaimonTimestamp( + micros, arrowType, new TimestampType(6)); + + Assertions.assertEquals( + wallClock, result.toLocalDateTime()); + } + + @Test + public void testLocalZonedTimestampPreservesInstant() { + LocalDateTime wallClock = LocalDateTime.parse("2024-01-15T10:30:00.123456"); + long civilMicros = wallClock.toEpochSecond(ZoneOffset.UTC) * 1_000_000L + + wallClock.getNano() / 1_000L; + ArrowType.Timestamp arrowType = new ArrowType.Timestamp( + TimeUnit.MICROSECOND, null); + + PaimonArrowConverter converter = new PaimonArrowConverter( + ZoneId.of("Asia/Shanghai")); + Timestamp result = converter.toPaimonTimestamp( + civilMicros, arrowType, new LocalZonedTimestampType(6)); + + long expectedMicros = wallClock.toEpochSecond(ZoneOffset.ofHours(8)) * 1_000_000L + + wallClock.getNano() / 1_000L; + Assertions.assertEquals(expectedMicros, result.toMicros()); + Assertions.assertEquals(expectedMicros, + converter.toPaimonTimestamp( + wallClock, new LocalZonedTimestampType(6)).toMicros()); + } + + @Test + public void testPaimonWriteRejectsTimezoneInArrowType() { + PaimonArrowConverter converter = new PaimonArrowConverter(ZoneId.of("UTC")); + ArrowType.Timestamp arrowType = new ArrowType.Timestamp( + TimeUnit.MICROSECOND, "Asia/Shanghai"); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> converter.toPaimonTimestamp( + 0, arrowType, new TimestampType(6))); + } + + @Test + public void testVariantJsonKindsUsePaimonVariant() { + String[] jsonValues = { + "\"scalar\"", + "[1,true,null]", + "{\"id\":1,\"name\":\"doris\"}" + }; + for (String json : jsonValues) { + Object value = PaimonArrowConverter.convertText( + json.getBytes(StandardCharsets.UTF_8), new VariantType()); + Assertions.assertInstanceOf(GenericVariant.class, value); + Assertions.assertEquals(json, ((GenericVariant) value).toJson()); + } + } + + @Test + public void testStructSchemaUsesPositionAndAcceptsCaseInsensitiveNames() { + RowType rowType = mixedCaseRowType(); + Assertions.assertDoesNotThrow(() -> PaimonArrowConverter.validateStructSchema( + rowType, Arrays.asList("foo", "FOO"))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> PaimonArrowConverter.validateStructSchema( + rowType, Arrays.asList("foo", "different"))); + } + + private static RowType mixedCaseRowType() { + return DataTypes.ROW( + DataTypes.FIELD(0, "Foo", DataTypes.INT()), + DataTypes.FIELD(1, "foo", DataTypes.INT())); + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonCommitCodecTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonCommitCodecTest.java new file mode 100644 index 00000000000000..6847925e254ce8 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonCommitCodecTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class PaimonCommitCodecTest { + + @Test + public void testFrameContainsMagicVersionAndLength() { + byte[] data = new byte[] {1, 2, 3}; + + byte[] payload = PaimonCommitCodec.frame(data, 7); + + Assertions.assertArrayEquals(new byte[] {'D', 'P', 'C', 'M'}, + new byte[] {payload[0], payload[1], payload[2], payload[3]}); + Assertions.assertEquals(7, ByteBuffer.wrap(payload, 4, 4).getInt()); + Assertions.assertEquals(data.length, ByteBuffer.wrap(payload, 8, 4).getInt()); + Assertions.assertArrayEquals(data, + java.util.Arrays.copyOfRange(payload, PaimonCommitCodec.HEADER_BYTES, payload.length)); + } + + @Test + public void testEncodeEmptyMessages() throws Exception { + PaimonCommitCodec codec = new PaimonCommitCodec(); + + Assertions.assertEquals(0, codec.encode(Collections.emptyList()).length); + } + + @Test + public void testRejectOversizedSinglePayload() { + PaimonCommitCodec codec = new PaimonCommitCodec(1024, 1); + + Exception exception = Assertions.assertThrows( + Exception.class, + () -> codec.encode(Collections.singletonList( + commitMessage("x".repeat(2048))))); + + Assertions.assertTrue(exception.getMessage().contains("exceeds")); + } + + @Test + public void testAdaptiveChunkingStaysWithinFramedLimit() throws Exception { + PaimonCommitCodec codec = new PaimonCommitCodec(1024, 2); + List messages = new ArrayList<>(); + messages.add(commitMessage("x".repeat(400))); + messages.add(commitMessage("y".repeat(400))); + + byte[][] payloads = codec.encode(messages); + + Assertions.assertEquals(2, payloads.length); + Assertions.assertTrue(payloads[0].length <= 1024); + Assertions.assertTrue(payloads[1].length <= 1024); + } + + private static CommitMessage commitMessage(String fileName) { + DataFileMeta dataFile = DataFileMeta.forAppend( + fileName, + 1, + 1, + SimpleStats.EMPTY_STATS, + 0, + 0, + 0, + Collections.emptyList(), + null, + null, + null, + null, + null, + null); + return new CommitMessageImpl( + BinaryRow.EMPTY_ROW, + 0, + 1, + new DataIncrement( + Collections.singletonList(dataFile), + Collections.emptyList(), + Collections.emptyList()), + CompactIncrement.emptyIncrement()); + } +} diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJdbcDriverUtilsTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJdbcDriverUtilsTest.java similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJdbcDriverUtilsTest.java rename to fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJdbcDriverUtilsTest.java diff --git a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java similarity index 100% rename from fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java rename to fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniWriterTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniWriterTest.java new file mode 100644 index 00000000000000..445bf6efc55857 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniWriterTest.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.ByteBuffer; +import java.util.Collections; + +public class PaimonJniWriterTest { + + @Test + public void testManagedMemoryPoolRequiresAtLeastOnePage() { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> new DorisMemorySegmentPool(32 * 1024, 64 * 1024, 1L)); + Assertions.assertTrue(exception.getMessage().contains("at least one page")); + } + + @Test + public void testOpenFailureRestoresContextClassLoader() throws Exception { + Thread thread = Thread.currentThread(); + ClassLoader originalClassLoader = thread.getContextClassLoader(); + URLClassLoader testClassLoader = new URLClassLoader(new URL[0], originalClassLoader); + PaimonJniWriter writer = new PaimonJniWriter(); + thread.setContextClassLoader(testClassLoader); + try { + Assertions.assertThrows(Exception.class, () -> writer.open( + "not-a-serialized-table", Collections.emptyMap(), new String[0], + 1L, "test-user", false, "UTC", System.getProperty("java.io.tmpdir"), + 64L * 1024 * 1024, 1L)); + Assertions.assertSame(testClassLoader, thread.getContextClassLoader()); + } finally { + try { + writer.close(); + Assertions.assertSame(testClassLoader, thread.getContextClassLoader()); + } finally { + thread.setContextClassLoader(originalClassLoader); + testClassLoader.close(); + } + } + } + + @Test + public void testAbortRestoresContextClassLoader() throws Exception { + Thread thread = Thread.currentThread(); + ClassLoader originalClassLoader = thread.getContextClassLoader(); + URLClassLoader testClassLoader = new URLClassLoader(new URL[0], originalClassLoader); + PaimonJniWriter writer = new PaimonJniWriter(); + thread.setContextClassLoader(testClassLoader); + try { + writer.abort(); + Assertions.assertSame(testClassLoader, thread.getContextClassLoader()); + } finally { + try { + writer.close(); + } finally { + thread.setContextClassLoader(originalClassLoader); + testClassLoader.close(); + } + } + } + + @Test + public void testDataEntryPointFailuresRestoreContextClassLoader() throws Exception { + Thread thread = Thread.currentThread(); + ClassLoader originalClassLoader = thread.getContextClassLoader(); + URLClassLoader testClassLoader = new URLClassLoader(new URL[0], originalClassLoader); + PaimonJniWriter writer = new PaimonJniWriter(); + thread.setContextClassLoader(testClassLoader); + try { + Assertions.assertThrows(Exception.class, + () -> writer.write(ByteBuffer.allocateDirect(0))); + Assertions.assertSame(testClassLoader, thread.getContextClassLoader()); + + Assertions.assertThrows(Exception.class, writer::prepareCommit); + Assertions.assertSame(testClassLoader, thread.getContextClassLoader()); + } finally { + thread.setContextClassLoader(originalClassLoader); + testClassLoader.close(); + } + } +} diff --git a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java new file mode 100644 index 00000000000000..e7136ff6960631 --- /dev/null +++ b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.paimon; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class PaimonWriteSchemaTest { + + @Test + public void testReorderedInputProducesTableSchemaRow() { + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType(), + new String[] {"region", "score", "name", "id"}); + Object[] values = new Object[] { + BinaryString.fromString("south"), + 86.5D, + BinaryString.fromString("erin"), + 5 + }; + + InternalRow tableRow = schema.tableRow(values); + + Assertions.assertEquals(5, tableRow.getInt(0)); + Assertions.assertEquals("erin", tableRow.getString(1).toString()); + Assertions.assertEquals(86.5D, tableRow.getDouble(2)); + Assertions.assertEquals("south", tableRow.getString(3).toString()); + } + + @Test + public void testPartialInputLeavesMissingTableFieldsNull() { + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType(), + new String[] {"region", "id"}); + Object[] values = new Object[] { + BinaryString.fromString("east"), + 6 + }; + + InternalRow tableRow = schema.tableRow(values); + + Assertions.assertEquals(6, tableRow.getInt(0)); + Assertions.assertTrue(tableRow.isNullAt(1)); + Assertions.assertTrue(tableRow.isNullAt(2)); + Assertions.assertEquals("east", tableRow.getString(3).toString()); + } + + @Test + public void testUnknownColumnRejectedDuringInitialization() { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> PaimonWriteSchema.create(tableType(), new String[] {"unknown"})); + + Assertions.assertTrue(exception.getMessage().contains("unknown")); + } + + @Test + public void testOmittedNotNullDefaultIsAppliedBeforeTableWriter() { + RowType tableType = new RowType(Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "name", new VarCharType(false, VarCharType.MAX_LENGTH), + null, "unknown"))); + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType, new String[] {"id"}); + + InternalRow tableRow = schema.tableRow(new Object[] {7}); + + Assertions.assertEquals(7, tableRow.getInt(0)); + Assertions.assertEquals("unknown", tableRow.getString(1).toString()); + } + + @Test + public void testOmittedNotNullWithoutDefaultIsLeftForTableWriterValidation() { + RowType tableType = new RowType(Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "name", DataTypes.STRING().notNull()))); + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType, new String[] {"id"}); + + InternalRow tableRow = schema.tableRow(new Object[] {7}); + + Assertions.assertEquals(7, tableRow.getInt(0)); + Assertions.assertTrue(tableRow.isNullAt(1)); + } + + @Test + public void testReorderedInputOverridesDefaultsWhileOmittedFieldUsesDefault() { + RowType tableType = new RowType(Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "name", DataTypes.STRING(), null, "unknown"), + new DataField(2, "score", new DoubleType()), + new DataField(3, "region", DataTypes.STRING(), null, "north"))); + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType, + new String[] {"region", "score", "id"}); + + InternalRow tableRow = schema.tableRow(new Object[] { + BinaryString.fromString("south"), + 92.5D, + 8 + }); + + Assertions.assertEquals(8, tableRow.getInt(0)); + Assertions.assertEquals("unknown", tableRow.getString(1).toString()); + Assertions.assertEquals(92.5D, tableRow.getDouble(2)); + Assertions.assertEquals("south", tableRow.getString(3).toString()); + } + + @Test + public void testExplicitNullIsNotReplacedByOmittedFieldDefault() { + RowType tableType = new RowType(Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "name", DataTypes.STRING(), null, "unknown"))); + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType, + new String[] {"name", "id"}); + + InternalRow tableRow = schema.tableRow(new Object[] { + null, + 9 + }); + + Assertions.assertEquals(9, tableRow.getInt(0)); + Assertions.assertTrue(tableRow.isNullAt(1)); + } + + @Test + public void testOmittedComplexDefaultsUsePaimonInternalValues() { + RowType nestedType = RowType.of(DataTypes.INT(), DataTypes.STRING()); + RowType tableType = new RowType(Arrays.asList( + new DataField(0, "id", DataTypes.INT()), + new DataField(1, "numbers", DataTypes.ARRAY(DataTypes.INT()), null, "[1, 2, 3]"), + new DataField(2, "properties", + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()), null, "{one -> 1, two -> 2}"), + new DataField(3, "nested", nestedType, null, "{42, default-value}"))); + PaimonWriteSchema schema = PaimonWriteSchema.create(tableType, new String[] {"id"}); + + InternalRow tableRow = schema.tableRow(new Object[] {10}); + + InternalArray numbers = tableRow.getArray(1); + Assertions.assertEquals(3, numbers.size()); + Assertions.assertEquals(1, numbers.getInt(0)); + Assertions.assertEquals(3, numbers.getInt(2)); + + InternalMap properties = tableRow.getMap(2); + Assertions.assertEquals(2, properties.size()); + Map actualProperties = new HashMap<>(); + for (int i = 0; i < properties.size(); i++) { + actualProperties.put( + properties.keyArray().getString(i).toString(), + properties.valueArray().getInt(i)); + } + Assertions.assertEquals(1, actualProperties.get("one")); + Assertions.assertEquals(2, actualProperties.get("two")); + + InternalRow nested = tableRow.getRow(3, 2); + Assertions.assertEquals(42, nested.getInt(0)); + Assertions.assertEquals("default-value", nested.getString(1).toString()); + } + + @Test + public void testDuplicateColumnRejectedDuringInitialization() { + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> PaimonWriteSchema.create(tableType(), new String[] {"id", "id"})); + + Assertions.assertTrue(exception.getMessage().contains("Duplicate")); + } + + private static RowType tableType() { + return new RowType(Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "name", new VarCharType()), + new DataField(2, "score", new DoubleType()), + new DataField(3, "region", new VarCharType()))); + } +} diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml index 4d832a0ceeafce..c0b2aaebb166c7 100644 --- a/fe/be-java-extensions/pom.xml +++ b/fe/be-java-extensions/pom.xml @@ -26,7 +26,7 @@ under the License. java-common java-udf jdbc-scanner - paimon-scanner + paimon-connector max-compute-connector avro-scanner diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 48900b0121fba5..62e5ab6ba53ecf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -392,6 +392,10 @@ private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List alterClauses) throws UserException { + if (alterClauses.size() > 1) { + throw new UserException("External table does not support multiple ALTER clauses " + + "in one statement"); + } long updateTime = System.currentTimeMillis(); for (AlterClause alterClause : alterClauses) { if (alterClause instanceof ModifyTablePropertiesClause) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index d4a5c51a50eef9..5893207cf68207 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -148,6 +148,11 @@ public static ZoneId getDorisZoneId() { return getTimeZone().toZoneId(); } + /** Resolve a Doris time-zone name to the canonical ID understood by execution backends. */ + public static String getCanonicalTimeZone(String timeZone) { + return ZoneId.of(timeZone, timeZoneAliasMap).getId(); + } + public static TimeZone getUTCTimeZone() { return TimeZone.getTimeZone(UTC_TIME_ZONE); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java index 2dc8c52165be59..1b7f36d101328f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java @@ -59,7 +59,8 @@ public PaimonSnapshotCacheValue load(NameMapping nameMapping, Table paimonTable) PaimonSnapshot latestSnapshot = resolveLatestSnapshot(paimonTable); List partitionColumns = schemaValueLoader.load(nameMapping, latestSnapshot.getSchemaId()) .getPartitionColumns(); - PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, paimonTable, partitionColumns); + PaimonPartitionInfo partitionInfo = + partitionInfoLoader.load(nameMapping, latestSnapshot.getTable(), partitionColumns); return new PaimonSnapshotCacheValue(partitionInfo, latestSnapshot); } catch (Exception e) { throw new CacheException("failed to load paimon snapshot %s.%s.%s: %s", @@ -69,18 +70,19 @@ public PaimonSnapshotCacheValue load(NameMapping nameMapping, Table paimonTable) } private PaimonSnapshot resolveLatestSnapshot(Table paimonTable) { - Table snapshotTable = paimonTable; + FileStoreTable latestSchemaTable = ((FileStoreTable) paimonTable).copyWithLatestSchema(); + Table snapshotTable = latestSchemaTable; long latestSnapshotId = PaimonSnapshot.INVALID_SNAPSHOT_ID; - Optional optionalSnapshot = paimonTable.latestSnapshot(); + Optional optionalSnapshot = latestSchemaTable.latestSnapshot(); if (optionalSnapshot.isPresent()) { latestSnapshotId = optionalSnapshot.get().id(); - // A schema-only change can be newer than the latest data snapshot. Preserve the - // snapshot pin while exposing the current schema so added columns read as null. - snapshotTable = ((FileStoreTable) paimonTable.copy( - Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(latestSnapshotId)))) - .copyWithLatestSchema(); + // Pin the data snapshot for MVCC while retaining the latest table schema. A normal + // copy applies time travel and falls back to the snapshot's schema, which can be stale + // immediately after a schema change that has not produced a new data snapshot. + snapshotTable = latestSchemaTable.copyWithoutTimeTravel( + Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(latestSnapshotId))); } - DataTable dataTable = (DataTable) paimonTable; + DataTable dataTable = (DataTable) latestSchemaTable; long latestSchemaId = dataTable.schemaManager().latest().map(TableSchema::id).orElse(0L); return new PaimonSnapshot(latestSnapshotId, latestSchemaId, snapshotTable); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonPartitionInfoLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonPartitionInfoLoader.java index c29a359b9592d1..8225802a2c8850 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonPartitionInfoLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonPartitionInfoLoader.java @@ -25,30 +25,28 @@ import org.apache.doris.datasource.paimon.PaimonUtil; import org.apache.commons.collections4.CollectionUtils; -import org.apache.paimon.partition.Partition; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.table.Table; import java.util.List; /** - * Loads partition info for a snapshot projection from the base Paimon table and catalog metadata. + * Loads typed partition metadata from the same snapshot-scoped table used to plan the query. */ public final class PaimonPartitionInfoLoader { - private final PaimonTableLoader tableLoader; - - public PaimonPartitionInfoLoader(PaimonTableLoader tableLoader) { - this.tableLoader = tableLoader; - } - - public PaimonPartitionInfo load(NameMapping nameMapping, Table paimonTable, List partitionColumns) + public PaimonPartitionInfo load(NameMapping nameMapping, Table snapshotTable, List partitionColumns) throws AnalysisException { if (CollectionUtils.isEmpty(partitionColumns)) { return PaimonPartitionInfo.EMPTY; } try { - List paimonPartitions = tableLoader.catalog(nameMapping).getPaimonPartitions(nameMapping); - boolean legacyPartitionName = PaimonUtil.isLegacyPartitionName(paimonTable); - return PaimonUtil.generatePartitionInfo(partitionColumns, paimonPartitions, legacyPartitionName); + // Catalog.listPartitions exposes path-oriented string specs. Paimon intentionally + // maps null and blank strings to the same partition.default-name there, so those + // specs cannot be used as logical partition identities. PartitionEntry retains the + // typed BinaryRow and is also bound to the data snapshot represented by this table. + List partitionEntries = + snapshotTable.newReadBuilder().newScan().listPartitionEntries(); + return PaimonUtil.generatePartitionInfo(snapshotTable, partitionColumns, partitionEntries); } catch (Exception e) { throw new CacheException("failed to load paimon partition info %s.%s.%s: %s", e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java index aad8106563b4f4..a400d55e713504 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/DorisToPaimonTypeVisitor.java @@ -36,7 +36,9 @@ import org.apache.paimon.types.FloatType; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.SmallIntType; import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; import org.apache.paimon.types.VariantType; @@ -83,6 +85,10 @@ public DataType atomic(Type atomic) { PrimitiveType primitiveType = atomic.getPrimitiveType(); if (primitiveType.equals(PrimitiveType.BOOLEAN)) { return new BooleanType(); + } else if (primitiveType.equals(PrimitiveType.TINYINT)) { + return new TinyIntType(); + } else if (primitiveType.equals(PrimitiveType.SMALLINT)) { + return new SmallIntType(); } else if (primitiveType.equals(PrimitiveType.INT)) { return new IntType(); } else if (primitiveType.equals(PrimitiveType.BIGINT)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index a0f489ad065d9a..f4eae370f3cc61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractPaimonProperties; +import org.apache.doris.transaction.TransactionManagerFactory; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.logging.log4j.LogManager; @@ -70,6 +71,7 @@ protected void initLocalObjectsImpl() { catalog = createCatalog(); initPreExecutionAuthenticator(); metadataOps = ExternalMetadataOperations.newPaimonMetaOps(this, catalog); + transactionManager = TransactionManagerFactory.createPaimonTransactionManager((PaimonMetadataOps) metadataOps); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 1d08ba1274e256..81eca359c2c5a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -66,7 +66,7 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor) { super(ENGINE, refreshExecutor); tableLoader = new PaimonTableLoader(); latestSnapshotProjectionLoader = new PaimonLatestSnapshotProjectionLoader( - new PaimonPartitionInfoLoader(tableLoader), this::getPaimonSchemaCacheValue); + new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 6aeacc9058d115..df384d4d4d0071 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -132,6 +132,18 @@ public List getFullSchema(TableScanParams scanParams) { getCatalog().getEnableMappingTimestampTz()); } + /** + * Load the current remote table for a write target. + * + *

A statement MVCC snapshot belongs to a read relation. In a time-travel self-insert the + * same Doris table identity can therefore have a historical source snapshot registered in + * StatementContext. Write planning must never reuse that snapshot: the writer, target schema + * and partition metadata must all come from the latest remote table handle. + */ + public Table getPaimonTableForWrite() { + return ((PaimonExternalCatalog) catalog).getPaimonTable(getOrBuildNameMapping()); + } + private PaimonSnapshotCacheValue getPaimonSnapshotCacheValue(Optional tableSnapshot, Optional scanParams) { makeSureInitialized(); @@ -257,10 +269,12 @@ public Map getAndCopyPartitionItems(Optional snapshot) { - if (isPartitionInvalid(snapshot)) { + PaimonPartitionInfo partitionInfo = getOrFetchSnapshotCacheValue(snapshot).getPartitionInfo(); + if (partitionInfo.getPruningStatus() == PaimonPartitionInfo.PruningStatus.UNPRUNABLE) { return PartitionType.UNPARTITIONED; } - return getPartitionColumns(snapshot).size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED; + return getPaimonSchemaCacheValue(snapshot).getPartitionColumns().isEmpty() + ? PartitionType.UNPARTITIONED : PartitionType.LIST; } @Override @@ -271,17 +285,13 @@ public Set getPartitionColumnNames(Optional snapshot) { @Override public List getPartitionColumns(Optional snapshot) { - if (isPartitionInvalid(snapshot)) { + PaimonPartitionInfo partitionInfo = getOrFetchSnapshotCacheValue(snapshot).getPartitionInfo(); + if (partitionInfo.getPruningStatus() == PaimonPartitionInfo.PruningStatus.UNPRUNABLE) { return Collections.emptyList(); } return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); } - public boolean isPartitionInvalid(Optional snapshot) { - PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); - return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid(); - } - @Override public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, Optional snapshot) @@ -315,9 +325,11 @@ public MTMVSnapshotIf getTableSnapshot(Optional snapshot) throws A @Override public long getNewestUpdateVersionOrTime() { - return getPaimonSnapshotCacheValue(Optional.empty(), Optional.empty()).getPartitionInfo().getNameToPartition() - .values().stream() - .mapToLong(Partition::lastFileCreationTime).max().orElse(0); + // Dictionary loading records getTableSnapshot(), whose version is the Paimon snapshot ID. + // Use the same monotonic version here instead of deriving a timestamp from partition + // metadata. Partition metadata can intentionally be UNPRUNABLE and contain no Doris map. + return getPaimonSnapshotCacheValue(Optional.empty(), Optional.empty()) + .getSnapshot().getSnapshotId(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index bd3245f0f9f431..07f22ef5a54fe1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -17,9 +17,10 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.analysis.ColumnPosition; import org.apache.doris.analysis.PartitionDesc; -import org.apache.doris.catalog.StructField; -import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Type; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; @@ -50,16 +51,21 @@ import org.apache.paimon.catalog.Catalog.TableNotExistException; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; public class PaimonMetadataOps implements ExternalMetadataOps { @@ -213,14 +219,10 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName); } } - List columns = createTableInfo.getColumnDefinitions(); - List collect = columns.stream() - .map(col -> new StructField(col.getName(), col.getType().toCatalogDataType(), - col.getComment(), col.isNullable())) + List columns = createTableInfo.getColumnDefinitions().stream() + .map(ColumnDefinition::translateToCatalogStyle) .collect(Collectors.toList()); - StructType structType = new StructType(new ArrayList<>(collect)); - List rootFieldNames = columns.stream().map(ColumnDefinition::getName).collect(Collectors.toList()); - Schema schema = toPaimonSchema(structType, rootFieldNames, createTableInfo.getPartitionDesc(), + Schema schema = toPaimonSchema(columns, createTableInfo.getPartitionDesc(), createTableInfo.getProperties()); try { catalog.createTable(new Identifier(createTableInfo.getDbName(), createTableInfo.getTableName()), @@ -231,8 +233,7 @@ public boolean performCreateTable(CreateTableInfo createTableInfo) throws UserEx return false; } - private Schema toPaimonSchema(StructType structType, List rootFieldNames, PartitionDesc partitionDesc, - Map properties) { + private Schema toPaimonSchema(List columns, PartitionDesc partitionDesc, Map properties) { Map normalizedProperties = new HashMap<>(properties); normalizedProperties.remove(PRIMARY_KEY_IDENTIFIER); normalizedProperties.remove(PROP_COMMENT); @@ -246,6 +247,7 @@ private Schema toPaimonSchema(StructType structType, List rootFieldNames .map(String::trim) .collect(Collectors.toList()); List partitionKeys = partitionDesc == null ? new ArrayList<>() : partitionDesc.getPartitionColNames(); + List rootFieldNames = columns.stream().map(Column::getName).collect(Collectors.toList()); primaryKeys = getPaimonColumnNames(rootFieldNames, primaryKeys); partitionKeys = getPaimonColumnNames(rootFieldNames, partitionKeys); Schema.Builder schemaBuilder = Schema.newBuilder() @@ -253,12 +255,11 @@ private Schema toPaimonSchema(StructType structType, List rootFieldNames .primaryKey(primaryKeys) .partitionKeys(partitionKeys) .comment(properties.getOrDefault(PROP_COMMENT, null)); - List fields = structType.getFields(); - for (int i = 0; i < fields.size(); i++) { - StructField field = fields.get(i); - schemaBuilder.column(rootFieldNames.get(i), - toPaimontype(field.getType()).copy(field.getContainsNull()), - field.getComment()); + for (Column column : columns) { + schemaBuilder.column(column.getName(), + toPaimonType(column.getType()).copy(column.isAllowNull()), + column.getComment(), + column.getDefaultValue()); } return schemaBuilder.build(); } @@ -271,7 +272,7 @@ private List getPaimonColumnNames(List paimonColumnNames, List loadRemoteFields(ExternalTable dorisTable) throws UserException { + try { + return executionAuthenticator.execute( + () -> new ArrayList<>(catalog.getTable(tableIdentifier(dorisTable)).rowType().getFields())); + } catch (Exception e) { + throw new UserException("Failed to load schema for Paimon table " + dorisTable.getName() + + ": " + ExceptionUtils.getRootCauseMessage(e), e); + } + } + + private Map indexFieldsByDorisName(List fields) throws UserException { + Map fieldsByLowerCase = new HashMap<>(); + for (DataField field : fields) { + DataField previous = fieldsByLowerCase.put(field.name().toLowerCase(Locale.ROOT), field); + if (previous != null) { + throw new UserException("Paimon table contains columns which differ only by case: " + + previous.name() + " and " + field.name()); + } + } + return fieldsByLowerCase; + } + + private DataField resolveRemoteField(Map fieldsByDorisName, String columnName) + throws UserException { + DataField field = fieldsByDorisName.get(columnName.toLowerCase(Locale.ROOT)); + if (field == null) { + throw new UserException("Column " + columnName + " does not exist in Paimon table"); + } + return field; + } + + private DataType toPaimonColumnType(Column column) throws UserException { + try { + return toPaimonType(column.getType()).copy(column.isAllowNull()); + } catch (RuntimeException e) { + throw new UserException("Unsupported Paimon type for column " + column.getName() + + ": " + ExceptionUtils.getRootCauseMessage(e), e); + } + } + + private void registerDorisColumnName(Set columnNames, String columnName) throws UserException { + if (!columnNames.add(columnName.toLowerCase(Locale.ROOT))) { + throw new UserException("Column " + columnName + + " conflicts with an existing Paimon column (case-insensitive)"); + } + } + + private void checkUnsupportedColumnAttributes(Column column) throws UserException { + if (column.isAggregated()) { + throw new UserException("Paimon column does not support aggregation method: " + column.getName()); + } + if (column.isAutoInc()) { + throw new UserException("Paimon column does not support AUTO_INCREMENT: " + column.getName()); + } + if (column.isGeneratedColumn()) { + throw new UserException("Column " + column.getName() + + " cannot be a generated column in a Paimon table"); + } + } + + private void appendAddColumnChanges(List changes, Column column, SchemaChange.Move move) + throws UserException { + changes.add(SchemaChange.addColumn( + column.getName(), toPaimonColumnType(column), column.getComment(), move)); + if (column.getDefaultValue() != null) { + changes.add(SchemaChange.updateColumnDefaultValue( + new String[] {column.getName()}, column.getDefaultValue())); + } + } + + private void alterTable(ExternalTable dorisTable, List changes, String operation, + long updateTime) throws UserException { + if (changes.isEmpty()) { + return; + } + try { + executionAuthenticator.execute(() -> { + catalog.alterTable(tableIdentifier(dorisTable), changes, false); + return null; + }); + } catch (Exception e) { + throw new UserException("Failed to " + operation + " for Paimon table " + dorisTable.getName() + + ": " + ExceptionUtils.getRootCauseMessage(e), e); + } + refreshTable(dorisTable, updateTime); + } + + private void refreshTable(ExternalTable dorisTable, long updateTime) { + Optional> db = dorisCatalog.getDbForReplay(dorisTable.getDbName()); + if (db.isPresent()) { + Optional table = db.get().getTableForReplay(dorisTable.getName()); + if (table.isPresent()) { + Env.getCurrentEnv().getRefreshManager() + .refreshTableInternal(db.get(), (ExternalTable) table.get(), updateTime); + } + } + } + + @Override + public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) + throws UserException { + List fields = loadRemoteFields(dorisTable); + Map fieldsByDorisName = indexFieldsByDorisName(fields); + registerDorisColumnName(new HashSet<>(fieldsByDorisName.keySet()), column.getName()); + checkUnsupportedColumnAttributes(column); + + SchemaChange.Move move = null; + if (position != null) { + move = position.isFirst() + ? SchemaChange.Move.first(column.getName()) + : SchemaChange.Move.after(column.getName(), + resolveRemoteField(fieldsByDorisName, position.getLastCol()).name()); + } + + List changes = new ArrayList<>(); + appendAddColumnChanges(changes, column, move); + alterTable(dorisTable, changes, "add column " + column.getName(), updateTime); + } + + @Override + public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { + Map fieldsByDorisName = indexFieldsByDorisName(loadRemoteFields(dorisTable)); + Set columnNames = new HashSet<>(fieldsByDorisName.keySet()); + List changes = new ArrayList<>(); + for (Column column : columns) { + registerDorisColumnName(columnNames, column.getName()); + checkUnsupportedColumnAttributes(column); + appendAddColumnChanges(changes, column, null); + } + alterTable(dorisTable, changes, "add columns", updateTime); + } + + @Override + public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { + Map fieldsByDorisName = indexFieldsByDorisName(loadRemoteFields(dorisTable)); + String remoteColumnName = resolveRemoteField(fieldsByDorisName, columnName).name(); + alterTable(dorisTable, Collections.singletonList(SchemaChange.dropColumn(remoteColumnName)), + "drop column " + remoteColumnName, updateTime); + } + + @Override + public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) + throws UserException { + Map fieldsByDorisName = indexFieldsByDorisName(loadRemoteFields(dorisTable)); + DataField oldField = resolveRemoteField(fieldsByDorisName, oldName); + DataField conflictingField = fieldsByDorisName.get(newName.toLowerCase(Locale.ROOT)); + if (conflictingField != null && conflictingField != oldField) { + throw new UserException("Column " + newName + + " conflicts with an existing Paimon column (case-insensitive)"); + } + if (oldField.name().equals(newName)) { + return; + } + alterTable(dorisTable, + Collections.singletonList(SchemaChange.renameColumn(oldField.name(), newName)), + "rename column " + oldField.name() + " to " + newName, updateTime); + } + + @Override + public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) + throws UserException { + checkUnsupportedColumnAttributes(column); + Map fieldsByDorisName = indexFieldsByDorisName(loadRemoteFields(dorisTable)); + DataField currentField = resolveRemoteField(fieldsByDorisName, column.getName()); + DataType requestedType = requestedColumnType(column, currentField); + List changes = new ArrayList<>(); + + DataType requestedTypeWithCurrentNullability = + requestedType.copy(currentField.type().isNullable()); + if (!currentField.type().equalsIgnoreFieldId(requestedTypeWithCurrentNullability)) { + changes.add(SchemaChange.updateColumnType( + currentField.name(), requestedTypeWithCurrentNullability, true)); + } + if (currentField.type().isNullable() != requestedType.isNullable()) { + changes.add(SchemaChange.updateColumnNullability( + currentField.name(), requestedType.isNullable())); + } + if (!Objects.equals(currentField.description(), column.getComment())) { + changes.add(SchemaChange.updateColumnComment(currentField.name(), column.getComment())); + } + if (!Objects.equals(currentField.defaultValue(), column.getDefaultValue())) { + changes.add(SchemaChange.updateColumnDefaultValue( + new String[] {currentField.name()}, column.getDefaultValue())); + } + if (position != null) { + SchemaChange.Move move = position.isFirst() + ? SchemaChange.Move.first(currentField.name()) + : SchemaChange.Move.after(currentField.name(), + resolveRemoteField(fieldsByDorisName, position.getLastCol()).name()); + changes.add(SchemaChange.updateColumnPosition(move)); + } + + alterTable(dorisTable, changes, "modify column " + currentField.name(), updateTime); + } + + DataType requestedColumnType(Column column, DataField currentField) + throws UserException { + Type currentDorisType = PaimonUtil.paimonTypeToDorisType( + currentField.type(), + dorisCatalog.getEnableMappingVarbinary(), + dorisCatalog.getEnableMappingTimestampTz()); + // Doris external-table types are a projection of the remote schema. The projection can + // lose Paimon timestamp precision, binary/string length, LTZ identity and nested + // nullability. If ALTER did not change that projected type, retain the exact remote type + // and apply only the independently requested attributes below. + return currentDorisType.equals(column.getType()) + ? currentField.type().copy(column.isAllowNull()) + : toPaimonColumnType(column); + } + + @Override + public void reorderColumns(ExternalTable dorisTable, List newOrder, long updateTime) + throws UserException { + List fields = loadRemoteFields(dorisTable); + Map fieldsByDorisName = indexFieldsByDorisName(fields); + if (newOrder.size() != fields.size()) { + throw new UserException("Reorder columns must contain every Paimon column exactly once"); + } + + List remoteOrder = new ArrayList<>(newOrder.size()); + Set seen = new HashSet<>(); + for (String columnName : newOrder) { + DataField field = resolveRemoteField(fieldsByDorisName, columnName); + if (!seen.add(field.name().toLowerCase(Locale.ROOT))) { + throw new UserException("Duplicate column in reorder columns: " + columnName); + } + remoteOrder.add(field.name()); + } + List currentOrder = fields.stream().map(DataField::name).collect(Collectors.toList()); + if (currentOrder.equals(remoteOrder)) { + return; + } + + List changes = new ArrayList<>(); + changes.add(SchemaChange.updateColumnPosition(SchemaChange.Move.first(remoteOrder.get(0)))); + for (int i = 1; i < remoteOrder.size(); i++) { + changes.add(SchemaChange.updateColumnPosition( + SchemaChange.Move.after(remoteOrder.get(i), remoteOrder.get(i - 1)))); + } + alterTable(dorisTable, changes, "reorder columns", updateTime); + } + public Catalog getCatalog() { return catalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index a6339ef5155e15..207810b66f5a68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -19,24 +19,46 @@ import org.apache.doris.catalog.PartitionItem; -import com.google.common.collect.Maps; import org.apache.paimon.partition.Partition; +import java.util.Collections; import java.util.Map; +/** + * Snapshot-scoped Paimon partition metadata used by Doris. + * + *

The map key is the physical partition name generated by Paimon. If that name cannot + * uniquely represent every typed partition value, the complete result is {@link #UNPRUNABLE}. + */ public class PaimonPartitionInfo { - public static final PaimonPartitionInfo EMPTY = new PaimonPartitionInfo(); + /** + * Whether the complete snapshot-scoped Paimon partition set can be represented as Doris + * PartitionItems. UNPRUNABLE keeps queries correct by leaving scan planning to Paimon. + * + *

This capability state can be removed only after Doris supports every legal Paimon + * partition type and partition conversion is guaranteed not to fail. + */ + public enum PruningStatus { + PRUNABLE, + UNPRUNABLE + } + + public static final PaimonPartitionInfo EMPTY = new PaimonPartitionInfo(PruningStatus.PRUNABLE); + public static final PaimonPartitionInfo UNPRUNABLE = new PaimonPartitionInfo(PruningStatus.UNPRUNABLE); + private final PruningStatus pruningStatus; private final Map nameToPartitionItem; private final Map nameToPartition; - private PaimonPartitionInfo() { - this.nameToPartitionItem = Maps.newHashMap(); - this.nameToPartition = Maps.newHashMap(); + private PaimonPartitionInfo(PruningStatus pruningStatus) { + this.pruningStatus = pruningStatus; + this.nameToPartitionItem = Collections.emptyMap(); + this.nameToPartition = Collections.emptyMap(); } public PaimonPartitionInfo(Map nameToPartitionItem, Map nameToPartition) { + this.pruningStatus = PruningStatus.PRUNABLE; this.nameToPartitionItem = nameToPartitionItem; this.nameToPartition = nameToPartition; } @@ -49,8 +71,7 @@ public Map getNameToPartition() { return nameToPartition; } - public boolean isPartitionInvalid() { - // when transfer to partitionItem failed, will not equal - return nameToPartitionItem.size() != nameToPartition.size(); + public PruningStatus getPruningStatus() { + return pruningStatus; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTransaction.java new file mode 100644 index 00000000000000..b2416b853e3aac --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTransaction.java @@ -0,0 +1,460 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.UserException; +import org.apache.doris.thrift.TPaimonCommitMessage; +import org.apache.doris.transaction.Transaction; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageSerializer; +import org.apache.paimon.table.sink.InnerTableCommit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Paimon transaction. + * + * Lifecycle: + * 1. bind() — pin the concrete table and writer configuration + * 2. updateCommitMessages() — called multiple times as BE reports commit data + * 3. commit() — deserialize all CommitMessages, call StreamTableCommit.filterAndCommit() + * 4. rollback() — abort uncommitted data files + * + * CommitMessage wire format: + * BE ← JNI ← Java: byte[] with DPCM header (magic + version + length) + Paimon + * CommitMessageSerializer payload + */ +public class PaimonTransaction implements Transaction { + private static final Logger LOG = LogManager.getLogger(PaimonTransaction.class); + + enum CommitState { + PREPARED, + COMMITTING, + COMMITTED, + OUTCOME_UNKNOWN + } + + private static final int COMMIT_HEADER_SIZE = 12; + private static final byte[] COMMIT_MAGIC = new byte[] {'D', 'P', 'C', 'M'}; + + private final PaimonMetadataOps ops; + private final long transactionId; + private final String commitUser; + private PaimonWriteBinding binding; + private CommitState state = CommitState.PREPARED; + + private final List commitPayloads = Lists.newArrayList(); + private final Set commitPayloadSet = new HashSet<>(); + + public PaimonTransaction(PaimonMetadataOps ops, long transactionId) { + this.ops = Preconditions.checkNotNull(ops, "Paimon metadata ops must not be null"); + Preconditions.checkArgument(transactionId > 0, "Paimon transaction id must be positive"); + this.transactionId = transactionId; + this.commitUser = commitUser(transactionId); + } + + // ──────────────────────────────────────────────────────────── + // Transaction lifecycle + // ──────────────────────────────────────────────────────────── + + public synchronized void bind(PaimonWriteBinding binding) { + Preconditions.checkNotNull(binding, "Paimon write binding must not be null"); + Preconditions.checkState(this.binding == null, "Paimon transaction is already bound"); + Preconditions.checkState(state == CommitState.PREPARED, + "Paimon transaction can only be bound while prepared"); + this.binding = binding; + } + + @Override + public void commit() throws UserException { + PaimonWriteBinding writeBinding = requireBinding(); + List rawPayloads = snapshotPayloads(); + if (rawPayloads.isEmpty() && !writeBinding.isOverwrite()) { + LOG.info("Skip empty PaimonTransaction commit, txnId={}, table={}", + transactionId, tableName()); + markPreparedTransactionCommitted(); + return; + } + try { + List allMessages = deserializePayloads(rawPayloads); + LOG.info("Commit PaimonTransaction, txnId={}, table={}, payloads={}, messages={}, overwrite={}", + transactionId, tableName(), rawPayloads.size(), allMessages.size(), + writeBinding.isOverwrite()); + if (allMessages.isEmpty() && !writeBinding.isOverwrite()) { + throw new RuntimeException( + "Paimon commit messages are empty, payloads=" + rawPayloads.size()); + } + doCommitWithReconciliation(writeBinding, allMessages); + } catch (Exception e) { + throw new UserException("Failed to commit paimon transaction on FE", e); + } + } + + @Override + public void rollback() { + CommitState currentState = getState(); + if (currentState == CommitState.COMMITTED) { + LOG.info("Skip rollback for committed PaimonTransaction, txnId={}, table={}", + transactionId, tableName()); + return; + } + if (currentState == CommitState.COMMITTING + || currentState == CommitState.OUTCOME_UNKNOWN) { + LOG.warn("Skip rollback for PaimonTransaction in state {}, txnId={}, table={}. " + + "Preserving data files for snapshot safety", + currentState, transactionId, tableName()); + return; + } + List rawPayloads = snapshotPayloads(); + if (rawPayloads.isEmpty()) { + LOG.info("Skip empty PaimonTransaction rollback, txnId={}, table={}", + transactionId, tableName()); + return; + } + try { + PaimonWriteBinding writeBinding = requireBinding(); + List allMessages = deserializePayloads(rawPayloads); + if (allMessages.isEmpty()) { + LOG.info("Skip PaimonTransaction rollback with empty decoded messages, " + + "txnId={}, table={}", transactionId, tableName()); + return; + } + LOG.info("Rollback PaimonTransaction, txnId={}, table={}, payloads={}, messages={}", + transactionId, tableName(), rawPayloads.size(), allMessages.size()); + doAbort(writeBinding, allMessages); + } catch (Exception e) { + LOG.warn("Failed to rollback PaimonTransaction, txnId={}, table={}", + transactionId, tableName(), e); + } + } + + // ──────────────────────────────────────────────────────────── + // CommitMessage collection (called from Coordinator) + // ──────────────────────────────────────────────────────────── + + public void updateCommitMessages(List messages) { + if (messages == null || messages.isEmpty()) { + return; + } + synchronized (this) { + for (TPaimonCommitMessage msg : messages) { + addPayload(msg); + } + } + } + + private void addPayload(TPaimonCommitMessage message) { + if (message == null || !message.isSetPayload()) { + return; + } + byte[] payload = message.getPayload(); + if (payload == null || payload.length == 0) { + return; + } + // Treat the Thrift payload as immutable after report handling and adopt it directly. The + // report is not reused, so copying it would only create a second full representation. + if (commitPayloadSet.add(new CommitPayloadKey(payload))) { + commitPayloads.add(payload); + } + } + + // ──────────────────────────────────────────────────────────── + // Transaction identity + // ──────────────────────────────────────────────────────────── + + /** + * Build the stable Paimon commit identity for one Doris transaction. + * + *

Paimon identifiers are ordered within one commit user. Doris transactions can finish + * out of order, so each transaction remains a separate user; the persisted Doris cluster ID + * prevents another cluster with the same local transaction ID from being treated as a retry. + */ + public static String commitUser(long transactionId) { + return commitUser(Env.getCurrentEnv().getClusterId(), transactionId); + } + + static String commitUser(int clusterId, long transactionId) { + return "doris_cluster_" + clusterId + "_txn_" + transactionId; + } + + public String getCommitUser() { + return commitUser; + } + + public long getTransactionId() { + return transactionId; + } + + synchronized CommitState getState() { + return state; + } + + int getPayloadCount() { + synchronized (this) { + return commitPayloads.size(); + } + } + + // ──────────────────────────────────────────────────────────── + // Internal commit logic + // ──────────────────────────────────────────────────────────── + + private void doCommit(PaimonWriteBinding writeBinding, List messages) + throws Exception { + ops.dorisCatalog.getExecutionAuthenticator().execute(() -> { + InnerTableCommit committer = getCommitTable(writeBinding).newCommit(commitUser); + Exception commitFailure = null; + try { + if (writeBinding.isOverwrite()) { + committer.withOverwrite(writeBinding.getStaticPartition()); + } + Map> commitMap = new HashMap<>(); + commitMap.put(transactionId, messages); + markCommitting(); + committer.filterAndCommit(commitMap); + markCommitted(); + return null; + } catch (Exception e) { + commitFailure = e; + throw e; + } finally { + try { + committer.close(); + } catch (Exception closeFailure) { + if (getState() == CommitState.COMMITTED) { + LOG.warn("Ignore Paimon committer close failure after a successful commit, " + + "txnId={}, table={}", + transactionId, tableName(), closeFailure); + } else if (commitFailure != null) { + commitFailure.addSuppressed(closeFailure); + } else { + throw closeFailure; + } + } + } + }); + } + + private void doCommitWithReconciliation(PaimonWriteBinding writeBinding, + List messages) throws Exception { + Exception firstFailure; + try { + doCommit(writeBinding, messages); + return; + } catch (Exception e) { + if (getState() == CommitState.COMMITTED) { + LOG.warn("Paimon commit completed but post-commit processing failed, " + + "txnId={}, table={}", + transactionId, tableName(), e); + return; + } + if (getState() == CommitState.PREPARED) { + throw e; + } + firstFailure = e; + } + + LOG.warn("Paimon atomic commit failed with an unknown outcome; retrying idempotently, " + + "txnId={}, table={}", transactionId, tableName(), firstFailure); + Exception retryFailure; + try { + doCommit(writeBinding, messages); + return; + } catch (Exception e) { + if (getState() == CommitState.COMMITTED) { + LOG.warn("Paimon retry completed but post-commit processing failed, " + + "txnId={}, table={}", + transactionId, tableName(), e); + return; + } + retryFailure = e; + } + + try { + if (ops.dorisCatalog.getExecutionAuthenticator().execute(() -> !writeBinding.getTable() + .snapshotManager().findSnapshotsForIdentifiers( + commitUser, Collections.singletonList(transactionId)).isEmpty())) { + markCommitted(); + LOG.info("Reconciled Paimon transaction as committed, txnId={}, table={}", + transactionId, tableName()); + return; + } + } catch (Exception reconciliationFailure) { + retryFailure.addSuppressed(reconciliationFailure); + } + retryFailure.addSuppressed(firstFailure); + markOutcomeUnknown(); + throw retryFailure; + } + + private void doAbort(PaimonWriteBinding writeBinding, List messages) + throws Exception { + ops.dorisCatalog.getExecutionAuthenticator().execute(() -> { + InnerTableCommit committer = writeBinding.getTable().newCommit(commitUser); + try { + committer.abort(messages); + return null; + } finally { + committer.close(); + } + }); + } + + private FileStoreTable getCommitTable(PaimonWriteBinding writeBinding) { + FileStoreTable paimonTable = writeBinding.getTable(); + if (!writeBinding.isOverwrite() || writeBinding.getStaticPartition().isEmpty()) { + return paimonTable; + } + return paimonTable.copy(Collections.singletonMap( + CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key(), Boolean.FALSE.toString())); + } + + private synchronized void markPreparedTransactionCommitted() { + Preconditions.checkState(state == CommitState.PREPARED, + "Only a prepared Paimon transaction can complete without a commit"); + state = CommitState.COMMITTED; + } + + private synchronized void markCommitting() { + Preconditions.checkState(state == CommitState.PREPARED || state == CommitState.COMMITTING, + "Cannot enter Paimon commit from state " + state); + state = CommitState.COMMITTING; + } + + private synchronized void markCommitted() { + Preconditions.checkState(state == CommitState.COMMITTING, + "Only a committing Paimon transaction can be committed"); + state = CommitState.COMMITTED; + } + + private synchronized void markOutcomeUnknown() { + Preconditions.checkState(state == CommitState.COMMITTING, + "Only a committing Paimon transaction can have an unknown outcome"); + state = CommitState.OUTCOME_UNKNOWN; + } + + private synchronized PaimonWriteBinding requireBinding() throws UserException { + if (binding == null) { + throw new UserException("Missing Paimon write binding for transaction " + transactionId); + } + return binding; + } + + // ──────────────────────────────────────────────────────────── + // Serialization helpers + // ──────────────────────────────────────────────────────────── + + private List snapshotPayloads() { + synchronized (this) { + return new ArrayList<>(commitPayloads); + } + } + + private static final class CommitPayloadKey { + private final byte[] payload; + private final int hashCode; + + private CommitPayloadKey(byte[] payload) { + this.payload = payload; + this.hashCode = Arrays.hashCode(payload); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CommitPayloadKey)) { + return false; + } + CommitPayloadKey other = (CommitPayloadKey) obj; + return Arrays.equals(payload, other.payload); + } + + @Override + public int hashCode() { + return hashCode; + } + } + + static List deserializePayloads(List payloads) throws IOException { + List all = new ArrayList<>(); + for (byte[] payload : payloads) { + all.addAll(deserializePayload(payload)); + } + return all; + } + + static List deserializePayload(byte[] payload) throws IOException { + if (payload == null || payload.length < COMMIT_HEADER_SIZE || !hasMagic(payload)) { + throw new IOException("Invalid paimon commit message payload header"); + } + int version = readInt(payload, 4); + int len = readInt(payload, 8); + if (len < 0 || payload.length != COMMIT_HEADER_SIZE + len) { + throw new IOException("Invalid paimon commit message payload length"); + } + byte[] raw = new byte[len]; + System.arraycopy(payload, COMMIT_HEADER_SIZE, raw, 0, len); + List messages = + new CommitMessageSerializer().deserializeList(version, new DataInputDeserializer(raw)); + if (messages == null) { + throw new IOException("Paimon commit message payload deserialized to null"); + } + return messages; + } + + private static boolean hasMagic(byte[] payload) { + for (int i = 0; i < COMMIT_MAGIC.length; i++) { + if (payload[i] != COMMIT_MAGIC[i]) { + return false; + } + } + return true; + } + + private static int readInt(byte[] payload, int offset) { + return ((payload[offset] & 0xFF) << 24) + | ((payload[offset + 1] & 0xFF) << 16) + | ((payload[offset + 2] & 0xFF) << 8) + | (payload[offset + 3] & 0xFF); + } + + private String tableName() { + synchronized (this) { + return binding == null ? "unbound" : binding.tableName(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 0ec548992bd129..c0fda16413dc03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Column; @@ -45,12 +46,14 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.io.DataOutputViewStreamWrapper; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.partition.Partition; import org.apache.paimon.predicate.Predicate; @@ -68,13 +71,16 @@ import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; import org.apache.paimon.types.VarBinaryType; import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.PartitionPathUtils; import org.apache.paimon.utils.Projection; @@ -103,7 +109,6 @@ public class PaimonUtil { private static final Logger LOG = LogManager.getLogger(PaimonUtil.class); private static final Base64.Encoder BASE64_ENCODER = java.util.Base64.getUrlEncoder().withoutPadding(); private static final Pattern DIGITAL_REGEX = Pattern.compile("\\d+"); - private static final String PARTITION_LEGACY_NAME = "partition.legacy-name"; private static final String SYS_TABLE_TYPE_AUDIT_LOG = "audit_log"; private static final String SYS_TABLE_TYPE_BINLOG = "binlog"; private static final String TABLE_READ_SEQUENCE_NUMBER_ENABLED = "table-read.sequence-number.enabled"; @@ -112,14 +117,6 @@ public static boolean isDigitalString(String value) { return value != null && DIGITAL_REGEX.matcher(value).matches(); } - /** - * Extract the legacy partition name configuration from Paimon table options. - */ - public static boolean isLegacyPartitionName(Table paimonTable) { - return Boolean.parseBoolean( - paimonTable.options().getOrDefault(PARTITION_LEGACY_NAME, "true")); - } - public static List read( Table table, @Nullable int[] projection, @Nullable Predicate predicate, Pair, String>... dynamicOptions) @@ -150,58 +147,120 @@ public static List read( return rows; } - public static PaimonPartitionInfo generatePartitionInfo(List partitionColumns, - List paimonPartitions, boolean legacyPartitionName) { + public static PaimonPartitionInfo generatePartitionInfo(Table table, List partitionColumns, + List partitionEntries) { - if (CollectionUtils.isEmpty(partitionColumns) || paimonPartitions.isEmpty()) { + if (CollectionUtils.isEmpty(partitionColumns) || partitionEntries.isEmpty()) { return PaimonPartitionInfo.EMPTY; } - Map nameToPartitionItem = Maps.newHashMap(); - Map nameToPartition = Maps.newHashMap(); - PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); + CoreOptions options = new CoreOptions(table.options()); + RowType partitionType = table.rowType().project(table.partitionKeys()); + if (partitionType.getFields().stream().anyMatch(field -> + field.type().getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { + // This metadata is cached by table snapshot, but LTZ values are represented as + // session-local civil times in Doris. Caching those bounds would let one session + // reuse pruning metadata produced in another time zone. Keep scan correctness by + // delegating pruning to Paimon until this cache can carry a time-zone-independent + // typed representation. + return PaimonPartitionInfo.UNPRUNABLE; + } + InternalRowPartitionComputer partitionComputer = new InternalRowPartitionComputer( + options.partitionDefaultName(), + partitionType, + table.partitionKeys().toArray(new String[0]), + options.legacyPartitionName()); List types = partitionColumns.stream() .map(Column::getType) .collect(Collectors.toList()); + List candidates = Lists.newArrayListWithExpectedSize(partitionEntries.size()); + Map> displayNameToTypedSpec = Maps.newHashMap(); + + for (PartitionEntry partitionEntry : partitionEntries) { + Map typedSpec = getPartitionInfoMap( + table, partitionEntry.partition(), TimeUtils.getTimeZone().getID()); + if (typedSpec == null) { + return PaimonPartitionInfo.UNPRUNABLE; + } - for (Partition partition : paimonPartitions) { - Map spec = partition.spec(); - // Paimon partition specs contain logical values, which may include path separators. - // Build partition values directly instead of parsing them as a Hive partition path. List partitionValues = Lists.newArrayListWithExpectedSize(partitionColumns.size()); - LinkedHashMap orderedPartitionSpec = new LinkedHashMap<>(); + LinkedHashMap orderedTypedSpec = new LinkedHashMap<>(); for (Column partitionColumn : partitionColumns) { String partitionColumnName = partitionColumn.getName(); - String partitionValue = spec.get(partitionColumnName); - // When partition.legacy-name = true (default), Paimon stores DATE type as days since - // 1970-01-01 (epoch integer), so we need to convert the integer to a date string. - // When partition.legacy-name = false, the value is already a human read date string. - if (legacyPartitionName && partitionColumn.getType().isDateV2()) { - partitionValue = DateTimeUtils.formatDate(Integer.parseInt(partitionValue)); - } + Preconditions.checkState(typedSpec.containsKey(partitionColumnName), + "Partition column not found in Paimon typed spec: " + partitionColumnName); + String partitionValue = typedSpec.get(partitionColumnName); partitionValues.add(partitionValue); - orderedPartitionSpec.put(partitionColumnName, partitionValue); + orderedTypedSpec.put(partitionColumnName, partitionValue); } - String partitionPath = PartitionPathUtils.generatePartitionPath(orderedPartitionSpec); - String partitionName = partitionPath.substring(0, partitionPath.length() - 1); - Partition previousPartition = nameToPartition.putIfAbsent(partitionName, partition); - Preconditions.checkState(previousPartition == null, - "Duplicate Paimon partition name: " + partitionName); + PartitionItem partitionItem; try { - // partition values return by paimon api, may have problem, - // to avoid affecting the query, we catch exceptions here partitionItem = toListPartitionItem(partitionValues, types); } catch (Exception e) { LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", - partitionColumns, partition.spec(), e); - continue; + partitionColumns, partitionValues, e); + return PaimonPartitionInfo.UNPRUNABLE; } - PartitionItem previousPartitionItem = nameToPartitionItem.putIfAbsent(partitionName, partitionItem); - Preconditions.checkState(previousPartitionItem == null, - "Duplicate Paimon partition item name: " + partitionName); + + LinkedHashMap displaySpec; + try { + // Delegate display-name generation to Paimon so partition.default-name and + // partition.legacy-name exactly follow the table's physical partition naming. + // The canonical typed spec above remains the logical identity used for pruning. + displaySpec = partitionComputer.generatePartValues(partitionEntry.partition()); + } catch (Exception e) { + LOG.warn("Failed to generate Paimon partition display name, table: {}, partition: {}", + table.name(), orderedTypedSpec, e); + return PaimonPartitionInfo.UNPRUNABLE; + } + String partitionPath = PartitionPathUtils.generatePartitionPath(displaySpec); + String displayName = partitionPath.substring(0, partitionPath.length() - 1); + Map previousTypedSpec = displayNameToTypedSpec.putIfAbsent( + displayName, orderedTypedSpec); + if (previousTypedSpec != null) { + Preconditions.checkState(!previousTypedSpec.equals(orderedTypedSpec), + "Duplicate typed Paimon partition: " + displayName); + // Doris partition metadata and downstream consumers such as MTMV require a + // stable one-to-one mapping between a partition name and its typed value. + // Paimon may map distinct values (for example null and blank strings) to the + // same physical partition name. A private suffix would only make the map key + // unique; it would be lost when consumers reconstruct a name from PartitionItem. + // Keep the complete mapping all-or-nothing and delegate pruning to Paimon. + LOG.warn("Ambiguous Paimon partition display name {}, typed specs: {} and {}; " + + "disable Doris partition pruning", + displayName, previousTypedSpec, orderedTypedSpec); + return PaimonPartitionInfo.UNPRUNABLE; + } + candidates.add(new PaimonPartitionCandidate( + partitionEntry, orderedTypedSpec, partitionItem, displayName)); + } + + Map nameToPartitionItem = Maps.newHashMap(); + Map nameToPartition = Maps.newHashMap(); + for (PaimonPartitionCandidate candidate : candidates) { + PartitionEntry entry = candidate.partitionEntry; + Partition partition = new Partition(candidate.typedSpec, entry.recordCount(), + entry.fileSizeInBytes(), entry.fileCount(), entry.lastFileCreationTime(), false); + nameToPartitionItem.put(candidate.displayName, candidate.partitionItem); + nameToPartition.put(candidate.displayName, partition); + } + return new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); + } + + private static final class PaimonPartitionCandidate { + private final PartitionEntry partitionEntry; + private final Map typedSpec; + private final PartitionItem partitionItem; + private final String displayName; + + private PaimonPartitionCandidate(PartitionEntry partitionEntry, Map typedSpec, + PartitionItem partitionItem, String displayName) { + this.partitionEntry = partitionEntry; + this.typedSpec = typedSpec; + this.partitionItem = partitionItem; + this.displayName = displayName; } - return partitionInfo; } public static ListPartitionItem toListPartitionItem(List partitionValues, List types) @@ -209,12 +268,8 @@ public static ListPartitionItem toListPartitionItem(List partitionValues Preconditions.checkState(partitionValues.size() == types.size(), partitionValues + " vs. " + types); List values = Lists.newArrayListWithExpectedSize(types.size()); for (String partitionValue : partitionValues) { - // null will in partition 'null' - // "null" will in partition 'null' - // NULL will in partition 'null' - // "NULL" will in partition 'NULL' - // values.add(new PartitionValue(partitionValue, "null".equals(partitionValue))); - values.add(new PartitionValue(partitionValue, false)); + // Keep a typed null distinct from an empty string and the literal string "null". + values.add(new PartitionValue(partitionValue, partitionValue == null)); } PartitionKey key = PartitionKey.createListPartitionKeyWithTypes(values, types, true); ListPartitionItem listPartitionItem = new ListPartitionItem(Lists.newArrayList(key)); @@ -305,6 +360,8 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT .map(field -> new org.apache.doris.catalog.StructField(field.name(), paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping))) .collect(Collectors.toCollection(ArrayList::new))); + case VARIANT: + return Type.VARIANT; case TIME_WITHOUT_TIME_ZONE: return Type.UNSUPPORTED; default: @@ -650,8 +707,13 @@ private static String serializePartitionValue(org.apache.paimon.types.DataType t if (value == null) { return null; } - // Paimon timestamp is stored as Timestamp type in utc - return ((Timestamp) value).toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + // Format through Doris' target type instead of translating between Paimon's + // timestamp text and Doris' partition-literal syntax by hand. + TimestampType timestampType = (TimestampType) type; + ScalarType dorisType = ScalarType.createDatetimeV2Type( + Math.min(timestampType.getPrecision(), 6)); + return new DateLiteral(((Timestamp) value).toLocalDateTime(), dorisType) + .getStringValue(); case TIMESTAMP_WITH_LOCAL_TIME_ZONE: if (value == null) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonWriteBinding.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonWriteBinding.java new file mode 100644 index 00000000000000..cddb2864a70443 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonWriteBinding.java @@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.plans.commands.insert.PaimonInsertCommandContext; +import org.apache.doris.nereids.types.DataType; + +import com.google.common.base.Preconditions; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypeRoot; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * Transaction-scoped binding between one Doris insert and one pinned Paimon write target. + * + *

Schema analysis and writer distribution have already used the target's table handle. + * Finalizing the transaction adds statement-specific overwrite and authentication state without + * reloading remote metadata. + */ +public class PaimonWriteBinding { + private final PaimonExternalTable dorisTable; + private final FileStoreTable table; + private final String serializedTable; + private final Map hadoopConfig; + private final boolean overwrite; + private final Map staticPartition; + + private PaimonWriteBinding(PaimonExternalTable dorisTable, FileStoreTable table, + Map hadoopConfig, boolean overwrite, + Map staticPartition) { + this.dorisTable = dorisTable; + this.table = table; + this.serializedTable = PaimonUtil.encodeObjectToString(table); + this.hadoopConfig = Collections.unmodifiableMap(new HashMap<>(hadoopConfig)); + this.overwrite = overwrite; + this.staticPartition = Collections.unmodifiableMap(new LinkedHashMap<>(staticPartition)); + } + + public static PaimonWriteBinding create(PaimonWriteTarget writeTarget, + PaimonInsertCommandContext context) throws UserException { + PaimonExternalTable dorisTable = writeTarget.getDorisTable(); + PaimonExternalCatalog catalog = (PaimonExternalCatalog) dorisTable.getCatalog(); + FileStoreTable table = writeTarget.getTable(); + Map typedStaticPartition = context.getStaticPartition(); + Map staticPartition = resolveStaticPartition( + table, + writeTarget.getColumnTypes(), + typedStaticPartition, + context.isOverwrite()); + return new PaimonWriteBinding( + dorisTable, + table, + buildHadoopConfig(catalog), + context.isOverwrite(), + staticPartition); + } + + public FileStoreTable getTable() { + return table; + } + + public String getSerializedTable() { + return serializedTable; + } + + public Map getHadoopConfig() { + return hadoopConfig; + } + + public boolean isOverwrite() { + return overwrite; + } + + public Map getStaticPartition() { + return staticPartition; + } + + public String tableName() { + return dorisTable.getDbName() + "." + dorisTable.getName(); + } + + static Map resolveStaticPartition(FileStoreTable table, + Map writeColumnTypes, + Map typedStaticPartition, + boolean overwrite) throws AnalysisException { + Map canonicalNames = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (String partitionKey : table.partitionKeys()) { + canonicalNames.put(partitionKey, partitionKey); + } + + String defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName(); + Map resolved = new LinkedHashMap<>(); + for (Map.Entry entry : typedStaticPartition.entrySet()) { + String canonicalName = canonicalNames.get(entry.getKey()); + if (canonicalName == null) { + throw new AnalysisException("Column '" + entry.getKey() + + "' is not a partition column of Paimon table"); + } + Expression value = entry.getValue(); + if (!(value instanceof Literal)) { + throw new AnalysisException("Static partition value must be a literal, but got: " + + value); + } + DataField partitionField = table.rowType().getField(canonicalName); + org.apache.doris.catalog.Type writeType = writeColumnTypes.get(canonicalName); + Preconditions.checkNotNull(writeType, + "Paimon partition column is missing from the write schema: " + canonicalName); + Literal castValue = castPartitionValue((Literal) value, writeType); + boolean isNull = castValue instanceof NullLiteral; + String partitionValue = isNull + ? defaultPartitionName + : canonicalPartitionValue(castValue, partitionField); + if (overwrite && !isNull + && defaultPartitionName.equals(partitionValue)) { + // Paimon 1.3's public static-overwrite API uses this string as the + // NULL marker for every partition type, so the corresponding literal + // value cannot be represented without changing its typed identity. + throw new AnalysisException("Static partition value for column '" + canonicalName + + "' equals Paimon partition.default-name '" + defaultPartitionName + + "' and cannot be represented in a static overwrite"); + } + resolved.put(canonicalName, partitionValue); + } + return resolved; + } + + private static Literal castPartitionValue( + Literal literal, org.apache.doris.catalog.Type writeType) throws AnalysisException { + Expression castValue = literal.checkedCastTo(DataType.fromCatalogType(writeType)); + Preconditions.checkState(castValue instanceof Literal, + "Static Paimon partition cast must produce a literal"); + return (Literal) castValue; + } + + private static String canonicalPartitionValue( + Literal literal, DataField partitionField) { + String value = literal.getStringValue(); + if (partitionField.type().getTypeRoot() + != DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + return value; + } + + // Doris writes an LTZ literal as civil time in the session zone. Paimon 1.3 + // parses the string accepted by withOverwrite in the FE JVM default zone. + // Translate the same instant into that zone so the overwrite predicate and + // the row written by the JNI writer identify the same typed partition. + LocalDateTime sessionValue = LocalDateTime.parse( + value.replace(' ', 'T'), DateTimeFormatter.ISO_LOCAL_DATE_TIME); + return sessionValue.atZone(TimeUtils.getDorisZoneId()) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime() + // Paimon 1.3's timestamp parser accepts a space, but not ISO's + // 'T', between the date and time components. + .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .replace('T', ' '); + } + + private static Map buildHadoopConfig(PaimonExternalCatalog catalog) { + Map hadoopConfig = new HashMap<>( + catalog.getCatalogProperty().getHadoopProperties()); + String warehouse = catalog.getPaimonOptionsMap().get(CatalogOptions.WAREHOUSE.key()); + String defaultFs = resolveDefaultFsName(warehouse); + if (defaultFs != null && !defaultFs.isEmpty()) { + String current = hadoopConfig.get("fs.defaultFS"); + if (current == null || current.isEmpty() || current.startsWith("file:/")) { + hadoopConfig.put("fs.defaultFS", defaultFs); + } + } + + String hadoopUser = hadoopConfig.get("hadoop.username"); + if (hadoopUser == null || hadoopUser.isEmpty()) { + hadoopUser = hadoopConfig.get("hadoop.user.name"); + } + if (hadoopUser == null || hadoopUser.isEmpty()) { + hadoopUser = "hadoop"; + } + hadoopConfig.put("hadoop.username", hadoopUser); + hadoopConfig.put("hadoop.user.name", hadoopUser); + return hadoopConfig; + } + + private static String resolveDefaultFsName(String warehouse) { + if (warehouse == null || warehouse.isEmpty()) { + return null; + } + try { + java.net.URI uri = java.net.URI.create(warehouse); + String scheme = uri.getScheme(); + String authority = uri.getAuthority(); + if (scheme != null && !scheme.isEmpty() && authority != null && !authority.isEmpty()) { + return scheme + "://" + authority; + } + } catch (IllegalArgumentException ignored) { + return null; + } + return null; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonWriteTarget.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonWriteTarget.java new file mode 100644 index 00000000000000..86fe665a28a065 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonWriteTarget.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.ImmutableList; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypeRoot; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * One immutable view of the current Paimon write target. + * + *

A time-travel snapshot attached to the Doris table is read-side state and must not affect + * sink analysis. Capturing the latest remote table once also keeps schema binding, writer + * distribution and the serialized JNI table on the same table generation. + */ +public final class PaimonWriteTarget { + private final PaimonExternalTable dorisTable; + private final FileStoreTable table; + private final List schema; + private final Map columnsByName; + private final Map columnTypes; + private final Set partitionColumnNames; + + private PaimonWriteTarget(PaimonExternalTable dorisTable, FileStoreTable table) + throws AnalysisException { + this.dorisTable = dorisTable; + this.table = table; + PaimonExternalCatalog catalog = (PaimonExternalCatalog) dorisTable.getCatalog(); + + ImmutableList.Builder schemaBuilder = ImmutableList.builder(); + Map columns = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + Map types = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (DataField field : table.rowType().getFields()) { + Column conflictingColumn = columns.get(field.name()); + if (conflictingColumn != null) { + throw new AnalysisException("Paimon table contains columns which differ only by case: " + + conflictingColumn.getName() + " and " + field.name()); + } + Type type = PaimonUtil.paimonTypeToDorisType( + field.type(), catalog.getEnableMappingVarbinary(), false); + // Doris exposes external-table columns as nullable. The real Paimon nullability and + // defaults remain in the pinned FileStoreTable and are enforced by the Paimon writer. + Column column = new Column(field.name(), type, true, null, true, + field.description(), true, -1); + PaimonUtil.updatePaimonColumnUniqueId(column, field); + if (field.type().getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + column.setWithTZExtraInfo(); + } + schemaBuilder.add(column); + columns.put(field.name(), column); + types.put(field.name(), type); + } + this.schema = schemaBuilder.build(); + this.columnsByName = Collections.unmodifiableMap(columns); + this.columnTypes = Collections.unmodifiableMap(types); + + Set partitionColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + partitionColumns.addAll(table.partitionKeys()); + this.partitionColumnNames = Collections.unmodifiableSet(partitionColumns); + } + + public static PaimonWriteTarget create(PaimonExternalTable dorisTable) + throws AnalysisException { + try { + Table table = dorisTable.getPaimonTableForWrite(); + if (!(table instanceof FileStoreTable)) { + throw new AnalysisException("Paimon write requires a file store table"); + } + return new PaimonWriteTarget(dorisTable, (FileStoreTable) table); + } catch (AnalysisException e) { + throw e; + } catch (Exception e) { + throw new AnalysisException("Failed to load the latest Paimon write target: " + + e.getMessage(), e); + } + } + + public PaimonExternalTable getDorisTable() { + return dorisTable; + } + + public FileStoreTable getTable() { + return table; + } + + public List getSchema() { + return schema; + } + + public Column getColumn(String name) { + return columnsByName.get(name); + } + + public Map getColumnTypes() { + return columnTypes; + } + + public Set getPartitionColumnNames() { + return partitionColumnNames; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 4f30f193216e72..1b2dc56a9c8e58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.AllPartitionDesc; import org.apache.doris.analysis.DropPartitionClause; import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; @@ -43,6 +44,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -64,6 +66,8 @@ public class MTMVPartitionUtil { private static final Logger LOG = LogManager.getLogger(MTMVPartitionUtil.class); private static final Pattern PARTITION_NAME_PATTERN = Pattern.compile("[^a-zA-Z0-9,]"); private static final String PARTITION_NAME_PREFIX = "p_"; + private static final int MAX_PARTITION_NAME_LENGTH = 50; + private static final int PARTITION_IDENTITY_HASH_LENGTH = 16; private static final List partitionDescGenerators = ImmutableList .of( @@ -361,13 +365,41 @@ public static boolean isSyncWithPartitions(MTMVRefreshContext context, String mt public static String generatePartitionName(PartitionKeyDesc desc) { Matcher matcher = PARTITION_NAME_PATTERN.matcher(desc.toSql()); String partitionName = PARTITION_NAME_PREFIX + matcher.replaceAll("").replaceAll("\\,", "_"); - if (partitionName.length() > 50) { - partitionName = partitionName.substring(0, 30) + Math.abs(Objects.hash(partitionName)) - + "_" + System.currentTimeMillis(); + // The legacy readable name removes SQL quotes, so typed NULL, the string "NULL", and + // strings containing separators can collapse to the same name. Preserve ordinary names, + // but attach a stable identity derived from the complete typed descriptor where that + // lossy encoding is known to be ambiguous. The same identity also makes long names + // deterministic across retries and FE restarts. + if (containsLiteralNull(desc) || partitionName.length() > MAX_PARTITION_NAME_LENGTH) { + String identitySuffix = "_" + DigestUtils.sha256Hex(desc.toSql()) + .substring(0, PARTITION_IDENTITY_HASH_LENGTH); + int readableLength = MAX_PARTITION_NAME_LENGTH - identitySuffix.length(); + partitionName = partitionName.substring(0, Math.min(partitionName.length(), readableLength)) + + identitySuffix; } return partitionName; } + private static boolean containsLiteralNull(PartitionKeyDesc desc) { + if (desc.hasInValues()) { + for (List values : desc.getInValues()) { + if (containsLiteralNull(values)) { + return true; + } + } + return false; + } + return containsLiteralNull(desc.getLowerValues()) || containsLiteralNull(desc.getUpperValues()); + } + + private static boolean containsLiteralNull(List values) { + if (values == null) { + return false; + } + return values.stream().anyMatch(value -> !value.isNullPartition() + && "NULL".equals(value.getStringValue())); + } + /** * drop partition of mtmv * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPaimonTableSink.java new file mode 100644 index 00000000000000..4bcd7ac9bd88fe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPaimonTableSink.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Unbound Paimon table sink plan node. + */ +public class UnboundPaimonTableSink + extends UnboundBaseExternalTableSink { + private final Map staticPartitionKeyValues; + + public UnboundPaimonTableSink(List nameParts, List colNames, + List hints, List partitions, + CHILD_TYPE child) { + this(nameParts, colNames, hints, partitions, DMLCommandType.NONE, + Optional.empty(), Optional.empty(), child, null); + } + + public UnboundPaimonTableSink(List nameParts, + List colNames, + List hints, + List partitions, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + this(nameParts, colNames, hints, partitions, dmlCommandType, + groupExpression, logicalProperties, child, null); + } + + public UnboundPaimonTableSink(List nameParts, + List colNames, + List hints, + List partitions, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child, + Map staticPartitionKeyValues) { + super(nameParts, PlanType.LOGICAL_UNBOUND_PAIMON_TABLE_SINK, ImmutableList.of(), + groupExpression, logicalProperties, colNames, dmlCommandType, child, + hints, partitions); + this.staticPartitionKeyValues = staticPartitionKeyValues == null + ? ImmutableMap.of() : ImmutableMap.copyOf(staticPartitionKeyValues); + } + + public Map getStaticPartitionKeyValues() { + return staticPartitionKeyValues; + } + + public boolean hasStaticPartition() { + return !staticPartitionKeyValues.isEmpty(); + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "UnboundPaimonTableSink only accepts one child"); + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.empty(), children.get(0), + staticPartitionKeyValues); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundPaimonTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child(), + staticPartitionKeyValues); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, logicalProperties, children.get(0), + staticPartitionKeyValues); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java index 28651f86299653..1c5b5bf6064c32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java @@ -25,6 +25,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; @@ -64,6 +65,9 @@ public static LogicalSink createUnboundTableSink(List na return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, query); } else if (curCatalog instanceof MaxComputeExternalCatalog) { return new UnboundMaxComputeTableSink<>(nameParts, colNames, hints, partitions, query); + } else if (curCatalog instanceof PaimonExternalCatalog) { + validatePaimonPartitionSyntax(false, partitions); + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, query); } throw new UserException("Load data to " + curCatalog.getClass().getSimpleName() + " is not supported."); } @@ -105,6 +109,10 @@ public static LogicalSink createUnboundTableSink(List na } else if (curCatalog instanceof MaxComputeExternalCatalog) { return new UnboundMaxComputeTableSink<>(nameParts, colNames, hints, partitions, dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues); + } else if (curCatalog instanceof PaimonExternalCatalog) { + validatePaimonPartitionSyntax(temporaryPartition, partitions); + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues); } throw new RuntimeException("Load data to " + curCatalog.getClass().getSimpleName() + " is not supported."); } @@ -146,6 +154,10 @@ public static LogicalSink createUnboundTableSinkMaybeOverwrite(L } else if (curCatalog instanceof MaxComputeExternalCatalog && !isAutoDetectPartition) { return new UnboundMaxComputeTableSink<>(nameParts, colNames, hints, partitions, dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues); + } else if (curCatalog instanceof PaimonExternalCatalog && !isAutoDetectPartition) { + validatePaimonPartitionSyntax(temporaryPartition, partitions); + return new UnboundPaimonTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, Optional.empty(), Optional.empty(), plan, staticPartitionKeyValues); } throw new AnalysisException( @@ -155,6 +167,18 @@ public static LogicalSink createUnboundTableSinkMaybeOverwrite(L ? " PARTITION(*) is only supported in overwrite partition for OLAP table" : "")); } + private static void validatePaimonPartitionSyntax(boolean temporaryPartition, + List partitions) { + if (temporaryPartition) { + throw new AnalysisException("Paimon tables do not support temporary partitions"); + } + if (partitions != null && !partitions.isEmpty()) { + throw new AnalysisException("Paimon tables do not support PARTITION name lists; " + + "use PARTITION (key = value) for static partitions or omit PARTITION " + + "for dynamic partition overwrite"); + } + } + /** * create unbound sink for dictionary sink */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index edfed3c249ec4c..6e1393e1b51624 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -162,6 +162,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; @@ -221,6 +222,7 @@ import org.apache.doris.planner.NestedLoopJoinNode; import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.OlapTableSink; +import org.apache.doris.planner.PaimonTableSink; import org.apache.doris.planner.PartitionSortNode; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanNode; @@ -617,6 +619,24 @@ public PlanFragment visitPhysicalIcebergTableSink(PhysicalIcebergTableSink paimonTableSink, + PlanTranslatorContext context) { + PlanFragment rootFragment = paimonTableSink.child().accept(this, context); + rootFragment.setOutputPartition(DataPartition.UNPARTITIONED); + List outputExprs = Lists.newArrayList(); + paimonTableSink.getOutput().stream().map(Slot::getExprId) + .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId))); + PaimonTableSink sink = new PaimonTableSink(paimonTableSink.getWriteTarget()); + sink.setCols(paimonTableSink.getCols()); + rootFragment.setSink(sink); + sink.setOutputExprs(outputExprs); + if (paimonTableSink.requiresSingleWriter()) { + rootFragment.setForceSingleInstance(); + } + return rootFragment; + } + @Override public PlanFragment visitPhysicalMaxComputeTableSink(PhysicalMaxComputeTableSink mcTableSink, PlanTranslatorContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 153799232c606c..02b9a213481509 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1144,6 +1144,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -1631,9 +1632,10 @@ private InsertPartitionSpec parseInsertPartitionSpec(PartitionSpecContext ctx) { // PARTITION (col1='val1', col2='val2') - static partition if (ctx.partitionKeyValue() != null && !ctx.partitionKeyValue().isEmpty()) { Map staticValues = Maps.newLinkedHashMap(); + Set staticColumnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (DorisParser.PartitionKeyValueContext kvCtx : ctx.partitionKeyValue()) { String colName = kvCtx.identifier().getText(); - if (staticValues.containsKey(colName)) { + if (!staticColumnNames.add(colName)) { throw new AnalysisException("Duplicate partition column: " + colName); } Expression valueExpr = typedVisit(kvCtx.expression()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index 55f0a240776425..b70db5a9e4a703 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -55,6 +55,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalRecursiveUnion; @@ -174,6 +175,13 @@ public Void visitPhysicalIcebergTableSink( return null; } + @Override + public Void visitPhysicalPaimonTableSink( + PhysicalPaimonTableSink paimonTableSink, PlanContext context) { + addRequestPropertyToChildren(paimonTableSink.getRequirePhysicalProperties()); + return null; + } + @Override public Void visitPhysicalMaxComputeTableSink( PhysicalMaxComputeTableSink mcTableSink, PlanContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index c19bc98e252164..580fb79f88fa6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -90,6 +90,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalOlapScanToPhysicalOlapScan; import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink; import org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation; +import org.apache.doris.nereids.rules.implementation.LogicalPaimonTableSinkToPhysicalPaimonTableSink; import org.apache.doris.nereids.rules.implementation.LogicalPartitionTopNToPhysicalPartitionTopN; import org.apache.doris.nereids.rules.implementation.LogicalProjectToPhysicalProject; import org.apache.doris.nereids.rules.implementation.LogicalRecursiveUnionAnchorToPhysicalRecursiveUnionAnchor; @@ -239,6 +240,7 @@ public class RuleSet { .add(new LogicalOlapTableSinkToPhysicalOlapTableSink()) .add(new LogicalHiveTableSinkToPhysicalHiveTableSink()) .add(new LogicalIcebergTableSinkToPhysicalIcebergTableSink()) + .add(new LogicalPaimonTableSinkToPhysicalPaimonTableSink()) .add(new LogicalMaxComputeTableSinkToPhysicalMaxComputeTableSink()) .add(new LogicalIcebergDeleteSinkToPhysicalIcebergDeleteSink()) .add(new LogicalIcebergMergeSinkToPhysicalIcebergMergeSink()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 10d057027664db..bc0944a35500a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -40,6 +40,7 @@ public enum RuleType { BINDING_INSERT_BLACKHOLE_SINK(RuleTypeClass.REWRITE), BINDING_INSERT_HIVE_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_ICEBERG_TABLE(RuleTypeClass.REWRITE), + BINDING_INSERT_PAIMON_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_MAX_COMPUTE_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_JDBC_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_TARGET_TABLE(RuleTypeClass.REWRITE), @@ -561,6 +562,7 @@ public enum RuleType { LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_HIVE_TABLE_SINK_TO_PHYSICAL_HIVE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_ICEBERG_TABLE_SINK_TO_PHYSICAL_ICEBERG_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), + LOGICAL_PAIMON_TABLE_SINK_TO_PHYSICAL_PAIMON_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_MAX_COMPUTE_TABLE_SINK_TO_PHYSICAL_MAX_COMPUTE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_ICEBERG_DELETE_SINK_TO_PHYSICAL_ICEBERG_DELETE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_ICEBERG_MERGE_SINK_TO_PHYSICAL_ICEBERG_MERGE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 3ac7d8f05fa74d..9e9d6079d29cc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; @@ -40,6 +41,9 @@ import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalDatabase; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonWriteTarget; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -50,6 +54,7 @@ import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundJdbcTableSink; import org.apache.doris.nereids.analyzer.UnboundMaxComputeTableSink; +import org.apache.doris.nereids.analyzer.UnboundPaimonTableSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTVFTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; @@ -86,6 +91,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; @@ -117,6 +123,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -162,6 +169,8 @@ public List buildRules() { RuleType.BINDING_INSERT_HIVE_TABLE.build(unboundHiveTableSink().thenApply(this::bindHiveTableSink)), RuleType.BINDING_INSERT_ICEBERG_TABLE.build( unboundIcebergTableSink().thenApply(this::bindIcebergTableSink)), + RuleType.BINDING_INSERT_PAIMON_TABLE.build( + unboundPaimonTableSink().thenApply(this::bindPaimonTableSink)), RuleType.BINDING_INSERT_MAX_COMPUTE_TABLE.build( unboundMaxComputeTableSink().thenApply(this::bindMaxComputeTableSink)), RuleType.BINDING_INSERT_JDBC_TABLE.build(unboundJdbcTableSink().thenApply(this::bindJdbcTableSink)), @@ -304,6 +313,11 @@ private Plan bindOlapTableSink(MatchingContext> ctx) { private LogicalProject getOutputProjectByCoercion(List tableSchema, LogicalPlan child, Map columnToOutput) { + return getOutputProjectByCoercion(tableSchema, child, columnToOutput, Collections.emptyMap()); + } + + private LogicalProject getOutputProjectByCoercion(List tableSchema, LogicalPlan child, + Map columnToOutput, Map targetColumnTypes) { List fullOutputExprs = Utils.fastToImmutableList(columnToOutput.values()); if (child instanceof LogicalOneRowRelation) { // remove default value slot in one row relation @@ -331,7 +345,8 @@ private LogicalProject getOutputProjectByCoercion(List tableSchema, L } expr = expr.toSlot(); DataType inputType = expr.getDataType(); - DataType targetType = DataType.fromCatalogType(tableSchema.get(i).getType()); + Type targetCatalogType = targetColumnTypes.getOrDefault(col.getName(), col.getType()); + DataType targetType = DataType.fromCatalogType(targetCatalogType); Expression castExpr = expr; // TODO move string like type logic into TypeCoercionUtils#castIfNotSameType if (isSourceAndTargetStringLikeType(inputType, targetType) && !inputType.equals(targetType)) { @@ -859,6 +874,96 @@ private void validateStaticPartition(UnboundIcebergTableSink sink, IcebergExt } } + private Plan bindPaimonTableSink(MatchingContext> ctx) { + UnboundPaimonTableSink sink = ctx.root; + Pair pair = bind(ctx.cascadesContext, sink); + PaimonExternalDatabase database = pair.first; + PaimonExternalTable table = pair.second; + PaimonWriteTarget writeTarget; + try { + writeTarget = PaimonWriteTarget.create(table); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + LogicalPlan child = ((LogicalPlan) sink.child()); + + Map staticPartitions = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + staticPartitions.putAll(sink.getStaticPartitionKeyValues()); + Set staticPartitionColNames = staticPartitions.keySet(); + if (!staticPartitionColNames.isEmpty()) { + Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + partitionColumnNames.addAll(writeTarget.getPartitionColumnNames()); + for (String columnName : staticPartitionColNames) { + if (!partitionColumnNames.contains(columnName)) { + throw new AnalysisException(String.format( + "Column '%s' is not a partition column of Paimon table '%s'", + columnName, table.getName())); + } + Expression partitionValue = staticPartitions.get(columnName); + if (!(partitionValue instanceof Literal)) { + throw new AnalysisException(String.format( + "Partition value for column '%s' must be a literal, but got: %s", + columnName, partitionValue)); + } + } + } + + List bindColumns; + if (sink.getColNames().isEmpty()) { + bindColumns = writeTarget.getSchema().stream() + .filter(Column::isVisible) + .filter(column -> !staticPartitionColNames.contains(column.getName())) + .collect(ImmutableList.toImmutableList()); + } else { + Set specifiedColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (String columnName : sink.getColNames()) { + if (!specifiedColumnNames.add(columnName)) { + throw new AnalysisException( + "Duplicate column '" + columnName + "' in Paimon insert column list"); + } + } + bindColumns = sink.getColNames().stream().map(cn -> { + if (staticPartitionColNames.contains(cn)) { + throw new AnalysisException(String.format( + "Static partition column '%s' must not appear in the insert column list", cn)); + } + Column column = writeTarget.getColumn(cn); + if (column == null) { + throw new AnalysisException(String.format( + "column %s is not found in table %s", cn, table.getName())); + } + return column; + }).collect(ImmutableList.toImmutableList()); + } + + if (bindColumns.size() != child.getOutput().size()) { + throw new AnalysisException("insert into cols should be corresponding to the query output"); + } + Map columnToOutput = getJdbcColumnToOutput(bindColumns, child); + List writeColumns = new ArrayList<>(bindColumns); + if (!staticPartitionColNames.isEmpty()) { + for (Column column : writeTarget.getSchema()) { + Expression staticValue = staticPartitions.get(column.getName()); + if (staticValue != null) { + Expression castExpr = TypeCoercionUtils.castIfNotSameType( + staticValue, DataType.fromCatalogType(column.getType())); + columnToOutput.put(column.getName(), new Alias(castExpr, column.getName())); + writeColumns.add(column); + } + } + } + + LogicalPaimonTableSink boundSink = new LogicalPaimonTableSink<>( + database, writeTarget, writeColumns, + child.getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()), + sink.getDMLCommandType(), Optional.empty(), Optional.empty(), child); + LogicalProject outputProject = getOutputProjectByCoercion( + writeColumns, child, columnToOutput, writeTarget.getColumnTypes()); + return boundSink.withChildAndUpdateOutput(outputProject); + } + private Plan bindMaxComputeTableSink(MatchingContext> ctx) { UnboundMaxComputeTableSink sink = ctx.root; Pair pair = bind(ctx.cascadesContext, sink); @@ -1068,6 +1173,18 @@ private Pair bind(CascadesContext throw new AnalysisException("the target table of insert into is not an iceberg table"); } + private Pair bind(CascadesContext cascadesContext, + UnboundPaimonTableSink sink) { + List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), + sink.getNameParts()); + Pair, TableIf> pair = RelationUtil.getDbAndTable(tableQualifier, + cascadesContext.getConnectContext().getEnv(), Optional.empty()); + if (pair.second instanceof PaimonExternalTable) { + return Pair.of(((PaimonExternalDatabase) pair.first), (PaimonExternalTable) pair.second); + } + throw new AnalysisException("the target table of insert into is not a paimon table"); + } + private Pair bind(CascadesContext cascadesContext, UnboundMaxComputeTableSink sink) { List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 76b57eece78a48..fb88caa081b6c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -309,12 +309,8 @@ private static Map evalOnBE(Map> TQueryGlobals queryGlobals = new TQueryGlobals(); queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now())); queryGlobals.setTimestampMs(System.currentTimeMillis()); - queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE); - if (context.getSessionVariable().getTimeZone().equals("CST")) { - queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE); - } else { - queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone()); - } + queryGlobals.setTimeZone( + TimeUtils.getCanonicalTimeZone(context.getSessionVariable().getTimeZone())); TQueryOptions tQueryOptions = new TQueryOptions(); tQueryOptions.setBeExecVersion(Config.be_exec_version); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPaimonTableSinkToPhysicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPaimonTableSinkToPhysicalPaimonTableSink.java new file mode 100644 index 00000000000000..a9612f1280b03c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPaimonTableSinkToPhysicalPaimonTableSink.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPaimonTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; + +import java.util.Optional; + +/** + * Implementation rule that converts LogicalPaimonTableSink to PhysicalPaimonTableSink. + */ +public class LogicalPaimonTableSinkToPhysicalPaimonTableSink extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalPaimonTableSink().thenApply(ctx -> { + LogicalPaimonTableSink sink = ctx.root; + return new PhysicalPaimonTableSink<>( + sink.getDatabase(), + sink.getWriteTarget(), + sink.getCols(), + sink.getOutputExprs(), + Optional.empty(), + sink.getLogicalProperties(), + null, + null, + sink.child()); + }).toRule(RuleType.LOGICAL_PAIMON_TABLE_SINK_TO_PHYSICAL_PAIMON_TABLE_SINK_RULE); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index bda7fc82da1cd9..b6aea691fc96ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -50,6 +50,7 @@ public enum PlanType { LOGICAL_OLAP_TABLE_SINK, LOGICAL_HIVE_TABLE_SINK, LOGICAL_ICEBERG_TABLE_SINK, + LOGICAL_PAIMON_TABLE_SINK, LOGICAL_MAX_COMPUTE_TABLE_SINK, LOGICAL_ICEBERG_DELETE_SINK, LOGICAL_ICEBERG_MERGE_SINK, @@ -60,6 +61,7 @@ public enum PlanType { LOGICAL_UNBOUND_OLAP_TABLE_SINK, LOGICAL_UNBOUND_HIVE_TABLE_SINK, LOGICAL_UNBOUND_ICEBERG_TABLE_SINK, + LOGICAL_UNBOUND_PAIMON_TABLE_SINK, LOGICAL_UNBOUND_MAX_COMPUTE_TABLE_SINK, LOGICAL_UNBOUND_JDBC_TABLE_SINK, LOGICAL_UNBOUND_RESULT_SINK, @@ -123,6 +125,7 @@ public enum PlanType { PHYSICAL_OLAP_TABLE_SINK, PHYSICAL_HIVE_TABLE_SINK, PHYSICAL_ICEBERG_TABLE_SINK, + PHYSICAL_PAIMON_TABLE_SINK, PHYSICAL_MAX_COMPUTE_TABLE_SINK, PHYSICAL_ICEBERG_DELETE_SINK, PHYSICAL_ICEBERG_MERGE_SINK, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 05935a64d770f0..ac1464525deaf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -34,6 +34,7 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.load.loadv2.LoadJob; import org.apache.doris.load.loadv2.LoadStatistic; @@ -74,6 +75,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -517,6 +519,19 @@ private ExecutorFactory selectInsertExecutorFactory( emptyInsert, jobId ) ); + } else if (physicalSink instanceof PhysicalPaimonTableSink) { + PaimonExternalTable paimonTable = (PaimonExternalTable) targetTableIf; + PaimonInsertCommandContext paimonCtx = insertCtx + .map(ctx1 -> (PaimonInsertCommandContext) ctx1) + .orElseGet(PaimonInsertCommandContext::new); + boolean emptyInsert = childIsEmptyRelation(physicalSink) && !paimonCtx.isOverwrite(); + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new PaimonInsertExecutor(ctx, paimonTable, label, planner, + Optional.of(paimonCtx), emptyInsert, jobId) + ); } else if (physicalSink instanceof PhysicalJdbcTableSink) { boolean emptyInsert = childIsEmptyRelation(physicalSink); List cols = ((PhysicalJdbcTableSink) physicalSink).getCols(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index efe049eda20bb6..86cd699b467bf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -29,6 +29,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.insertoverwrite.InsertOverwriteManager; import org.apache.doris.insertoverwrite.InsertOverwriteUtil; import org.apache.doris.mtmv.MTMVUtil; @@ -39,6 +40,7 @@ import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundMaxComputeTableSink; +import org.apache.doris.nereids.analyzer.UnboundPaimonTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -137,7 +139,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx); //check allow insert overwrite if (!allowInsertOverwrite(targetTableIf)) { - String errMsg = "insert into overwrite only support OLAP and HMS/ICEBERG table." + String errMsg = "insert into overwrite only support OLAP and " + + "HMS/ICEBERG/MAXCOMPUTE/PAIMON table." + " But current table type is " + targetTableIf.getType(); LOG.error(errMsg); throw new AnalysisException(errMsg); @@ -309,7 +312,8 @@ private boolean allowInsertOverwrite(TableIf targetTable) { } else { return targetTable instanceof HMSExternalTable || targetTable instanceof IcebergExternalTable - || targetTable instanceof MaxComputeExternalTable; + || targetTable instanceof MaxComputeExternalTable + || targetTable instanceof PaimonExternalTable; } } @@ -408,6 +412,17 @@ private void insertIntoPartitions(ConnectContext ctx, StmtExecutor executor, Lis mcCtx.setStaticPartitionSpec(staticSpec); } insertCtx = mcCtx; + } else if (logicalQuery instanceof UnboundPaimonTableSink) { + UnboundPaimonTableSink sink = (UnboundPaimonTableSink) logicalQuery; + copySink = (UnboundLogicalSink) UnboundTableSinkCreator.createUnboundTableSink( + sink.getNameParts(), sink.getColNames(), sink.getHints(), false, + sink.getPartitions(), false, TPartialUpdateNewRowPolicy.APPEND, + sink.getDMLCommandType(), (LogicalPlan) sink.child(0), + sink.getStaticPartitionKeyValues()); + PaimonInsertCommandContext paimonCtx = new PaimonInsertCommandContext(); + paimonCtx.setOverwrite(true); + setStaticPartitionToContext(sink, paimonCtx); + insertCtx = paimonCtx; } else { throw new UserException("Current catalog does not support insert overwrite yet."); } @@ -459,6 +474,13 @@ private void setStaticPartitionToContext(UnboundIcebergTableSink sink, } } + private void setStaticPartitionToContext(UnboundPaimonTableSink sink, + PaimonInsertCommandContext insertCtx) { + if (sink.hasStaticPartition()) { + insertCtx.setStaticPartition(sink.getStaticPartitionKeyValues()); + } + } + @Override public Plan getExplainPlan(ConnectContext ctx) { Optional analyzeContext = Optional.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 6302b6aaed1a3a..ee98f0ece78bfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -41,6 +41,7 @@ import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.analyzer.UnboundJdbcTableSink; import org.apache.doris.nereids.analyzer.UnboundMaxComputeTableSink; +import org.apache.doris.nereids.analyzer.UnboundPaimonTableSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.analyzer.UnboundTableSink; @@ -382,10 +383,13 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, staticPartitions = ((UnboundIcebergTableSink) unboundLogicalSink).getStaticPartitionKeyValues(); } else if (unboundLogicalSink instanceof UnboundMaxComputeTableSink) { staticPartitions = ((UnboundMaxComputeTableSink) unboundLogicalSink).getStaticPartitionKeyValues(); + } else if (unboundLogicalSink instanceof UnboundPaimonTableSink) { + staticPartitions = ((UnboundPaimonTableSink) unboundLogicalSink).getStaticPartitionKeyValues(); } if (staticPartitions != null && !staticPartitions.isEmpty() && CollectionUtils.isEmpty(unboundLogicalSink.getColNames())) { - Set staticPartitionColNames = staticPartitions.keySet(); + Set staticPartitionColNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + staticPartitionColNames.addAll(staticPartitions.keySet()); columns = columns.stream() .filter(column -> !staticPartitionColNames.contains(column.getName())) .collect(ImmutableList.toImmutableList()); @@ -611,9 +615,12 @@ public static List getTargetTableQualified(Plan plan, ConnectContext ctx unboundTableSink = (UnboundBlackholeSink) plan; } else if (plan instanceof UnboundMaxComputeTableSink) { unboundTableSink = (UnboundMaxComputeTableSink) plan; + } else if (plan instanceof UnboundPaimonTableSink) { + unboundTableSink = (UnboundPaimonTableSink) plan; } else { throw new AnalysisException( - "the root of plan only accept Olap, Dictionary, Hive, Iceberg or Jdbc table sink, but it is " + "the root of plan only accept Olap, Dictionary, Hive, Iceberg, Paimon" + + " or Jdbc table sink, but it is " + plan.getType()); } return RelationUtil.getQualifierName(ctx, unboundTableSink.getNameParts()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertCommandContext.java new file mode 100644 index 00000000000000..9353af35ca2674 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertCommandContext.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.nereids.trees.expressions.Expression; + +import java.util.HashMap; +import java.util.Map; + +/** + * Insert command context for Paimon tables. + */ +public class PaimonInsertCommandContext extends BaseExternalTableInsertCommandContext { + private long txnId = 0; + private String commitUser; + private Map staticPartition = new HashMap<>(); + + public long getTxnId() { + return txnId; + } + + public void setTxnId(long txnId) { + this.txnId = txnId; + } + + public String getCommitUser() { + return commitUser; + } + + public void setCommitUser(String commitUser) { + this.commitUser = commitUser; + } + + public Map getStaticPartition() { + return staticPartition; + } + + public void setStaticPartition(Map staticPartition) { + this.staticPartition = staticPartition != null + ? new HashMap<>(staticPartition) : new HashMap<>(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertExecutor.java new file mode 100644 index 00000000000000..ed540dbabd5ff5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PaimonInsertExecutor.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonTransaction; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.transaction.TransactionType; + +import java.util.Optional; + +/** + * Insert executor for Paimon tables. + */ +public class PaimonInsertExecutor extends BaseExternalTableInsertExecutor { + public PaimonInsertExecutor(ConnectContext ctx, PaimonExternalTable table, + String labelName, NereidsPlanner planner, + Optional insertCtx, + boolean emptyInsert, long jobId) { + super(ctx, table, labelName, planner, insertCtx, emptyInsert, jobId); + } + + @Override + public void beginTransaction() { + super.beginTransaction(); + PaimonInsertCommandContext paimonCtx = (PaimonInsertCommandContext) insertCtx.get(); + paimonCtx.setTxnId(txnId); + paimonCtx.setCommitUser(PaimonTransaction.commitUser(txnId)); + } + + @Override + protected void beforeExec() throws UserException { + } + + @Override + protected void doBeforeCommit() throws UserException { + } + + @Override + protected TransactionType transactionType() { + return TransactionType.PAIMON; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPaimonTableSink.java new file mode 100644 index 00000000000000..c1e1d678b4b437 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPaimonTableSink.java @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.paimon.PaimonExternalDatabase; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonWriteTarget; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Logical Paimon table sink for INSERT INTO paimon_table. + */ +public class LogicalPaimonTableSink extends LogicalTableSink + implements Sink { + + private final PaimonExternalDatabase database; + private final PaimonExternalTable targetTable; + private final PaimonWriteTarget writeTarget; + private final DMLCommandType dmlCommandType; + + /** + * Create a logical Paimon sink bound to one immutable write target. + */ + public LogicalPaimonTableSink(PaimonExternalDatabase database, + PaimonWriteTarget writeTarget, + List cols, + List outputExprs, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + super(PlanType.LOGICAL_PAIMON_TABLE_SINK, outputExprs, groupExpression, logicalProperties, + cols, child); + this.database = Objects.requireNonNull(database, "database != null"); + this.writeTarget = Objects.requireNonNull(writeTarget, "writeTarget != null"); + this.targetTable = writeTarget.getDorisTable(); + this.dmlCommandType = dmlCommandType; + } + + /** Update output expressions based on child output and replace child. */ + public Plan withChildAndUpdateOutput(Plan child) { + List output = child.getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + return new LogicalPaimonTableSink<>(database, writeTarget, cols, output, + dmlCommandType, Optional.empty(), Optional.empty(), child); + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "LogicalPaimonTableSink only accepts one child"); + return new LogicalPaimonTableSink<>(database, writeTarget, cols, outputExprs, + dmlCommandType, Optional.empty(), Optional.empty(), children.get(0)); + } + + public LogicalPaimonTableSink withOutputExprs(List outputExprs) { + return new LogicalPaimonTableSink<>(database, writeTarget, cols, outputExprs, + dmlCommandType, Optional.empty(), Optional.empty(), child()); + } + + public PaimonExternalDatabase getDatabase() { + return database; + } + + public PaimonExternalTable getTargetTable() { + return targetTable; + } + + public PaimonWriteTarget getWriteTarget() { + return writeTarget; + } + + public DMLCommandType getDmlCommandType() { + return dmlCommandType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + LogicalPaimonTableSink that = (LogicalPaimonTableSink) o; + return dmlCommandType == that.dmlCommandType + && Objects.equals(database, that.database) + && Objects.equals(targetTable, that.targetTable) + && Objects.equals(writeTarget, that.writeTarget) + && Objects.equals(cols, that.cols); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), database, targetTable, writeTarget, cols, dmlCommandType); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalPaimonTableSink[" + id.asInt() + "]", + "outputExprs", outputExprs, + "database", database.getFullName(), + "targetTable", targetTable.getName(), + "cols", cols, + "dmlCommandType", dmlCommandType); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalPaimonTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new LogicalPaimonTableSink<>(database, writeTarget, cols, outputExprs, + dmlCommandType, groupExpression, + Optional.of(getLogicalProperties()), child()); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new LogicalPaimonTableSink<>(database, writeTarget, cols, outputExprs, + dmlCommandType, groupExpression, logicalProperties, children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java new file mode 100644 index 00000000000000..824625de3cb163 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSink.java @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.paimon.PaimonExternalDatabase; +import org.apache.doris.datasource.paimon.PaimonWriteTarget; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.statistics.Statistics; + +import com.google.common.base.Preconditions; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.table.FileStoreTable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Physical Paimon table sink. + */ +public class PhysicalPaimonTableSink + extends PhysicalBaseExternalTableSink { + private final PaimonWriteTarget writeTarget; + + public PhysicalPaimonTableSink(PaimonExternalDatabase database, + PaimonWriteTarget writeTarget, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + CHILD_TYPE child) { + this(database, writeTarget, cols, outputExprs, groupExpression, logicalProperties, + PhysicalProperties.SINK_RANDOM_PARTITIONED, null, child); + } + + public PhysicalPaimonTableSink(PaimonExternalDatabase database, + PaimonWriteTarget writeTarget, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + CHILD_TYPE child) { + super(PlanType.PHYSICAL_PAIMON_TABLE_SINK, database, writeTarget.getDorisTable(), cols, outputExprs, + groupExpression, logicalProperties, physicalProperties, statistics, child); + this.writeTarget = writeTarget; + } + + @Override + public Plan withChildren(List children) { + return new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, writeTarget, cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, statistics, children.get(0)); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, writeTarget, cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, statistics, child()); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, writeTarget, cols, outputExprs, groupExpression, + logicalProperties.get(), physicalProperties, statistics, children.get(0)); + } + + @Override + public PhysicalPaimonTableSink withPhysicalPropertiesAndStats( + PhysicalProperties physicalProperties, Statistics stats) { + return new PhysicalPaimonTableSink<>( + (PaimonExternalDatabase) database, writeTarget, cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, stats, child()); + } + + @Override + public PhysicalProperties getRequirePhysicalProperties() { + FileStoreTable paimonTable = writeTarget.getTable(); + if (requiresSingleWriter(paimonTable)) { + return PhysicalProperties.GATHER; + } + + List primaryKeys = paimonTable.primaryKeys(); + if (primaryKeys.isEmpty()) { + return PhysicalProperties.SINK_RANDOM_PARTITIONED; + } + + List outputSlots = child().getOutput(); + Preconditions.checkState(cols.size() == outputSlots.size(), + "Paimon sink columns must match child output"); + Map columnExprIds = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + for (int i = 0; i < cols.size(); i++) { + columnExprIds.put(cols.get(i).getName(), outputSlots.get(i).getExprId()); + } + + List primaryKeyExprIds = new ArrayList<>(primaryKeys.size()); + for (String primaryKey : primaryKeys) { + primaryKeyExprIds.add(Preconditions.checkNotNull( + columnExprIds.get(primaryKey), + "Paimon primary-key column is missing from sink output")); + } + return PhysicalProperties.createHash(primaryKeyExprIds, ShuffleType.REQUIRE); + } + + /** + * Whether this sink must use one writer to preserve Paimon write semantics. + * + *

For dynamic bucket tables, GATHER guarantees one writer only within the current + * INSERT. Paimon does not support concurrent write jobs to the same partition, and Doris + * does not add a process-local lease which could not coordinate FE failover or external + * Flink/Spark writers. Concurrent INSERTs into a dynamic bucket table are therefore + * unsupported. + */ + public boolean requiresSingleWriter() { + return requiresSingleWriter(writeTarget.getTable()); + } + + static boolean requiresSingleWriter(FileStoreTable paimonTable) { + BucketMode bucketMode = paimonTable.bucketMode(); + CoreOptions coreOptions = CoreOptions.fromMap(paimonTable.options()); + if (bucketMode == BucketMode.HASH_DYNAMIC + || bucketMode == BucketMode.KEY_DYNAMIC + // Until Doris has a Paimon bucket-aware exchange, a fixed-bucket + // primary-key table must not let independent writers own the same bucket. + // An append-only writer has the same ownership requirement while automatic + // compaction is enabled, because two writers can restore and replace the same + // existing files in one Doris transaction. + || (bucketMode == BucketMode.HASH_FIXED + && (!paimonTable.primaryKeys().isEmpty() || !coreOptions.writeOnly()))) { + return true; + } + + return !coreOptions.writeOnly() + && (coreOptions.needLookup() + || coreOptions.changelogProducer() + == CoreOptions.ChangelogProducer.FULL_COMPACTION); + } + + public PaimonWriteTarget getWriteTarget() { + return writeTarget; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalPaimonTableSink(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java index f4abd1dd562c57..b02ea52b2a88f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundJdbcTableSink; import org.apache.doris.nereids.analyzer.UnboundMaxComputeTableSink; +import org.apache.doris.nereids.analyzer.UnboundPaimonTableSink; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTVFTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; @@ -38,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; @@ -53,6 +55,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalMaxComputeTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFTableSink; @@ -87,6 +90,10 @@ default R visitUnboundIcebergTableSink(UnboundIcebergTableSink u return visitLogicalSink(unboundTableSink, context); } + default R visitUnboundPaimonTableSink(UnboundPaimonTableSink unboundTableSink, C context) { + return visitLogicalSink(unboundTableSink, context); + } + default R visitUnboundJdbcTableSink(UnboundJdbcTableSink unboundTableSink, C context) { return visitLogicalSink(unboundTableSink, context); } @@ -135,6 +142,10 @@ default R visitLogicalIcebergTableSink(LogicalIcebergTableSink i return visitLogicalTableSink(icebergTableSink, context); } + default R visitLogicalPaimonTableSink(LogicalPaimonTableSink paimonTableSink, C context) { + return visitLogicalTableSink(paimonTableSink, context); + } + default R visitLogicalMaxComputeTableSink(LogicalMaxComputeTableSink mcTableSink, C context) { return visitLogicalTableSink(mcTableSink, context); } @@ -203,6 +214,10 @@ default R visitPhysicalIcebergTableSink(PhysicalIcebergTableSink return visitPhysicalTableSink(icebergTableSink, context); } + default R visitPhysicalPaimonTableSink(PhysicalPaimonTableSink paimonTableSink, C context) { + return visitPhysicalTableSink(paimonTableSink, context); + } + default R visitPhysicalMaxComputeTableSink(PhysicalMaxComputeTableSink mcTableSink, C context) { return visitPhysicalTableSink(mcTableSink, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java new file mode 100644 index 00000000000000..a6f2e095c3ba44 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PaimonTableSink.java @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.Column; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonTransaction; +import org.apache.doris.datasource.paimon.PaimonWriteBinding; +import org.apache.doris.datasource.paimon.PaimonWriteTarget; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; +import org.apache.doris.nereids.trees.plans.commands.insert.PaimonInsertCommandContext; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TPaimonTableSink; +import org.apache.doris.thrift.TPaimonWriteBackendType; +import org.apache.doris.thrift.TPaimonWriteMode; + +import com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * Paimon table sink. + * + * Generates TPaimonTableSink payload consumed by BE, including serialized table + * metadata, Hadoop authentication config, transaction identity, write mode, + * and sink column names. + * + * v1: single-writer architecture; partition/bucket routing delegated to SDK. + */ +public class PaimonTableSink extends BaseExternalTableDataSink { + private final PaimonExternalTable targetTable; + private final PaimonWriteTarget writeTarget; + private List outputExprs; + private List cols; + + private static final HashSet supportedTypes = new HashSet() {{ + add(TFileFormatType.FORMAT_ORC); + add(TFileFormatType.FORMAT_PARQUET); + }}; + + public PaimonTableSink(PaimonWriteTarget writeTarget) { + super(); + this.writeTarget = writeTarget; + this.targetTable = writeTarget.getDorisTable(); + } + + public void setCols(List cols) { + this.cols = cols; + } + + public void setOutputExprs(List outputExprs) { + this.outputExprs = outputExprs; + } + + @Override + protected Set supportedFileFormatTypes() { + return supportedTypes; + } + + @Override + public String getExplainString(String prefix, TExplainLevel explainLevel) { + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append(prefix).append("PAIMON TABLE SINK\n"); + if (explainLevel == TExplainLevel.BRIEF) { + return strBuilder.toString(); + } + strBuilder.append(prefix).append(" table: ").append(targetTable.getName()).append("\n"); + return strBuilder.toString(); + } + + @Override + public void bindDataSink(Optional insertCtx) throws AnalysisException { + TPaimonTableSink tSink = new TPaimonTableSink(); + PaimonInsertCommandContext ctx = (PaimonInsertCommandContext) insertCtx.get(); + Preconditions.checkState(ctx.getTxnId() > 0, + "Paimon transaction must begin before sink binding"); + + PaimonTransaction transaction; + PaimonWriteBinding binding; + try { + transaction = (PaimonTransaction) targetTable.getCatalog() + .getTransactionManager().getTransaction(ctx.getTxnId()); + binding = PaimonWriteBinding.create(writeTarget, ctx); + } catch (AnalysisException e) { + throw e; + } catch (UserException e) { + throw new AnalysisException("Failed to bind Paimon write transaction: " + + e.getMessage(), e); + } + transaction.bind(binding); + + tSink.setTransactionId(ctx.getTxnId()); + tSink.setCommitUser(ctx.getCommitUser()); + + // Thrift column_names is the single column-order protocol shared by BE + // Arrow conversion and the Java writer schema. + List outputColumnNames = outputColumnNames(); + + // FE owns table metadata resolution. BE and the JNI writer consume this + // exact table instance instead of loading catalog metadata independently. + tSink.setSerializedTable(binding.getSerializedTable()); + + tSink.setBackendType(TPaimonWriteBackendType.JNI); + if (ctx.isOverwrite()) { + tSink.setWriteMode(TPaimonWriteMode.OVERWRITE); + } else { + tSink.setWriteMode(TPaimonWriteMode.APPEND); + } + + tSink.setHadoopConfig(binding.getHadoopConfig()); + + tSink.setColumnNames(outputColumnNames); + + tDataSink = new TDataSink(TDataSinkType.PAIMON_TABLE_SINK); + tDataSink.setPaimonTableSink(tSink); + } + + private List outputColumnNames() throws AnalysisException { + if (cols.size() != outputExprs.size()) { + throw new AnalysisException("Paimon sink output column size mismatch, columns=" + + cols.size() + ", exprs=" + outputExprs.size()); + } + List names = new ArrayList<>(cols.size()); + for (Column col : cols) { + names.add(col.getName()); + } + return names; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 9face334255cb7..a08a8de50bfaa9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -40,6 +40,7 @@ import org.apache.doris.datasource.hive.HMSTransaction; import org.apache.doris.datasource.iceberg.IcebergTransaction; import org.apache.doris.datasource.maxcompute.MCTransaction; +import org.apache.doris.datasource.paimon.PaimonTransaction; import org.apache.doris.load.loadv2.LoadJob; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.MysqlCommand; @@ -369,11 +370,8 @@ public Coordinator(ConnectContext context, Planner planner) { this.queryGlobals.setTimestampMs(System.currentTimeMillis()); this.queryGlobals.setNanoSeconds(LocalDateTime.now().getNano()); this.queryGlobals.setLoadZeroTolerance(false); - if (context.getSessionVariable().getTimeZone().equals("CST")) { - this.queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE); - } else { - this.queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone()); - } + this.queryGlobals.setTimeZone( + TimeUtils.getCanonicalTimeZone(context.getSessionVariable().getTimeZone())); this.queryGlobals.setLcTimeNames(context.getSessionVariable().getLcTimeNames()); this.assignedRuntimeFilters = planner.getRuntimeFilters(); this.topnFilters = planner.getTopnFilters(); @@ -2642,6 +2640,11 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { ((MCTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) .updateMCCommitData(params.getMcCommitDatas()); } + if (params.isSetPaimonCommitMessages()) { + ((PaimonTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(txnId)) + .updateCommitMessages(params.getPaimonCommitMessages()); + } if (ctx.done) { if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java index 61aee063452984..8311311b51816b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java @@ -349,11 +349,8 @@ private static TQueryGlobals initQueryGlobals(ConnectContext context) { queryGlobals.setTimestampMs(System.currentTimeMillis()); queryGlobals.setNanoSeconds(LocalDateTime.now().getNano()); queryGlobals.setLoadZeroTolerance(false); - if (context.getSessionVariable().getTimeZone().equals("CST")) { - queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE); - } else { - queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone()); - } + queryGlobals.setTimeZone( + TimeUtils.getCanonicalTimeZone(context.getSessionVariable().getTimeZone())); queryGlobals.setLcTimeNames(context.getSessionVariable().getLcTimeNames()); return queryGlobals; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java index 128704c9e65320..745e10af822491 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExecutor.java @@ -29,6 +29,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Status; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -408,11 +409,8 @@ private InternalService.PTabletKeyLookupRequest buildLookupRequest(boolean inclu } // Set timezone for functions like from_unixtime - String timeZone = ConnectContext.get().getSessionVariable().getTimeZone(); - if ("CST".equals(timeZone)) { - timeZone = "Asia/Shanghai"; - } - requestBuilder.setTimeZone(timeZone); + requestBuilder.setTimeZone(TimeUtils.getCanonicalTimeZone( + ConnectContext.get().getSessionVariable().getTimeZone())); if (snapshotVisibleVersions != null && !snapshotVisibleVersions.isEmpty()) { requestBuilder.setVersion(snapshotVisibleVersions.get(0)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java index a605f3039aa805..c2b769196af07e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/LoadProcessor.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.hive.HMSTransaction; import org.apache.doris.datasource.iceberg.IcebergTransaction; import org.apache.doris.datasource.maxcompute.MCTransaction; +import org.apache.doris.datasource.paimon.PaimonTransaction; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.AbstractJobProcessor; import org.apache.doris.qe.CoordinatorContext; @@ -228,6 +229,11 @@ protected void doProcessReportExecStatus(TReportExecStatusParams params, SingleF ((MCTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().getTxnById(txnId)) .updateMCCommitData(params.getMcCommitDatas()); } + if (params.isSetPaimonCommitMessages()) { + ((PaimonTransaction) Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr() + .getTxnById(txnId)) + .updateCommitMessages(params.getPaimonCommitMessages()); + } if (fragmentTask.isDone()) { if (LOG.isDebugEnabled()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/AbstractExternalTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/AbstractExternalTransactionManager.java index da80b8f77bd6f0..c72177b2451874 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/AbstractExternalTransactionManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/AbstractExternalTransactionManager.java @@ -36,12 +36,12 @@ public AbstractExternalTransactionManager(ExternalMetadataOps ops) { this.ops = ops; } - abstract T createTransaction(); + abstract T createTransaction(long transactionId); @Override public long begin() { long id = Env.getCurrentEnv().getNextId(); - T transaction = createTransaction(); + T transaction = createTransaction(id); transactions.put(id, transaction); Env.getCurrentEnv().getGlobalExternalTransactionInfoMgr().putTxnById(id, transaction); return id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java index 65f0c2bd5e3cb3..bc5179d8e67e76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java @@ -36,7 +36,7 @@ public HiveTransactionManager(HiveMetadataOps ops, FileSystemProvider fileSystem } @Override - HMSTransaction createTransaction() { + HMSTransaction createTransaction(long transactionId) { return new HMSTransaction((HiveMetadataOps) ops, fileSystemProvider, fileSystemExecutor); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java index 8f4d25a19b3ac5..ab62138d0364e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java @@ -28,7 +28,7 @@ public IcebergTransactionManager(IcebergMetadataOps ops) { } @Override - IcebergTransaction createTransaction() { + IcebergTransaction createTransaction(long transactionId) { return new IcebergTransaction((IcebergMetadataOps) ops); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/MCTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/MCTransactionManager.java index a7d1428f641a95..52e61fe957e76e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/MCTransactionManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/MCTransactionManager.java @@ -30,7 +30,7 @@ public MCTransactionManager(MaxComputeExternalCatalog catalog) { } @Override - MCTransaction createTransaction() { + MCTransaction createTransaction(long transactionId) { return new MCTransaction(catalog); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/PaimonTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/PaimonTransactionManager.java new file mode 100644 index 00000000000000..257a125e29250f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/PaimonTransactionManager.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.transaction; + +import org.apache.doris.datasource.paimon.PaimonMetadataOps; +import org.apache.doris.datasource.paimon.PaimonTransaction; + +/** + * Transaction manager for Paimon external tables. + */ +public class PaimonTransactionManager extends AbstractExternalTransactionManager { + + public PaimonTransactionManager(PaimonMetadataOps ops) { + super(ops); + } + + @Override + PaimonTransaction createTransaction(long transactionId) { + return new PaimonTransaction((PaimonMetadataOps) ops, transactionId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java index e08f13ad0a8734..01833129b8f3cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java @@ -20,6 +20,7 @@ import org.apache.doris.datasource.hive.HiveMetadataOps; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonMetadataOps; import org.apache.doris.fs.FileSystemProvider; import java.util.concurrent.Executor; @@ -38,4 +39,8 @@ public static TransactionManager createIcebergTransactionManager(IcebergMetadata public static TransactionManager createMCTransactionManager(MaxComputeExternalCatalog catalog) { return new MCTransactionManager(catalog); } + + public static TransactionManager createPaimonTransactionManager(PaimonMetadataOps ops) { + return new PaimonTransactionManager(ops); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java index 83e092c0ed7136..d6a587eaa2effe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionType.java @@ -22,5 +22,6 @@ public enum TransactionType { HMS, ICEBERG, JDBC, - MAXCOMPUTE + MAXCOMPUTE, + PAIMON } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java index f57e8fe2fe65d2..b5c7424fc50c96 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java @@ -168,6 +168,8 @@ public void testTimezone() throws AnalysisException { Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("+8:00")); Assert.assertEquals("-08:00", TimeUtils.checkTimeZoneValidAndStandardize("-8:00")); Assert.assertEquals("+08:00", TimeUtils.checkTimeZoneValidAndStandardize("8:00")); + Assert.assertEquals("Asia/Shanghai", TimeUtils.getCanonicalTimeZone("CST")); + Assert.assertEquals("-05:00", TimeUtils.getCanonicalTimeZone("EST")); } catch (DdlException ex) { Assert.assertTrue(ex.getMessage(), false); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 3389eb23918b07..e7ba0495b5903e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -44,7 +44,7 @@ public class PaimonExternalMetaCacheTest { @Test public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( - new PaimonPartitionInfoLoader(null), + new PaimonPartitionInfoLoader(), (nameMapping, schemaId) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); @@ -55,11 +55,11 @@ public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { SchemaManager schemaManager = Mockito.mock(SchemaManager.class); TableSchema latestSchema = Mockito.mock(TableSchema.class); Mockito.when(snapshot.id()).thenReturn(12L); - Mockito.when(baseTable.latestSnapshot()).thenReturn(Optional.of(snapshot)); - Mockito.when(baseTable.copy(Collections.singletonMap( + Mockito.when(baseTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(snapshot)); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Collections.singletonMap( CoreOptions.SCAN_SNAPSHOT_ID.key(), "12"))).thenReturn(pinnedTable); - Mockito.when(pinnedTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); - Mockito.when(baseTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); Mockito.when(latestSchema.id()).thenReturn(4L); @@ -67,8 +67,9 @@ public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { Assert.assertEquals(12L, value.getSnapshot().getSnapshotId()); Assert.assertEquals(4L, value.getSnapshot().getSchemaId()); - Assert.assertSame(latestSchemaTable, value.getSnapshot().getTable()); - Mockito.verify(pinnedTable).copyWithLatestSchema(); + Assert.assertSame(pinnedTable, value.getSnapshot().getTable()); + Mockito.verify(latestSchemaTable).copyWithoutTimeTravel(Collections.singletonMap( + CoreOptions.SCAN_SNAPSHOT_ID.key(), "12")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java index dda2c3d23447a4..61fd4d83063a49 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogFactory; @@ -245,6 +247,22 @@ public void testBucket() throws Exception { Assert.assertEquals("c0", table.options().get("bucket-key")); } + @Test + public void testModifyColumnPreservesRemoteTypeBehindLossyProjection() + throws Exception { + TimestampType remoteType = new TimestampType(9); + DataField remoteField = new DataField(0, "ts", remoteType); + Column projectedColumn = new Column( + "ts", ScalarType.createDatetimeV2Type(6), true); + + org.apache.paimon.types.DataType requestedType = + ops.requestedColumnType(projectedColumn, remoteField); + + Assert.assertTrue(requestedType instanceof TimestampType); + Assert.assertEquals(9, ((TimestampType) requestedType).getPrecision()); + Assert.assertEquals(projectedColumn.isAllowNull(), requestedType.isNullable()); + } + public void createTable(String sql) throws UserException { LogicalPlan plan = new NereidsParser().parseSingle(sql); Assertions.assertTrue(plan instanceof CreateTableCommand); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonTransactionTest.java new file mode 100644 index 00000000000000..d7e9e440cf5e24 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonTransactionTest.java @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.common.UserException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.thrift.TPaimonCommitMessage; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.io.DataOutputViewStreamWrapper; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.sink.CommitMessageSerializer; +import org.apache.paimon.table.sink.TableCommitImpl; +import org.apache.paimon.utils.SnapshotManager; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.Callable; + +public class PaimonTransactionTest { + private static final long TRANSACTION_ID = 12345L; + + private PaimonWriteBinding binding; + private PaimonMetadataOps ops; + private PaimonExternalCatalog dorisCatalog; + private ExecutionAuthenticator authenticator; + private FileStoreTable table; + private TableCommitImpl committer; + private SnapshotManager snapshotManager; + + @Before + public void setUp() throws Exception { + binding = Mockito.mock(PaimonWriteBinding.class); + ops = Mockito.mock(PaimonMetadataOps.class); + dorisCatalog = Mockito.mock(PaimonExternalCatalog.class); + authenticator = Mockito.mock(ExecutionAuthenticator.class); + table = Mockito.mock(FileStoreTable.class); + committer = Mockito.mock(TableCommitImpl.class); + snapshotManager = Mockito.mock(SnapshotManager.class); + + ops.dorisCatalog = dorisCatalog; + Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.when(binding.getTable()).thenReturn(table); + Mockito.when(binding.isOverwrite()).thenReturn(true); + Mockito.when(binding.getStaticPartition()).thenReturn(Collections.emptyMap()); + Mockito.when(binding.tableName()).thenReturn("db.tbl"); + Mockito.when(table.newCommit(PaimonTransaction.commitUser(TRANSACTION_ID))) + .thenReturn(committer); + Mockito.when(table.snapshotManager()).thenReturn(snapshotManager); + Mockito.doAnswer(invocation -> { + Callable task = invocation.getArgument(0); + return task.call(); + }).when(authenticator).execute(ArgumentMatchers.>any()); + } + + @Test + public void testPublishedSnapshotReconcilesAsCommitted() throws Exception { + RuntimeException firstFailure = new RuntimeException("first atomic failure"); + RuntimeException retryFailure = new RuntimeException("retry atomic failure"); + Mockito.doThrow(firstFailure, retryFailure) + .when(committer).filterAndCommit(ArgumentMatchers.anyMap()); + Mockito.when(snapshotManager.findSnapshotsForIdentifiers( + PaimonTransaction.commitUser(TRANSACTION_ID), + Collections.singletonList(TRANSACTION_ID))) + .thenReturn(Collections.singletonList(Mockito.mock(Snapshot.class))); + + PaimonTransaction transaction = createBoundTransaction(); + transaction.commit(); + + Assert.assertEquals(PaimonTransaction.CommitState.COMMITTED, transaction.getState()); + Mockito.verify(committer, Mockito.times(2)).filterAndCommit(ArgumentMatchers.anyMap()); + transaction.rollback(); + Mockito.verify(committer, Mockito.never()).abort(ArgumentMatchers.anyList()); + } + + @Test + public void testMissingSnapshotLeavesOutcomeUnknown() throws Exception { + Mockito.doThrow(new RuntimeException("first atomic failure"), + new RuntimeException("retry atomic failure")) + .when(committer).filterAndCommit(ArgumentMatchers.anyMap()); + Mockito.when(snapshotManager.findSnapshotsForIdentifiers( + PaimonTransaction.commitUser(TRANSACTION_ID), + Collections.singletonList(TRANSACTION_ID))).thenReturn(Collections.emptyList()); + + PaimonTransaction transaction = createBoundTransaction(); + Assert.assertThrows(UserException.class, transaction::commit); + + Assert.assertEquals(PaimonTransaction.CommitState.OUTCOME_UNKNOWN, transaction.getState()); + transaction.rollback(); + Mockito.verify(committer, Mockito.never()).abort(ArgumentMatchers.anyList()); + } + + @Test + public void testPreCommitFailureRemainsAbortable() throws Exception { + Mockito.when(table.newCommit(PaimonTransaction.commitUser(TRANSACTION_ID))) + .thenThrow(new RuntimeException("pre-commit failure")) + .thenReturn(committer); + PaimonTransaction transaction = createBoundTransaction(); + transaction.updateCommitMessages(Collections.singletonList(commitPayload())); + + Assert.assertThrows(UserException.class, transaction::commit); + Assert.assertEquals(PaimonTransaction.CommitState.PREPARED, transaction.getState()); + Mockito.verify(committer, Mockito.never()).filterAndCommit(ArgumentMatchers.anyMap()); + + transaction.rollback(); + Mockito.verify(committer).abort(ArgumentMatchers.anyList()); + } + + @Test + public void testCloseFailureAfterCommitDoesNotChangeOutcome() throws Exception { + Mockito.doThrow(new RuntimeException("close failure")).when(committer).close(); + PaimonTransaction transaction = createBoundTransaction(); + + transaction.commit(); + + Assert.assertEquals(PaimonTransaction.CommitState.COMMITTED, transaction.getState()); + Mockito.verify(authenticator).execute(ArgumentMatchers.>any()); + } + + @Test + public void testCommitPayloadDedupUsesExactContent() { + byte[] first = new byte[] {0, 31}; + byte[] sameHash = new byte[] {1, 0}; + Assert.assertEquals(Arrays.hashCode(first), Arrays.hashCode(sameHash)); + + PaimonTransaction transaction = createBoundTransaction(); + transaction.updateCommitMessages(Arrays.asList( + new TPaimonCommitMessage().setPayload(first), + new TPaimonCommitMessage().setPayload(sameHash), + new TPaimonCommitMessage().setPayload(Arrays.copyOf(first, first.length)))); + + Assert.assertEquals(2, transaction.getPayloadCount()); + } + + @Test + public void testCommitUserIsNamespacedByDorisCluster() { + Assert.assertEquals( + PaimonTransaction.commitUser(10001, TRANSACTION_ID), + PaimonTransaction.commitUser(10001, TRANSACTION_ID)); + Assert.assertNotEquals( + PaimonTransaction.commitUser(10001, TRANSACTION_ID), + PaimonTransaction.commitUser(10002, TRANSACTION_ID)); + } + + private PaimonTransaction createBoundTransaction() { + PaimonTransaction transaction = new PaimonTransaction(ops, TRANSACTION_ID); + transaction.bind(binding); + return transaction; + } + + private TPaimonCommitMessage commitPayload() throws Exception { + CommitMessage message = new CommitMessageImpl( + BinaryRow.EMPTY_ROW, + 0, + 1, + DataIncrement.emptyIncrement(), + CompactIncrement.emptyIncrement()); + CommitMessageSerializer serializer = new CommitMessageSerializer(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + serializer.serializeList(Collections.singletonList(message), + new DataOutputViewStreamWrapper(output)); + byte[] serialized = output.toByteArray(); + ByteBuffer framed = ByteBuffer.allocate(12 + serialized.length); + framed.put(new byte[] {'D', 'P', 'C', 'M'}); + framed.putInt(serializer.getVersion()); + framed.putInt(serialized.length); + framed.put(serialized); + return new TPaimonCommitMessage().setPayload(framed.array()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 423c9ce2e270a4..236308d85a9260 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -17,10 +17,15 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.StructType; import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TFieldPtr; import org.apache.doris.thrift.schema.external.TSchema; @@ -28,9 +33,12 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; -import org.apache.paimon.partition.Partition; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.TableScan; import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; @@ -40,15 +48,44 @@ import org.junit.Test; import org.mockito.Mockito; +import java.time.LocalDateTime; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class PaimonUtilTest { private static final String TABLE_READ_SEQUENCE_NUMBER_ENABLED = "table-read.sequence-number.enabled"; + private static Table mockPartitionTable(Map options, DataField... partitionFields) { + Table table = Mockito.mock(Table.class); + Mockito.when(table.name()).thenReturn("mock_table"); + Mockito.when(table.partitionKeys()).thenReturn(Arrays.stream(partitionFields) + .map(DataField::name).collect(Collectors.toList())); + Mockito.when(table.rowType()).thenReturn(DataTypes.ROW(partitionFields)); + Mockito.when(table.options()).thenReturn(options); + return table; + } + + private static BinaryRow stringPartitionRow(String... values) { + BinaryRow row = new BinaryRow(values.length); + BinaryRowWriter writer = new BinaryRowWriter(row); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + writer.setNullAt(i); + } else { + writer.writeString(i, BinaryString.fromString(values[i])); + } + } + writer.complete(); + return row; + } + + private static PartitionEntry partitionEntry(BinaryRow partition, long sequence) { + return new PartitionEntry(partition, sequence, sequence, sequence, sequence); + } + @Test public void testSchemaForVarcharAndChar() { DataField c1 = new DataField(1, "c1", new VarCharType(32)); @@ -60,6 +97,38 @@ public void testSchemaForVarcharAndChar() { Assert.assertEquals(14, type2.getLength()); } + @Test + public void testTimestampWriteTypeMappingUsesDateTimeV2() { + RowType rowType = DataTypes.ROW( + DataTypes.FIELD(0, "ntz", DataTypes.TIMESTAMP(6)), + DataTypes.FIELD(1, "ltz", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)), + DataTypes.FIELD(2, "nested_ltz", + DataTypes.ARRAY(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)))); + + StructType writeType = (StructType) PaimonUtil.paimonTypeToDorisType(rowType, false, false); + + Assert.assertEquals(PrimitiveType.DATETIMEV2, + writeType.getFields().get(0).getType().getPrimitiveType()); + Assert.assertEquals(PrimitiveType.DATETIMEV2, + writeType.getFields().get(1).getType().getPrimitiveType()); + ArrayType nestedLtz = (ArrayType) writeType.getFields().get(2).getType(); + Assert.assertEquals(PrimitiveType.DATETIMEV2, + nestedLtz.getItemType().getPrimitiveType()); + } + + @Test + public void testVariantTypeMappingIncludesNestedTypes() { + RowType rowType = DataTypes.ROW( + DataTypes.FIELD(0, "direct", DataTypes.VARIANT()), + DataTypes.FIELD(1, "nested", DataTypes.ARRAY(DataTypes.VARIANT()))); + + StructType writeType = (StructType) PaimonUtil.paimonTypeToDorisType(rowType, false, false); + + Assert.assertTrue(writeType.getFields().get(0).getType().isVariantType()); + Assert.assertTrue(((ArrayType) writeType.getFields().get(1).getType()) + .getItemType().isVariantType()); + } + @Test public void testGetPartitionInfoMapSupportsFloatingPointPartitions() { DataField floatPartition = DataTypes.FIELD(0, "float_partition", DataTypes.FLOAT()); @@ -153,16 +222,17 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { new Column("source", Type.STRING), new Column("part_str", Type.STRING), new Column("pass", Type.STRING)); - Map spec = new LinkedHashMap<>(); - spec.put("source", "dataset/team-a/segment-01"); - spec.put("part_str", "/ymd=20260701/hour=[0-9][0-9]/*.jsonl"); - spec.put("pass", "s1"); - Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "source", DataTypes.STRING()), + DataTypes.FIELD(1, "part_str", DataTypes.STRING()), + DataTypes.FIELD(2, "pass", DataTypes.STRING())); + PartitionEntry partition = partitionEntry(stringPartitionRow( + "dataset/team-a/segment-01", "/ymd=20260701/hour=[0-9][0-9]/*.jsonl", "s1"), 1L); PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( - partitionColumns, Collections.singletonList(partition), false); + table, partitionColumns, Collections.singletonList(partition)); - Assert.assertFalse(partitionInfo.isPartitionInvalid()); + Assert.assertEquals(PaimonPartitionInfo.PruningStatus.PRUNABLE, partitionInfo.getPruningStatus()); Assert.assertEquals(1, partitionInfo.getNameToPartition().size()); Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" @@ -183,14 +253,15 @@ public void testGeneratePartitionInfoUsesPartitionColumnOrder() { new Column("source", Type.STRING), new Column("part_str", Type.STRING), new Column("pass", Type.STRING)); - Map spec = new LinkedHashMap<>(); - spec.put("pass", "s1"); - spec.put("part_str", "/ymd=20260721"); - spec.put("source", "dataset/team-a/segment-01"); - Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "source", DataTypes.STRING()), + DataTypes.FIELD(1, "part_str", DataTypes.STRING()), + DataTypes.FIELD(2, "pass", DataTypes.STRING())); + PartitionEntry partition = partitionEntry(stringPartitionRow( + "dataset/team-a/segment-01", "/ymd=20260721", "s1"), 1L); PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( - partitionColumns, Collections.singletonList(partition), false); + table, partitionColumns, Collections.singletonList(partition)); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260721/pass=s1"; @@ -203,14 +274,31 @@ public void testGeneratePartitionInfoUsesPartitionColumnOrder() { } @Test - public void testGeneratePartitionInfoPreservesLegacyDateConversion() { + public void testGeneratePartitionInfoUsesLegacyDateName() { List partitionColumns = Collections.singletonList(new Column("dt", Type.DATEV2)); - Map spec = new LinkedHashMap<>(); - spec.put("dt", "19737"); - Partition partition = new Partition(spec, 1L, 1L, 1L, 1L, false); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "dt", DataTypes.DATE())); + PartitionEntry partition = partitionEntry(BinaryRow.singleColumn(19737), 1L); PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( - partitionColumns, Collections.singletonList(partition), true); + table, partitionColumns, Collections.singletonList(partition)); + + String partitionName = "dt=19737"; + Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); + PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().get(partitionName); + Assert.assertEquals(Collections.singletonList("2024-01-15"), + ((ListPartitionItem) partitionItem).getItems().get(0).getPartitionValuesAsStringList()); + } + + @Test + public void testGeneratePartitionInfoUsesCanonicalDateNameWhenLegacyDisabled() { + List partitionColumns = Collections.singletonList(new Column("dt", Type.DATEV2)); + Table table = mockPartitionTable(Collections.singletonMap("partition.legacy-name", "false"), + DataTypes.FIELD(0, "dt", DataTypes.DATE())); + PartitionEntry partition = partitionEntry(BinaryRow.singleColumn(19737), 1L); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + table, partitionColumns, Collections.singletonList(partition)); String partitionName = "dt=2024-01-15"; Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); @@ -224,25 +312,22 @@ public void testGeneratePartitionInfoUsesCollisionFreePartitionNames() { List partitionColumns = Arrays.asList( new Column("a", Type.STRING), new Column("b", Type.STRING)); - Map firstSpec = new LinkedHashMap<>(); - firstSpec.put("a", "x/b=y"); - firstSpec.put("b", "z"); - Map secondSpec = new LinkedHashMap<>(); - secondSpec.put("a", "x"); - secondSpec.put("b", "y/b=z"); - Partition firstPartition = new Partition(firstSpec, 1L, 1L, 1L, 1L, false); - Partition secondPartition = new Partition(secondSpec, 2L, 2L, 2L, 2L, false); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "a", DataTypes.STRING()), + DataTypes.FIELD(1, "b", DataTypes.STRING())); + PartitionEntry firstPartition = partitionEntry(stringPartitionRow("x/b=y", "z"), 1L); + PartitionEntry secondPartition = partitionEntry(stringPartitionRow("x", "y/b=z"), 2L); PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( - partitionColumns, Arrays.asList(firstPartition, secondPartition), false); + table, partitionColumns, Arrays.asList(firstPartition, secondPartition)); String firstPartitionName = "a=x%2Fb%3Dy/b=z"; String secondPartitionName = "a=x/b=y%2Fb%3Dz"; - Assert.assertFalse(partitionInfo.isPartitionInvalid()); + Assert.assertEquals(PaimonPartitionInfo.PruningStatus.PRUNABLE, partitionInfo.getPruningStatus()); Assert.assertEquals(2, partitionInfo.getNameToPartition().size()); Assert.assertEquals(2, partitionInfo.getNameToPartitionItem().size()); - Assert.assertSame(firstPartition, partitionInfo.getNameToPartition().get(firstPartitionName)); - Assert.assertSame(secondPartition, partitionInfo.getNameToPartition().get(secondPartitionName)); + Assert.assertEquals(1L, partitionInfo.getNameToPartition().get(firstPartitionName).recordCount()); + Assert.assertEquals(2L, partitionInfo.getNameToPartition().get(secondPartitionName).recordCount()); Assert.assertEquals(Arrays.asList("x/b=y", "z"), ((ListPartitionItem) partitionInfo.getNameToPartitionItem().get(firstPartitionName)) .getItems().get(0).getPartitionValuesAsStringList()); @@ -254,16 +339,143 @@ public void testGeneratePartitionInfoUsesCollisionFreePartitionNames() { @Test public void testGeneratePartitionInfoRejectsDuplicatePartitionNames() { List partitionColumns = Collections.singletonList(new Column("part", Type.STRING)); - Map firstSpec = Collections.singletonMap("part", "same"); - Map secondSpec = Collections.singletonMap("part", "same"); - Partition firstPartition = new Partition(firstSpec, 1L, 1L, 1L, 1L, false); - Partition secondPartition = new Partition(secondSpec, 2L, 2L, 2L, 2L, false); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + PartitionEntry firstPartition = partitionEntry(stringPartitionRow("same"), 1L); + PartitionEntry secondPartition = partitionEntry(stringPartitionRow("same"), 2L); IllegalStateException exception = Assert.assertThrows(IllegalStateException.class, () -> PaimonUtil.generatePartitionInfo( - partitionColumns, Arrays.asList(firstPartition, secondPartition), false)); + table, partitionColumns, Arrays.asList(firstPartition, secondPartition))); + + Assert.assertTrue(exception.getMessage().contains("Duplicate typed Paimon partition")); + } + + @Test + public void testGeneratePartitionInfoReturnsUnprunableWithoutPartialMaps() { + List partitionColumns = Collections.singletonList(new Column("part", Type.INT)); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + List partitions = Arrays.asList( + partitionEntry(stringPartitionRow("1"), 1L), + partitionEntry(stringPartitionRow("not-an-int"), 2L)); + + PaimonPartitionInfo partitionInfo = + PaimonUtil.generatePartitionInfo(table, partitionColumns, partitions); + + Assert.assertSame(PaimonPartitionInfo.UNPRUNABLE, partitionInfo); + Assert.assertEquals(PaimonPartitionInfo.PruningStatus.UNPRUNABLE, partitionInfo.getPruningStatus()); + Assert.assertTrue(partitionInfo.getNameToPartition().isEmpty()); + Assert.assertTrue(partitionInfo.getNameToPartitionItem().isEmpty()); + } + + @Test + public void testGeneratePartitionInfoDoesNotCacheSessionZonedLtzBounds() { + List partitionColumns = + Collections.singletonList(new Column("part", Type.DATETIMEV2)); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6))); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + table, + partitionColumns, + Collections.singletonList(Mockito.mock(PartitionEntry.class))); + + Assert.assertSame(PaimonPartitionInfo.UNPRUNABLE, partitionInfo); + } + + @Test + public void testGeneratePartitionInfoSupportsTimestampWithoutTimeZone() { + List partitionColumns = Collections.singletonList( + new Column("part", org.apache.doris.catalog.ScalarType.createDatetimeV2Type(6))); + DataField partitionField = DataTypes.FIELD(0, "part", DataTypes.TIMESTAMP(9)); + Table table = mockPartitionTable(Collections.emptyMap(), partitionField); + BinaryRow partitionRow = new BinaryRow(1); + BinaryRowWriter writer = new BinaryRowWriter(partitionRow); + writer.writeTimestamp(0, Timestamp.fromLocalDateTime( + LocalDateTime.of(2026, 7, 29, 12, 34, 56, 123456789)), 9); + writer.complete(); + + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + table, partitionColumns, + Collections.singletonList(partitionEntry(partitionRow, 1L))); + + Assert.assertEquals(PaimonPartitionInfo.PruningStatus.PRUNABLE, + partitionInfo.getPruningStatus()); + PartitionItem partitionItem = + partitionInfo.getNameToPartitionItem().values().iterator().next(); + Assert.assertEquals(Collections.singletonList("2026-07-29 12:34:56.123456"), + ((ListPartitionItem) partitionItem).getItems().get(0) + .getPartitionValuesAsStringList()); + } + + @Test + public void testGeneratePartitionInfoSupportsTimestampWithoutFraction() { + List partitionColumns = Collections.singletonList( + new Column("part", org.apache.doris.catalog.ScalarType.createDatetimeV2Type(0))); + DataField partitionField = DataTypes.FIELD(0, "part", DataTypes.TIMESTAMP(0)); + Table table = mockPartitionTable(Collections.emptyMap(), partitionField); + BinaryRow partitionRow = new BinaryRow(1); + BinaryRowWriter writer = new BinaryRowWriter(partitionRow); + writer.writeTimestamp(0, Timestamp.fromLocalDateTime( + LocalDateTime.of(2026, 7, 29, 12, 34, 56)), 0); + writer.complete(); - Assert.assertTrue(exception.getMessage().contains("Duplicate Paimon partition name")); + PaimonPartitionInfo partitionInfo = PaimonUtil.generatePartitionInfo( + table, partitionColumns, + Collections.singletonList(partitionEntry(partitionRow, 1L))); + + Assert.assertEquals(PaimonPartitionInfo.PruningStatus.PRUNABLE, + partitionInfo.getPruningStatus()); + PartitionItem partitionItem = + partitionInfo.getNameToPartitionItem().values().iterator().next(); + Assert.assertEquals(Collections.singletonList("2026-07-29 12:34:56"), + ((ListPartitionItem) partitionItem).getItems().get(0) + .getPartitionValuesAsStringList()); + } + + @Test + public void testGeneratePartitionInfoReturnsUnprunableForAmbiguousDisplayNames() { + String defaultPartitionName = "__CUSTOM_DEFAULT_PARTITION__"; + List partitionColumns = Collections.singletonList(new Column("region", Type.STRING)); + Table table = mockPartitionTable( + Collections.singletonMap("partition.default-name", defaultPartitionName), + DataTypes.FIELD(0, "region", DataTypes.STRING())); + List partitions = Arrays.asList( + partitionEntry(stringPartitionRow((String) null), 1L), + partitionEntry(stringPartitionRow(""), 2L), + partitionEntry(stringPartitionRow("null"), 3L), + partitionEntry(stringPartitionRow(defaultPartitionName), 4L)); + + PaimonPartitionInfo partitionInfo = + PaimonUtil.generatePartitionInfo(table, partitionColumns, partitions); + + Assert.assertSame(PaimonPartitionInfo.UNPRUNABLE, partitionInfo); + Assert.assertTrue(partitionInfo.getNameToPartition().isEmpty()); + Assert.assertTrue(partitionInfo.getNameToPartitionItem().isEmpty()); + } + + @Test + public void testPartitionInfoLoaderUsesSnapshotTableEntries() throws Exception { + List partitionColumns = + Collections.singletonList(new Column("region", Type.STRING)); + Table snapshotTable = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "region", DataTypes.STRING())); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotTable.newReadBuilder()).thenReturn(readBuilder); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenReturn( + Collections.singletonList(partitionEntry(stringPartitionRow("east"), 1L))); + + NameMapping nameMapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonPartitionInfo partitionInfo = + new PaimonPartitionInfoLoader().load(nameMapping, snapshotTable, partitionColumns); + + Assert.assertEquals(PaimonPartitionInfo.PruningStatus.PRUNABLE, partitionInfo.getPruningStatus()); + Assert.assertTrue(partitionInfo.getNameToPartitionItem().containsKey("region=east")); + Mockito.verify(tableScan).listPartitionEntries(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonWriteBindingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonWriteBindingTest.java new file mode 100644 index 00000000000000..f81426fe0484bc --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonWriteBindingTest.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.qe.ConnectContext; + +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.TypeUtils; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class PaimonWriteBindingTest { + + @Test + public void testStaticPartitionUsesValueAfterTargetTypeCast() + throws Exception { + FileStoreTable table = mockPartitionTable( + Collections.emptyMap(), DataTypes.FIELD(0, "part", DataTypes.INT())); + Map writeTypes = + Collections.singletonMap("part", Type.INT); + Map partition = + Collections.singletonMap("part", BooleanLiteral.TRUE); + + Map resolved = PaimonWriteBinding.resolveStaticPartition( + table, writeTypes, partition, true); + + Assert.assertEquals("1", resolved.get("part")); + } + + @Test + public void testStaticPartitionNullUsesPaimonDefaultName() + throws Exception { + String defaultName = "__CUSTOM_DEFAULT__"; + FileStoreTable table = mockPartitionTable( + Collections.singletonMap("partition.default-name", defaultName), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + + Map resolved = PaimonWriteBinding.resolveStaticPartition( + table, + Collections.singletonMap("part", Type.STRING), + Collections.singletonMap("part", new NullLiteral()), + true); + + Assert.assertEquals(defaultName, resolved.get("part")); + } + + @Test + public void testStaticOverwriteRejectsLiteralDefaultName() + throws Exception { + String defaultName = "__CUSTOM_DEFAULT__"; + FileStoreTable table = mockPartitionTable( + Collections.singletonMap("partition.default-name", defaultName), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + + AnalysisException exception = Assert.assertThrows( + AnalysisException.class, + () -> PaimonWriteBinding.resolveStaticPartition( + table, + Collections.singletonMap("part", Type.STRING), + Collections.singletonMap( + "part", new StringLiteral(defaultName)), + true)); + + Assert.assertTrue(exception.getMessage().contains("cannot be represented")); + } + + @Test + public void testStaticLtzPartitionUsesSdkZoneAndPaimonFormat() + throws Exception { + ConnectContext previousContext = ConnectContext.get(); + ConnectContext context = new ConnectContext(); + context.getSessionVariable().setTimeZone("Asia/Shanghai"); + context.setThreadLocalInfo(); + try { + FileStoreTable table = mockPartitionTable( + Collections.emptyMap(), + DataTypes.FIELD(0, "part", + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6))); + String input = "2024-01-15 08:30:45.123456"; + + Map resolved = PaimonWriteBinding.resolveStaticPartition( + table, + Collections.singletonMap( + "part", org.apache.doris.catalog.ScalarType.createDatetimeV2Type(6)), + Collections.singletonMap( + "part", new DateTimeV2Literal(input)), + true); + + String expected = LocalDateTime.parse( + input.replace(' ', 'T'), DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .atZone(ZoneId.of("Asia/Shanghai")) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime() + .format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + .replace('T', ' '); + String value = resolved.get("part"); + Assert.assertEquals(expected, value); + Assert.assertFalse(value.contains("T")); + Assert.assertNotNull(TypeUtils.castFromString( + value, table.rowType().getTypeAt(0))); + } finally { + if (previousContext == null) { + ConnectContext.remove(); + } else { + previousContext.setThreadLocalInfo(); + } + } + } + + private static FileStoreTable mockPartitionTable( + Map options, + org.apache.paimon.types.DataField field) { + FileStoreTable table = Mockito.mock(FileStoreTable.class); + Mockito.when(table.partitionKeys()) + .thenReturn(Collections.singletonList(field.name())); + Mockito.when(table.rowType()).thenReturn(DataTypes.ROW(field)); + Mockito.when(table.options()).thenReturn(new HashMap<>(options)); + return table; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonWriteTargetTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonWriteTargetTest.java new file mode 100644 index 00000000000000..d4ccecf1fb11e6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonWriteTargetTest.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.common.AnalysisException; + +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataTypes; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; + +public class PaimonWriteTargetTest { + + @Test + public void testVariantColumnIsAvailableToWriteBinding() throws Exception { + PaimonWriteTarget target = createTarget( + DataTypes.FIELD(0, "payload", DataTypes.VARIANT())); + + Assert.assertTrue(target.getColumn("payload").getType().isVariantType()); + Assert.assertTrue(target.getColumnTypes().get("payload").isVariantType()); + } + + @Test + public void testCaseInsensitiveColumnCollisionIsRejected() { + AnalysisException exception = Assert.assertThrows( + AnalysisException.class, + () -> createTarget( + DataTypes.FIELD(0, "payload", DataTypes.INT()), + DataTypes.FIELD(1, "PAYLOAD", DataTypes.BIGINT()))); + + Assert.assertTrue(exception.getMessage().contains( + "columns which differ only by case: payload and PAYLOAD")); + } + + private static PaimonWriteTarget createTarget( + org.apache.paimon.types.DataField... fields) throws Exception { + PaimonExternalTable dorisTable = Mockito.mock(PaimonExternalTable.class); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + Mockito.when(dorisTable.getCatalog()).thenReturn(catalog); + Mockito.when(dorisTable.getPaimonTableForWrite()).thenReturn(table); + Mockito.when(table.rowType()).thenReturn(DataTypes.ROW(fields)); + Mockito.when(table.partitionKeys()).thenReturn(Collections.emptyList()); + return PaimonWriteTarget.create(dorisTable); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index bceddc8e16f95b..24f0fcf88a8bc4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -284,6 +284,21 @@ public void testGeneratePartitionName() { ); String rangeName = MTMVPartitionUtil.generatePartitionName(rangeDesc); Assert.assertEquals("p_1_2", rangeName); + + PartitionKeyDesc nullDesc = PartitionKeyDesc.createIn( + Lists.>newArrayList( + Lists.newArrayList(new PartitionValue("NULL", true)))); + PartitionKeyDesc literalNullDesc = PartitionKeyDesc.createIn( + Lists.>newArrayList( + Lists.newArrayList(new PartitionValue("NULL")))); + PartitionKeyDesc legacySuffixCollisionDesc = PartitionKeyDesc.createIn( + Lists.>newArrayList( + Lists.newArrayList(new PartitionValue("NULL,literal")))); + Assert.assertEquals("p_NULL", MTMVPartitionUtil.generatePartitionName(nullDesc)); + String literalNullName = MTMVPartitionUtil.generatePartitionName(literalNullDesc); + Assert.assertEquals(literalNullName, MTMVPartitionUtil.generatePartitionName(literalNullDesc)); + Assert.assertNotEquals(MTMVPartitionUtil.generatePartitionName(nullDesc), literalNullName); + Assert.assertNotEquals(MTMVPartitionUtil.generatePartitionName(legacySuffixCollisionDesc), literalNullName); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParseInsertPartitionSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParseInsertPartitionSpecTest.java index 185141d687563d..faf7a2938f4522 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParseInsertPartitionSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParseInsertPartitionSpecTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.parser; import org.apache.doris.nereids.DorisParser; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.qe.ConnectContext; @@ -30,6 +31,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Map; @@ -268,4 +270,16 @@ public void testParseStaticPartitionMixedTypes() throws Exception { Assertions.assertTrue(staticValues.get("month") instanceof StringLikeLiteral); Assertions.assertEquals("01", ((StringLikeLiteral) staticValues.get("month")).getStringValue()); } + + @Test + public void testRejectCaseInsensitiveDuplicateStaticPartitionColumns() throws Exception { + String sql = "INSERT OVERWRITE TABLE tbl " + + "PARTITION (region='east', REGION='west') SELECT * FROM src"; + Object ctx = parsePartitionSpec(sql); + + InvocationTargetException exception = Assertions.assertThrows( + InvocationTargetException.class, () -> invokeParseInsertPartitionSpec(ctx)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + Assertions.assertEquals("Duplicate partition column: REGION", exception.getCause().getMessage()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java new file mode 100644 index 00000000000000..842428d79047bb --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPaimonTableSinkTest.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.table.FileStoreTable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +class PhysicalPaimonTableSinkTest { + + @Test + void testFixedAppendCompactionRequiresSingleWriter() { + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.HASH_FIXED, Collections.emptyList(), Collections.emptyMap()))); + Assertions.assertFalse(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.HASH_FIXED, Collections.emptyList(), + Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true")))); + } + + @Test + void testFixedPrimaryKeyAndDynamicModesRequireSingleWriter() { + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.HASH_FIXED, Collections.singletonList("id"), + Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true")))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.HASH_DYNAMIC, Collections.singletonList("id"), + Collections.emptyMap()))); + Assertions.assertTrue(PhysicalPaimonTableSink.requiresSingleWriter( + table(BucketMode.KEY_DYNAMIC, Collections.singletonList("id"), + Collections.emptyMap()))); + } + + private static FileStoreTable table( + BucketMode bucketMode, List primaryKeys, Map options) { + FileStoreTable table = Mockito.mock(FileStoreTable.class); + Mockito.when(table.bucketMode()).thenReturn(bucketMode); + Mockito.when(table.primaryKeys()).thenReturn(primaryKeys); + Mockito.when(table.options()).thenReturn(options); + return table; + } +} diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index a0cb46f11669d9..47ad91e3f274ff 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -46,6 +46,7 @@ enum TDataSinkType { MAXCOMPUTE_TABLE_SINK = 18, ICEBERG_DELETE_SINK = 19, ICEBERG_MERGE_SINK = 20, + PAIMON_TABLE_SINK = 21, } enum TResultSinkType { @@ -621,6 +622,30 @@ struct TMaxComputeTableSink { 18: optional i64 txn_id // FE external transaction ID for runtime block_id allocation } +enum TPaimonWriteBackendType { + JNI = 0, + FFI = 1, +} + +enum TPaimonWriteMode { + APPEND = 0, + OVERWRITE = 1, +} + +struct TPaimonCommitMessage { + 1: optional binary payload // Paimon native CommitMessageSerializer bytes (DPCM-framed) +} + +struct TPaimonTableSink { + 1: optional string serialized_table // required at runtime; serialized Paimon Table object (base64) + 2: optional map hadoop_config + 3: optional list column_names + 4: optional TPaimonWriteBackendType backend_type + 5: optional TPaimonWriteMode write_mode + 6: optional i64 transaction_id + 7: optional string commit_user +} + struct TDataSink { 1: required TDataSinkType type 2: optional TDataStreamSink stream_sink @@ -641,4 +666,5 @@ struct TDataSink { 18: optional TMaxComputeTableSink max_compute_table_sink 19: optional TIcebergDeleteSink iceberg_delete_sink 20: optional TIcebergMergeSink iceberg_merge_sink + 21: optional TPaimonTableSink paimon_table_sink } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 61b5b00b04375c..bf119f414f9ae8 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -330,6 +330,8 @@ struct TReportExecStatusParams { 32: optional list mc_commit_datas 33: optional string first_error_msg + + 34: optional list paimon_commit_messages } struct TFeResult { diff --git a/regression-test/data/external_table_p0/paimon/paimon_schema_change_ddl.out b/regression-test/data/external_table_p0/paimon/paimon_schema_change_ddl.out new file mode 100644 index 00000000000000..7718c425e02479 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/paimon_schema_change_ddl.out @@ -0,0 +1,125 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !paimon_alter_initial_desc -- +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N + +-- !paimon_alter_initial_schema -- +0 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT NOT NULL","description":""},{"id":2,"name":"score","type":"INT","description":"initial score","defaultValue":"1"},{"id":3,"name":"MixedCase","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(8, 2)","description":""}] [] ["id"] + +-- !paimon_alter_add_column_desc -- +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N + +-- !paimon_alter_add_column_schema -- +1 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT NOT NULL","description":""},{"id":2,"name":"score","type":"INT","description":"initial score","defaultValue":"1"},{"id":6,"name":"added_after","type":"STRING","description":"added column","defaultValue":"unknown"},{"id":3,"name":"MixedCase","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(8, 2)","description":""}] + +-- !paimon_alter_add_columns_desc -- +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small column +profile struct Yes true \N + +-- !paimon_alter_add_first_desc -- +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +MixedCase text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small column +profile struct Yes true \N + +-- !paimon_alter_rename_column_desc -- +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +score int Yes true \N initial score +added_after text Yes true \N added column +display_name text Yes true \N +obsolete text Yes true \N +amount decimal(8,2) Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small column +profile struct Yes true \N + +-- !paimon_alter_modify_column_desc -- +score bigint Yes true \N updated score +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +added_after text Yes true \N +display_name text Yes true \N +obsolete text Yes true \N +amount decimal(12,2) Yes true \N +tiny_col tinyint Yes true \N +small_col int Yes true \N +profile struct Yes true \N + +-- !paimon_alter_modify_column_schema -- +9 [{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":12,"name":"first_col","type":"BIGINT","description":""},{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":4,"name":"obsolete","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""}] + +-- !paimon_alter_drop_column_desc -- +score bigint Yes true \N updated score +first_col bigint Yes true \N +id int Yes true \N identifier +required_value bigint Yes true \N +added_after text Yes true \N +display_name text Yes true \N +amount decimal(12,2) Yes true \N +tiny_col tinyint Yes true \N +small_col int Yes true \N +profile struct Yes true \N + +-- !paimon_alter_reorder_columns_desc -- +id int Yes true \N identifier +display_name text Yes true \N +score bigint Yes true \N updated score +required_value bigint Yes true \N +small_col int Yes true \N +tiny_col tinyint Yes true \N +added_after text Yes true \N +amount decimal(12,2) Yes true \N +profile struct Yes true \N +first_col bigint Yes true \N + +-- !paimon_alter_final_schema -- +11 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""},{"id":12,"name":"first_col","type":"BIGINT","description":""}] [] ["id"] + +-- !paimon_alter_failed_batch_schema -- +11 [{"id":0,"name":"id","type":"INT NOT NULL","description":"identifier"},{"id":3,"name":"display_name","type":"STRING","description":""},{"id":2,"name":"score","type":"BIGINT","description":"updated score","defaultValue":"10"},{"id":1,"name":"required_value","type":"BIGINT","description":""},{"id":8,"name":"small_col","type":"INT","description":""},{"id":7,"name":"tiny_col","type":"TINYINT","description":""},{"id":6,"name":"added_after","type":"STRING","description":""},{"id":5,"name":"amount","type":"DECIMAL(12, 2)","description":""},{"id":9,"name":"profile","type":{"type":"ROW","fields":[{"id":10,"name":"city","type":"STRING","description":""},{"id":11,"name":"zip","type":"INT","description":""}]},"description":""},{"id":12,"name":"first_col","type":"BIGINT","description":""}] + +-- !paimon_alter_partition_initial_desc -- +id int Yes true \N +pt text Yes true \N +payload int Yes true \N + +-- !paimon_alter_partition_initial_schema -- +0 [{"id":0,"name":"id","type":"INT NOT NULL","description":""},{"id":1,"name":"pt","type":"STRING NOT NULL","description":""},{"id":2,"name":"payload","type":"INT","description":""}] ["pt"] ["id","pt"] + +-- !paimon_alter_partition_final_desc -- +id int Yes true \N +pt text Yes true \N +payload bigint Yes true \N +extra text Yes true \N + +-- !paimon_alter_partition_final_schema -- +2 [{"id":0,"name":"id","type":"INT NOT NULL","description":""},{"id":1,"name":"pt","type":"STRING NOT NULL","description":""},{"id":2,"name":"payload","type":"BIGINT","description":""},{"id":3,"name":"extra","type":"STRING","description":""}] ["pt"] ["id","pt"] + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out b/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out index 3230ad01841d21..a3132a2c1b33de 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show_partition_sales_by_date -- -sale_date=2024-01-15 sale_date 2 2216 1 -sale_date=2024-01-16 sale_date 2 4217 2 -sale_date=2024-01-17 sale_date 1 2118 1 +sale_date=19737 sale_date 2 2216 1 +sale_date=19738 sale_date 2 4217 2 +sale_date=19739 sale_date 1 2118 1 -- !show_partition_sales_by_region -- region=China-Beijing region 1 2425 1 @@ -10,12 +10,12 @@ region=Japan-Tokyo region 1 2420 1 region=USA-California region 1 2454 1 -- !show_partition_sales_by_date_region -- -sale_date=2024-01-15/region=China-Beijing sale_date,region 1 2627 1 -sale_date=2024-01-15/region=Japan-Tokyo sale_date,region 1 2614 1 -sale_date=2024-01-15/region=USA-California sale_date,region 1 2655 1 -sale_date=2024-01-16/region=China-Shanghai sale_date,region 1 2636 1 -sale_date=2024-01-16/region=Japan-Osaka sale_date,region 1 2636 1 -sale_date=2024-01-16/region=USA-New York sale_date,region 1 2643 1 +sale_date=19737/region=China-Beijing sale_date,region 1 2627 1 +sale_date=19737/region=Japan-Tokyo sale_date,region 1 2614 1 +sale_date=19737/region=USA-California sale_date,region 1 2655 1 +sale_date=19738/region=China-Shanghai sale_date,region 1 2636 1 +sale_date=19738/region=Japan-Osaka sale_date,region 1 2636 1 +sale_date=19738/region=USA-New York sale_date,region 1 2643 1 -- !show_partition_events_by_hour -- hour_partition=2024-01-15-10 hour_partition 2 2361 1 @@ -27,4 +27,3 @@ year_val=2024/month_val=1/day_val=15 year_val,month_val,day_val 2 2841 1 year_val=2024/month_val=1/day_val=16 year_val,month_val,day_val 2 5323 2 year_val=2024/month_val=1/day_val=17 year_val,month_val,day_val 1 2658 1 year_val=2024/month_val=2/day_val=1 year_val,month_val,day_val 1 2686 1 - diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out b/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out index f1118d0bd7069e..fbce9299699d50 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out @@ -6,10 +6,14 @@ -- !before_snapshots -- 1 --- !after_rows -- +-- !after_append_rows -- 1 10 base-1 2 20 base-2 +3 30 insert-values +4 40 insert-select --- !after_snapshots -- -1 +-- !after_rows -- +5 50 overwrite +-- !after_snapshots -- +4 diff --git a/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog_write.out b/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog_write.out new file mode 100644 index 00000000000000..0b8ab9001cc77d --- /dev/null +++ b/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog_write.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !hms_append -- +1 alice 95.5 east +2 bob 87 west +3 charlie 92.3 east +4 diana 88 north +5 erin 86.5 south +6 \N \N east + +-- !hms_pk -- +1 100 click_updated 99 +1 200 view 2 +2 100 click 3 diff --git a/regression-test/data/external_table_p2/paimon/test_paimon_rest_catalog_write.out b/regression-test/data/external_table_p2/paimon/test_paimon_rest_catalog_write.out new file mode 100644 index 00000000000000..b9f0934e9359b0 --- /dev/null +++ b/regression-test/data/external_table_p2/paimon/test_paimon_rest_catalog_write.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rest_append -- +1 alice 95.5 east +2 bob 87 west +3 charlie 92.3 east +4 diana 88 north +5 erin 86.5 south +6 \N \N east + +-- !rest_pk -- +1 100 click_updated 99 +1 200 view 2 +2 100 click 3 diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out b/regression-test/data/mtmv_p0/test_paimon_mtmv.out index 5c7547c0687c86..411a8667c4a581 100644 --- a/regression-test/data/mtmv_p0/test_paimon_mtmv.out +++ b/regression-test/data/mtmv_p0/test_paimon_mtmv.out @@ -137,6 +137,8 @@ true -- !null_partition -- 1 bj +2 \N +3 \N 4 null 5 NULL @@ -148,4 +150,3 @@ true -- !date_partition -- 1 2020-01-01 - diff --git a/regression-test/data/paimon_write/test_paimon_create_ddl_write_properties.out b/regression-test/data/paimon_write/test_paimon_create_ddl_write_properties.out new file mode 100644 index 00000000000000..15b6d4b36d7666 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_create_ddl_write_properties.out @@ -0,0 +1,84 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !create_custom_location_absent -- + +-- !create_sequence_result -- +1 100 newer p1 +2 20 updated-2 p1 +3 5 initial-3 p2 + +-- !create_sequence_schema -- +["dt"] ["id","dt"] created by Doris with write properties + +-- !create_sequence_file_format -- +orc + +-- !create_partial_result -- +1 alice 15 updated +2 bob 20 initial + +-- !create_first_row_result -- +1 first-1 10 +2 first-2 20 +3 first-3 30 + +-- !create_aggregation_result -- +1 17 90 latest +2 8 70 stable +3 4 50 new + +-- !create_lookup_changelog -- ++I 1 new 11 ++I 2 stable 20 ++I 3 added 30 + +-- !create_lookup_result -- +1 new 11 +2 stable 20 +3 added 30 + +-- !create_dynamic_bucket_result -- +12 0 11 + +-- !create_dynamic_bucket_rows -- +p1 0 v0 +p1 1 v1 +p1 10 v10 +p1 11 v11 +p1 2 v2 +p1 3 v3 +p1 4 v4 +p1 5 v5 +p1 6 v6 +p1 7 v7 +p1 8 v8 +p1 9 v9 + +-- !create_dynamic_bucket_files -- +0 +1 +2 +3 + +-- !create_write_options -- +aggregation bucket 1 +aggregation fields.highest.aggregate-function max +aggregation fields.total.aggregate-function sum +aggregation merge-engine aggregation +dynamic_bucket bucket -1 +dynamic_bucket dynamic-bucket.initial-buckets 1 +dynamic_bucket dynamic-bucket.max-buckets 4 +dynamic_bucket dynamic-bucket.target-row-num 2 +first_row bucket 1 +first_row merge-engine first-row +lookup bucket 1 +lookup changelog-producer lookup +partial_update bucket 2 +partial_update bucket-key id +partial_update merge-engine partial-update +sequence bucket 2 +sequence bucket-key id +sequence file.format orc +sequence sequence.field seq +sequence snapshot.num-retained.max 5 +sequence snapshot.num-retained.min 2 + diff --git a/regression-test/data/paimon_write/test_paimon_write_append_only.out b/regression-test/data/paimon_write/test_paimon_write_append_only.out new file mode 100644 index 00000000000000..f7e90eee87fab0 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_append_only.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ao_basic -- +1 alice 95.5 +2 bob 87 +3 charlie 92.3 + +-- !ao_part -- +1 alice 95.5 east +2 bob 87 west +3 charlie 92.3 east +4 diana 88 north +5 erin 86.5 south +6 \N \N east + +-- !ao_auto_partition_data -- +1 alpha 2026-07-01 +2 beta 2026-07-02 +3 gamma 2026-07-01 +4 delta 2026-07-01 +5 epsilon 2026-07-03 +6 default_partition \N + +-- !ao_auto_partition_metadata -- +{2026-07-01} 3 +{2026-07-02} 1 +{2026-07-03} 1 +{null} 1 + +-- !ao_empty -- +1 \N +2 reordered + +-- !ao_default_value -- +1 unknown + +-- !ao_default_after_explicit_null -- +1 unknown + +-- !ao_partition_default_data -- +1 omitted-partition 2026-07-01 + +-- !ao_partition_default_metadata -- +{2026-07-01} 1 diff --git a/regression-test/data/paimon_write/test_paimon_write_bucket_modes.out b/regression-test/data/paimon_write/test_paimon_write_bucket_modes.out new file mode 100644 index 00000000000000..425fdf51f4c99e --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_bucket_modes.out @@ -0,0 +1,56 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bucket_hash_fixed -- +16 0 15 2 + +-- !bucket_hash_dynamic -- +p1 1 v1_updated +p1 2 v2 +p1 3 v3 +p1 4 v4_updated +p1 5 v5 +p1 6 v6 +p2 1 p2_v1 +p2 2 p2_v2_updated + +-- !bucket_hash_dynamic_partial -- +p1 1 alice 15 +p1 2 bob 20 +p1 3 \N 30 + +-- !bucket_hash_dynamic_overwrite -- +10 new_10 +11 new_11 +12 new_12 + +-- !bucket_key_dynamic -- +p2 1 id1_moved +p2 2 id2_stable +p2 4 id4_added +p3 3 id3_moved + +-- !bucket_key_dynamic_partial -- +p1 10 old_10 15 +p2 20 stable_20 20 +p3 30 \N 30 + +-- !bucket_key_dynamic_first_row -- +p1 1 first_1 +p2 2 first_2 + +-- !bucket_key_dynamic_aggregation -- +p1 1 17 +p2 2 20 + +-- !bucket_key_dynamic_scale_samples -- +p2 1023 txn2_1023 +p2 2047 txn2_2047 +p3 0 txn2_0 +p4 3071 txn3_3071 +p4 4095 txn3_4095 +p5 2048 txn3_2048 + +-- !bucket_unaware -- +32 0 31 2 + +-- !bucket_postpone -- +0 diff --git a/regression-test/data/paimon_write/test_paimon_write_changelog_producer.out b/regression-test/data/paimon_write/test_paimon_write_changelog_producer.out new file mode 100644 index 00000000000000..590dd61fdc77ce --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_changelog_producer.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !changelog_input_partial -- +1 alice 15 +2 bob 20 +3 \N 30 + +-- !changelog_lookup -- +1 new 11 +2 stable 20 +3 added 30 + +-- !changelog_lookup_aggregation -- +1 17 +2 20 +3 30 + +-- !changelog_full_compaction -- +p1 1 new +p2 2 stable +p2 3 added + +-- !changelog_full_compaction_dynamic -- +p1 1 new_1 +p1 2 stable_2 +p1 4 added_4 +p2 3 new_3 +p2 5 added_5 diff --git a/regression-test/data/paimon_write/test_paimon_write_compaction.out b/regression-test/data/paimon_write/test_paimon_write_compaction.out new file mode 100644 index 00000000000000..3cddf7ced90981 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_compaction.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !compaction_pk -- +1 new 11 +2 stable 20 +3 added 30 + +-- !compaction_append -- +1 a +2 b +3 c +4 d diff --git a/regression-test/data/paimon_write/test_paimon_write_complex_types.out b/regression-test/data/paimon_write/test_paimon_write_complex_types.out new file mode 100644 index 00000000000000..268be0627e7f44 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_complex_types.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !cx_array -- +1 [1, 2, 3] ["a", "b", "c"] [1.1, 2.2] +2 [] [] [] +3 [10, null, 30] ["x", null, "z"] [null, 2] +4 \N \N \N + +-- !cx_map -- +1 {"math":90, "eng":95} {1:"one", 2:"two"} +2 {} {} +3 {"science":null} {3:null} +4 \N \N + +-- !cx_struct -- +1 {"name":"alice", "age":30} +2 {"name":null, "age":null} +3 \N + +-- !cx_nested -- +1 {"group2":[3, 4, 5], "group1":[1, 2]} +2 {"empty":[]} +3 \N + +-- !cx_recursive -- +1 [1.250000, -2.500000] ["2024-01-01", "2024-12-31"] ["2024-01-01 01:02:03.123456", "2024-12-31 23:59:59.654321"] {1.25:2.50, -3.75:4.00} {"flag":1, "amount":123.456789, "event_date":"2024-02-29", "event_time":"2024-02-29 12:34:56.000001"} {"term":[{"score":90, "label":"good"}, {"score":95, "label":"better"}]} +2 [null, 0.000001] [null, "1970-01-01"] [null, "1970-01-01 00:00:00.000001"] {5.25:null} {"flag":null, "amount":null, "event_date":null, "event_time":null} {"nullable":[{"score":null, "label":null}]} +3 [] [] [] {} {"flag":0, "amount":0.000000, "event_date":"1970-01-01", "event_time":"1970-01-01 00:00:00.000000"} {} +4 [8.800008] ["2025-01-01"] ["2025-01-01 00:00:00.000008"] {8.80:9.90} {"flag":1, "amount":8.800000, "event_date":"2025-01-01", "event_time":"2025-01-01 08:08:08.000008"} {"reverse":[{"score":88, "label":"reordered"}]} +5 \N ["2026-01-01"] \N \N \N {"partial":[{"score":77, "label":"subset"}]} + +-- !cx_binary -- +1 0001FEFF 2 41 1 102030 binary_1 DEADBEEF +2 \N 0 \N 0 \N binary_2 \N +3 E4B8ADE69687 \N \N \N \N \N \N +4 060708 2 03 1 0102 reordered ABCD + diff --git a/regression-test/data/paimon_write/test_paimon_write_edge_cases.out b/regression-test/data/paimon_write/test_paimon_write_edge_cases.out new file mode 100644 index 00000000000000..3d0418d4724068 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_edge_cases.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !edge_str -- +1 +2 short_str +3 abcdefghij +4 max10chars + +-- !edge_numeric -- +1 127 32767 2147483647 9223372036854775807 +2 -128 -32768 -2147483648 -9223372036854775808 +3 0 0 0 0 + +-- !edge_bool -- +1 true +2 false +3 \N + +-- !edge_pk_null -- +1 first +2 updated +3 third + +-- !edge_mixed -- +1 a 10 +2 b 20 +3 c 30 +4 a_copy 40 +5 b_copy 50 +6 c_copy 60 + diff --git a/regression-test/data/paimon_write/test_paimon_write_failures.out b/regression-test/data/paimon_write/test_paimon_write_failures.out new file mode 100644 index 00000000000000..02d2e530b2672a --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_failures.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !failure_atomic_before -- +1 baseline p0 + +-- !failure_atomic_snapshot_before -- +1 + +-- !failure_atomic_after -- +1 baseline p0 + +-- !failure_atomic_snapshot_after -- +1 + +-- !failure_recovered -- +1 baseline p0 +5 recovered p5 + +-- !failure_recovered_snapshot -- +2 + +-- !failure_overwrite_after -- +1 baseline p0 +5 recovered p5 + +-- !failure_overwrite_snapshot_after -- +2 + +-- !failure_pk_recovered -- +1 valid_after_failure + +-- !failure_pk_snapshot -- +1 diff --git a/regression-test/data/paimon_write/test_paimon_write_merge_engine.out b/regression-test/data/paimon_write/test_paimon_write_merge_engine.out new file mode 100644 index 00000000000000..9354ca3d8e5e02 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_merge_engine.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partial_update -- +1 alice 15.5 score_updated +2 bob_full 25 full_update +3 charlie \N \N + +-- !first_row -- +1 first_1 10 +2 first_2 20 +3 first_3 30 + +-- !aggregation -- +1 37 95 latest_1 +2 8 80 first_2 +3 7 60 first_3 diff --git a/regression-test/data/paimon_write/test_paimon_write_pk.out b/regression-test/data/paimon_write/test_paimon_write_pk.out new file mode 100644 index 00000000000000..c7ba2e52a398fd --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_pk.out @@ -0,0 +1,52 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !pk_dedup -- +1 alice 95.5 +2 bob 87 +3 charlie 92.3 +4 diana 91 +5 eve 85 + +-- !pk_interleaved -- +100 key100_v3 12 3000 +200 key200_v2 21 2000 + +-- !pk_bucket -- +0 row0 +1 row1 +10 row10 +11 row11 +12 row12 +13 row13 +14 row14 +15 row15 +16 row16 +17 row17 +18 row18 +19 row19 +2 row2 +3 row3 +4 row4 +5 row5 +6 row6 +7 row7 +8 row8 +9 row9 + +-- !pk_composite -- +1 100 click_updated 99 +1 200 view 2 +2 100 click 3 + +-- !pk_string_bucket -- +alpha 1 alpha_v2 +beta 2 中文_payload +emoji_😀 3 emoji_payload + +-- !pk_writer_scaling_plan -- +PhysicalPaimonTableSink +--PhysicalDistribute[DistributionSpecGather] +----PhysicalProject +------PhysicalTVFRelation + +-- !pk_writer_scaling -- +1 1 1 4096 4096 diff --git a/regression-test/data/paimon_write/test_paimon_write_schema_change.out b/regression-test/data/paimon_write/test_paimon_write_schema_change.out new file mode 100644 index 00000000000000..a7d17ed3786bb3 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_schema_change.out @@ -0,0 +1,532 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sc_append_initial -- +1 100 alice 10 1.10 old-a 2026-07-01 +2 200 bob 20 2.20 old-b 2026-07-02 + +-- !sc_add_after_before_insert -- +1 100 alice 10 \N 1.10 old-a 2026-07-01 +2 200 bob 20 \N 2.20 old-b 2026-07-02 + +-- !sc_add_after_after_insert -- +1 100 alice 10 \N 1.10 old-a 2026-07-01 +2 200 bob 20 \N 2.20 old-b 2026-07-02 +3 300 carol 30 added-3 3.30 old-c 2026-07-03 +4 400 dave 40 unknown 4.40 old-d 2026-07-01 + +-- !sc_add_default_omitted -- +100 unknown + +-- !sc_add_first_before_insert -- +1 \N 100 alice 10 \N 1.10 old-a 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d 2026-07-01 + +-- !sc_add_first_after_insert -- +1 \N 100 alice 10 \N 1.10 old-a 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d 2026-07-01 +5 5000 500 erin 50 added-first 5.50 old-e 2026-07-04 + +-- !sc_add_columns_before_insert -- +1 \N 100 alice 10 \N 1.10 old-a \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 old-e \N \N 2026-07-04 + +-- !sc_add_columns_after_insert -- +1 \N 100 alice 10 \N 1.10 old-a \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 old-default \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 old-b \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 old-c \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 old-d \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 old-e \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 old-f 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 old-partial \N \N 2026-07-05 + +-- !sc_drop_before_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 + +-- !sc_drop_after_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 +7 7000 700 grace 70 after-drop 7.70 7 700 2026-07-06 + +-- !sc_rename_before_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 +7 7000 700 grace 70 after-drop 7.70 7 700 2026-07-06 + +-- !sc_rename_after_insert -- +1 \N 100 alice 10 \N 1.10 \N \N 2026-07-01 +100 \N 10000 default-value 100 unknown 100.00 \N \N 2026-07-10 +2 \N 200 bob 20 \N 2.20 \N \N 2026-07-02 +3 \N 300 carol 30 added-3 3.30 \N \N 2026-07-03 +4 \N 400 dave 40 unknown 4.40 \N \N 2026-07-01 +5 5000 500 erin 50 added-first 5.50 \N \N 2026-07-04 +6 6000 600 frank 60 added-columns 6.60 6 600 2026-07-05 +60 \N 6000 partial-columns 600 explicit-default-column 60.60 \N \N 2026-07-05 +7 7000 700 grace 70 after-drop 7.70 7 700 2026-07-06 +8 8000 800 heidi 80 after-rename 8.80 8 800 2026-07-02 + +-- !sc_modify_bigint_before_insert -- +1 alice 10 1.10 100 2026-07-01 +100 default-value 100 100.00 10000 2026-07-10 +2 bob 20 2.20 200 2026-07-02 +3 carol 30 3.30 300 2026-07-03 +4 dave 40 4.40 400 2026-07-01 +5 erin 50 5.50 500 2026-07-04 +6 frank 60 6.60 600 2026-07-05 +60 partial-columns 600 60.60 6000 2026-07-05 +7 grace 70 7.70 700 2026-07-06 +8 heidi 80 8.80 800 2026-07-02 + +-- !sc_modify_bigint_after_insert -- +1 alice 10 1.10 100 2026-07-01 +100 default-value 100 100.00 10000 2026-07-10 +2 bob 20 2.20 200 2026-07-02 +3 carol 30 3.30 300 2026-07-03 +4 dave 40 4.40 400 2026-07-01 +5 erin 50 5.50 500 2026-07-04 +6 frank 60 6.60 600 2026-07-05 +60 partial-columns 600 60.60 6000 2026-07-05 +7 grace 70 7.70 700 2026-07-06 +8 heidi 80 8.80 800 2026-07-02 +9 ivan 3000000000 9.90 900 2026-07-07 + +-- !sc_modify_decimal_before_insert -- +1 alice 10 1.10 2026-07-01 +100 default-value 100 100.00 2026-07-10 +2 bob 20 2.20 2026-07-02 +3 carol 30 3.30 2026-07-03 +4 dave 40 4.40 2026-07-01 +5 erin 50 5.50 2026-07-04 +6 frank 60 6.60 2026-07-05 +60 partial-columns 600 60.60 2026-07-05 +7 grace 70 7.70 2026-07-06 +8 heidi 80 8.80 2026-07-02 +9 ivan 3000000000 9.90 2026-07-07 + +-- !sc_modify_decimal_after_insert -- +1 alice 10 1.10 2026-07-01 +10 judy 100 1234567890.12 2026-07-08 +100 default-value 100 100.00 2026-07-10 +2 bob 20 2.20 2026-07-02 +3 carol 30 3.30 2026-07-03 +4 dave 40 4.40 2026-07-01 +5 erin 50 5.50 2026-07-04 +6 frank 60 6.60 2026-07-05 +60 partial-columns 600 60.60 2026-07-05 +7 grace 70 7.70 2026-07-06 +8 heidi 80 8.80 2026-07-02 +9 ivan 3000000000 9.90 2026-07-07 + +-- !sc_modify_nullable_before_insert -- +1 alice 100 2026-07-01 +10 judy 1000 2026-07-08 +100 default-value 10000 2026-07-10 +2 bob 200 2026-07-02 +3 carol 300 2026-07-03 +4 dave 400 2026-07-01 +5 erin 500 2026-07-04 +6 frank 600 2026-07-05 +60 partial-columns 6000 2026-07-05 +7 grace 700 2026-07-06 +8 heidi 800 2026-07-02 +9 ivan 900 2026-07-07 + +-- !sc_modify_nullable_after_insert -- +1 alice 100 2026-07-01 +10 judy 1000 2026-07-08 +100 default-value 10000 2026-07-10 +11 kate \N 2026-07-09 +2 bob 200 2026-07-02 +3 carol 300 2026-07-03 +4 dave 400 2026-07-01 +5 erin 500 2026-07-04 +6 frank 600 2026-07-05 +60 partial-columns 6000 2026-07-05 +7 grace 700 2026-07-06 +8 heidi 800 2026-07-02 +9 ivan 900 2026-07-07 + +-- !sc_modify_metadata_desc -- +added_after text Yes true \N changed comment +first_col bigint Yes true \N +id int Yes true \N +required_value bigint Yes true \N +full_name text Yes true \N +score bigint Yes true \N +amount decimal(12,2) Yes true \N +dt text Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small integer + +-- !sc_modify_metadata_before_insert -- +1 alice \N 10 2026-07-01 +10 judy after-decimal 100 2026-07-08 +100 default-value unknown 100 2026-07-10 +11 kate after-nullable 110 2026-07-09 +2 bob \N 20 2026-07-02 +3 carol added-3 30 2026-07-03 +4 dave unknown 40 2026-07-01 +5 erin added-first 50 2026-07-04 +6 frank added-columns 60 2026-07-05 +60 partial-columns explicit-default-column 600 2026-07-05 +7 grace after-drop 70 2026-07-06 +8 heidi after-rename 80 2026-07-02 +9 ivan after-bigint 3000000000 2026-07-07 + +-- !sc_modify_default_omitted -- +120 changed-default + +-- !sc_modify_metadata_after_insert -- +1 alice \N 10 2026-07-01 +10 judy after-decimal 100 2026-07-08 +100 default-value unknown 100 2026-07-10 +11 kate after-nullable 110 2026-07-09 +12 leo after-metadata 120 2026-07-10 +120 modified-default changed-default 1200 2026-07-10 +2 bob \N 20 2026-07-02 +3 carol added-3 30 2026-07-03 +4 dave unknown 40 2026-07-01 +5 erin added-first 50 2026-07-04 +6 frank added-columns 60 2026-07-05 +60 partial-columns explicit-default-column 600 2026-07-05 +7 grace after-drop 70 2026-07-06 +8 heidi after-rename 80 2026-07-02 +9 ivan after-bigint 3000000000 2026-07-07 + +-- !sc_modify_remove_metadata_desc -- +first_col bigint Yes true \N +id int Yes true \N +required_value bigint Yes true \N +full_name text Yes true \N +score bigint Yes true \N +added_after text Yes true \N +amount decimal(12,2) Yes true \N +dt text Yes true \N +tiny_col tinyint Yes true \N +small_col smallint Yes true \N small integer + +-- !sc_modify_remove_metadata_before_insert -- +1 alice 10 \N 2026-07-01 +10 judy 100 after-decimal 2026-07-08 +100 default-value 100 unknown 2026-07-10 +11 kate 110 after-nullable 2026-07-09 +12 leo 120 after-metadata 2026-07-10 +120 modified-default 1200 changed-default 2026-07-10 +2 bob 20 \N 2026-07-02 +3 carol 30 added-3 2026-07-03 +4 dave 40 unknown 2026-07-01 +5 erin 50 added-first 2026-07-04 +6 frank 60 added-columns 2026-07-05 +60 partial-columns 600 explicit-default-column 2026-07-05 +7 grace 70 after-drop 2026-07-06 +8 heidi 80 after-rename 2026-07-02 +9 ivan 3000000000 after-bigint 2026-07-07 + +-- !sc_remove_default_omitted -- +130 \N + +-- !sc_modify_remove_metadata_after_insert -- +1 alice 10 \N 2026-07-01 +10 judy 100 after-decimal 2026-07-08 +100 default-value 100 unknown 2026-07-10 +11 kate 110 after-nullable 2026-07-09 +12 leo 120 after-metadata 2026-07-10 +120 modified-default 1200 changed-default 2026-07-10 +13 mallory 130 after-remove-metadata 2026-07-11 +130 removed-default 1300 \N 2026-07-11 +2 bob 20 \N 2026-07-02 +3 carol 30 added-3 2026-07-03 +4 dave 40 unknown 2026-07-01 +5 erin 50 added-first 2026-07-04 +6 frank 60 added-columns 2026-07-05 +60 partial-columns 600 explicit-default-column 2026-07-05 +7 grace 70 after-drop 2026-07-06 +8 heidi 80 after-rename 2026-07-02 +9 ivan 3000000000 after-bigint 2026-07-07 + +-- !sc_reorder_before_insert -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_reorder_after_insert -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +14 nick 140 14.14 1400 after-reorder 14000 14 1400 2026-07-12 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_after_failed_alters -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +14 nick 140 14.14 1400 after-reorder 14000 14 1400 2026-07-12 +15 olivia 150 15.15 \N after-failed-alters 15000 15 1500 2026-07-13 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_after_partition_evolution_failures -- +1 alice 10 1.10 100 \N \N \N \N 2026-07-01 +10 judy 100 1234567890.12 1000 after-decimal 10000 10 1000 2026-07-08 +100 default-value 100 100.00 10000 unknown \N \N \N 2026-07-10 +11 kate 110 11.11 \N after-nullable 11000 11 1100 2026-07-09 +12 leo 120 12.12 1200 after-metadata 12000 12 1200 2026-07-10 +120 modified-default 1200 120.00 12000 changed-default 120000 12 1200 2026-07-10 +13 mallory 130 13.13 1300 after-remove-metadata 13000 13 1300 2026-07-11 +130 removed-default 1300 130.00 13000 \N 130000 13 1300 2026-07-11 +14 nick 140 14.14 1400 after-reorder 14000 14 1400 2026-07-12 +15 olivia 150 15.15 \N after-failed-alters 15000 15 1500 2026-07-13 +16 peggy 160 16.16 1600 after-partition-evolution-failures 16000 16 1600 2026-07-14 +2 bob 20 2.20 200 \N \N \N \N 2026-07-02 +3 carol 30 3.30 300 added-3 \N \N \N 2026-07-03 +4 dave 40 4.40 400 unknown \N \N \N 2026-07-01 +5 erin 50 5.50 500 added-first 5000 \N \N 2026-07-04 +6 frank 60 6.60 600 added-columns 6000 6 600 2026-07-05 +60 partial-columns 600 60.60 6000 explicit-default-column \N \N \N 2026-07-05 +7 grace 70 7.70 700 after-drop 7000 7 700 2026-07-06 +8 heidi 80 8.80 800 after-rename 8000 8 800 2026-07-02 +9 ivan 3000000000 9.90 900 after-bigint 9000 9 900 2026-07-07 + +-- !sc_append_partitions -- +{2026-07-01} 2 +{2026-07-02} 2 +{2026-07-03} 1 +{2026-07-04} 1 +{2026-07-05} 2 +{2026-07-06} 1 +{2026-07-07} 1 +{2026-07-08} 1 +{2026-07-09} 1 +{2026-07-10} 3 +{2026-07-11} 2 +{2026-07-12} 1 +{2026-07-13} 1 +{2026-07-14} 1 + +-- !sc_types_initial -- +1 100 30000 2000000000 1.5 123456.78 + +-- !sc_types_tiny_to_small_before_insert -- +1 100 30000 2000000000 1.5 123456.78 + +-- !sc_types_tiny_to_small_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 + +-- !sc_types_small_to_int_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 + +-- !sc_types_small_to_int_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 + +-- !sc_types_int_to_bigint_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 + +-- !sc_types_int_to_bigint_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 + +-- !sc_types_float_to_double_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 + +-- !sc_types_float_to_double_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 + +-- !sc_types_decimal_widen_before_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 + +-- !sc_types_decimal_widen_after_insert -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 +6 204 40003 3000000002 2e+40 1234567890.12 + +-- !sc_types_after_failed_narrow -- +1 100 30000 2000000000 1.5 123456.78 +2 200 30001 2000000001 2.5 123456.79 +3 201 40000 2000000002 3.5 123456.80 +4 202 40001 3000000000 4.5 123456.81 +5 203 40002 3000000001 1e+40 123456.82 +6 204 40003 3000000002 2e+40 1234567890.12 +7 205 40004 3000000003 3e+40 1234567890.13 + +-- !sc_explicit_types_initial -- +1 100 +2 200 + +-- !sc_explicit_bigint_to_int_before_insert -- +1 100 +2 200 + +-- !sc_explicit_bigint_to_int_after_insert -- +1 100 +2 200 +3 300 + +-- !sc_explicit_int_to_string_before_insert -- +1 100 +2 200 +3 300 + +-- !sc_explicit_int_to_string_after_insert -- +1 100 +2 200 +3 300 +4 after-explicit-cast + +-- !sc_pk_initial -- +1 2026-08-01 10 pk-a +2 2026-08-01 20 pk-b + +-- !sc_pk_add_before_insert -- +1 2026-08-01 10 \N pk-a +2 2026-08-01 20 \N pk-b + +-- !sc_pk_add_after_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 20 \N pk-b +3 2026-08-02 30 new-after-add pk-c + +-- !sc_pk_rename_before_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 20 \N pk-b +3 2026-08-02 30 new-after-add pk-c + +-- !sc_pk_rename_after_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 22 updated-after-rename pk-b2 +3 2026-08-02 30 new-after-add pk-c +4 2026-08-02 40 new-after-rename pk-d + +-- !sc_pk_type_before_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 22 updated-after-rename pk-b2 +3 2026-08-02 30 new-after-add pk-c +4 2026-08-02 40 new-after-rename pk-d + +-- !sc_pk_type_after_insert -- +1 2026-08-01 11 updated-after-add pk-a2 +2 2026-08-01 22 updated-after-rename pk-b2 +3 2026-08-02 30 new-after-add pk-c +4 2026-08-02 40 new-after-rename pk-d +5 2026-08-03 3000000000 new-after-type pk-e + +-- !sc_pk_drop_before_insert -- +1 2026-08-01 11 updated-after-add +2 2026-08-01 22 updated-after-rename +3 2026-08-02 30 new-after-add +4 2026-08-02 40 new-after-rename +5 2026-08-03 3000000000 new-after-type + +-- !sc_pk_partial_default -- +8 2026-08-05 81 default-note +9 2026-08-05 \N default-note + +-- !sc_pk_drop_after_insert -- +1 2026-08-01 11 updated-after-add +2 2026-08-01 22 updated-after-rename +3 2026-08-02 30 new-after-add +4 2026-08-02 40 new-after-rename +5 2026-08-03 3000000000 new-after-type +6 2026-08-03 60 new-after-drop +8 2026-08-05 81 default-note +9 2026-08-05 \N default-note + +-- !sc_pk_after_key_failures -- +1 2026-08-01 11 updated-after-add +2 2026-08-01 22 updated-after-rename +3 2026-08-02 30 new-after-add +4 2026-08-02 40 new-after-rename +5 2026-08-03 3000000000 new-after-type +6 2026-08-03 60 new-after-drop +7 2026-08-04 70 after-key-failures +8 2026-08-05 81 default-note +9 2026-08-05 \N default-note + diff --git a/regression-test/data/paimon_write/test_paimon_write_transaction.out b/regression-test/data/paimon_write/test_paimon_write_transaction.out new file mode 100644 index 00000000000000..37eaca4cb5bcc9 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_transaction.out @@ -0,0 +1,150 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !txn_commit -- +1 alice +2 bob +3 charlie +4 diana + +-- !txn_batch -- +1 1 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +2 2 +20 20 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 + +-- !txn_overwrite -- +10 new1 +20 new2 + +-- !txn_empty_overwrite -- +0 + +-- !txn_empty_overwrite_limit_zero -- +0 + +-- !txn_static_partition -- +10 east_new east +2 west_old west + +-- !txn_static_partition_case -- +10 east_new east +2 west_old west + +-- !txn_static_partial -- +10 new_a 1 A +3 keep 2 C + +-- !txn_static_partial_empty -- +3 keep 2 C + +-- !txn_static_null -- +10 null_new \N +2 literal_null null +3 blank_old +4 east_old east + +-- !txn_static_empty -- +10 null_new \N +2 literal_null null +3 blank_old + +-- !txn_static_blank -- +10 null_new \N +2 literal_null null +30 blank_new + +-- !txn_static_typed_boundaries -- +10 null_new \N 2026-07-03 +3 blank 2026-07-01 +4 literal_null null 2026-07-01 +50 special_new a/b=c%20 2026-07-04 +7 keep keep 2026-07-01 + +-- !txn_dynamic_multi -- +10 p1_new p1 +20 p2_new_a p2 +21 p2_new_b p2 +4 p3_keep p3 +5 p4_keep p4 + +-- !txn_dynamic_partition -- +10 east_new east +2 west_old west +30 south_new south + +-- !txn_unsupported_partition_syntax -- +10 east_new east +2 west_old west +30 south_new south + +-- !txn_parallel_writers -- +256 0 255 32640 + +-- !txn_multi_block -- +20480 0 20479 209704960 18618 8 + +-- !txn_multi_block_samples -- +0 0 payload_0 \N p0 +16383 87 payload_16383 49149 p7 +16384 88 payload_16384 49152 p0 +20479 12 payload_20479 61437 p7 +4095 21 payload_4095 12285 p7 +4096 22 payload_4096 12288 p0 +8191 43 payload_8191 24573 p7 +8192 44 payload_8192 24576 p0 + +-- !txn_multi_block_snapshots -- +2 + +-- !txn_spill -- +2048 0 2047 2096128 + +-- !txn_failed_write_before -- +1 committed_before_failure + +-- !txn_failed_snapshot_before -- +1 + +-- !txn_failed_write_after -- +1 committed_before_failure + +-- !txn_failed_snapshot_after -- +1 + +-- !txn_multi -- +1 a 10 +10 j 100 +11 k 110 +12 l 120 +13 m 130 +14 n 140 +15 o 150 +16 p 160 +17 q 170 +18 r 180 +19 s 190 +2 b 20 +20 t 200 +3 c 30 +4 d 40 +5 e 50 +6 f 60 +7 g 70 +8 h 80 +9 i 90 + diff --git a/regression-test/data/paimon_write/test_paimon_write_types.out b/regression-test/data/paimon_write/test_paimon_write_types.out new file mode 100644 index 00000000000000..a12d29c2c5ce01 --- /dev/null +++ b/regression-test/data/paimon_write/test_paimon_write_types.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !types_basic -- +false -2147483648 -9223372036854775808 -3.4E38 -1.7e+308 -1.50 long_string_1234567890 max_varchar 2099-12-31 2099-12-31T23:59:59 +false 2147483647 9223372036854775807 3.4E38 1.7e+308 0.00 1970-01-01 1970-01-01T00:00 +true 0 0 0.0 0 12345678.90 hello fixed_len 2024-06-15 2024-06-15T12:00:00.123456 +true 1 100 1.5 2.71828 99.99 hello short 2024-01-15 2024-01-15T10:30 + +-- !types_null -- +1 100 data 1.5 true +2 \N \N \N \N +3 \N partial 2 false + +-- !types_decimal -- +1 1.5 12345678.90 123456789012.123456 1234567890123456789012345678.1234567890 +2 -1.5 -0.01 -1.000001 1E-10 +3 0.0 0.00 0.000000 0E-10 + +-- !types_dt -- +1970-01-01 1970-01-01T00:00 +2024-06-15 2024-06-15T12:00 +2099-12-31 2099-12-31T23:59:59.999999 + +-- !desc_types_timezone -- +event_time datetime(6) Yes true \N WITH_TIMEZONE + +-- !types_timezone_utc -- +1 2024-01-15T02:30:00.123456 +2 2024-01-15T02:30:00.654321 + +-- !types_timezone_shanghai -- +1 2024-01-15T10:30:00.123456 +2 2024-01-15T10:30:00.654321 + +-- !types_ntz -- +1 2024-03-10T02:30:00.123456 +2 2024-01-15T10:30:00.654321 + diff --git a/regression-test/suites/external_table_p0/paimon/paimon_schema_change_ddl.groovy b/regression-test/suites/external_table_p0/paimon/paimon_schema_change_ddl.groovy new file mode 100644 index 00000000000000..bccafaad4ae7c7 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/paimon_schema_change_ddl.groovy @@ -0,0 +1,398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("paimon_schema_change_ddl", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "paimon_schema_change_ddl" + String dbName = "paimon_schema_change_ddl_db" + String tableName = "paimon_alter_table" + String partitionTableName = "paimon_alter_partition_table" + + def schemaId = { String table -> + def rows = sql """ + SELECT MAX(schema_id) + FROM `${table}\$schemas` + """ + assertEquals(1, rows.size()) + return (rows[0][0] as Number).longValue() + } + + def columnNames = { String table -> + return sql("DESC `${table}`").collect { row -> row[0].toString() } + } + + def assertColumnOrder = { String table, List expected -> + assertEquals(expected, columnNames(table)) + } + + def assertColumnAbsent = { String table, String column -> + assertFalse(columnNames(table).any { name -> name.equalsIgnoreCase(column) }) + } + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + sql """SET show_column_comment_in_describe = true""" + + try { + // This suite intentionally contains no INSERT. ALTER correctness is + // verified from Doris metadata and Paimon's schema history table. + // Use strict type evolution so narrowing conversions are covered as + // deterministic failures; Paimon permits explicit casts by default. + sql """ + CREATE TABLE `${tableName}` ( + id INT NOT NULL COMMENT 'identifier', + required_value BIGINT NOT NULL, + score INT NULL DEFAULT '1' COMMENT 'initial score', + `MixedCase` STRING NULL, + obsolete STRING NULL, + amount DECIMAL(8, 2) NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'disable-explicit-type-casting' = 'true' + ) + """ + + assertColumnOrder( + tableName, + ["id", "required_value", "score", "MixedCase", "obsolete", "amount"]) + qt_paimon_alter_initial_desc """DESC `${tableName}`""" + qt_paimon_alter_initial_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // ADD COLUMN: default, comment and AFTER position are committed as one + // Paimon schema version. + long beforeSchemaId = schemaId(tableName) + sql """ + ALTER TABLE `${tableName}` + ADD COLUMN added_after STRING NULL DEFAULT 'unknown' + COMMENT 'added column' AFTER score + """ + assertEquals(beforeSchemaId + 1, schemaId(tableName)) + assertColumnOrder( + tableName, + [ + "id", "required_value", "score", "added_after", + "MixedCase", "obsolete", "amount" + ]) + qt_paimon_alter_add_column_desc """DESC `${tableName}`""" + qt_paimon_alter_add_column_schema """ + SELECT schema_id, fields + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // ADD COLUMNS is one Doris clause and one atomic Paimon schema commit. + // TINYINT and SMALLINT also cover the narrow integer type mapping. + beforeSchemaId = schemaId(tableName) + sql """ + ALTER TABLE `${tableName}` ADD COLUMN ( + tiny_col TINYINT NULL, + small_col SMALLINT NULL COMMENT 'small column', + profile STRUCT NULL + ) + """ + assertEquals(beforeSchemaId + 1, schemaId(tableName)) + qt_paimon_alter_add_columns_desc """DESC `${tableName}`""" + + // FIRST position. + sql """ALTER TABLE `${tableName}` ADD COLUMN first_col BIGINT NULL FIRST""" + assertColumnOrder( + tableName, + [ + "first_col", "id", "required_value", "score", "added_after", + "MixedCase", "obsolete", "amount", "tiny_col", "small_col", "profile" + ]) + qt_paimon_alter_add_first_desc """DESC `${tableName}`""" + + // Doris resolves column names case-insensitively but sends the canonical + // remote field name to Paimon. + sql """ALTER TABLE `${tableName}` RENAME COLUMN mixedcase display_name""" + assertColumnOrder( + tableName, + [ + "first_col", "id", "required_value", "score", "added_after", + "display_name", "obsolete", "amount", "tiny_col", "small_col", "profile" + ]) + qt_paimon_alter_rename_column_desc """DESC `${tableName}`""" + + // MODIFY COLUMN: widening type, nullability, default, comment and + // position changes are committed together. + beforeSchemaId = schemaId(tableName) + sql """ + ALTER TABLE `${tableName}` + MODIFY COLUMN score BIGINT NULL DEFAULT '10' + COMMENT 'updated score' FIRST + """ + assertEquals(beforeSchemaId + 1, schemaId(tableName)) + assertColumnOrder( + tableName, + [ + "score", "first_col", "id", "required_value", "added_after", + "display_name", "obsolete", "amount", "tiny_col", "small_col", "profile" + ]) + + // Additional supported widening conversions and NOT NULL -> NULL. + sql """ALTER TABLE `${tableName}` MODIFY COLUMN small_col INT NULL""" + sql """ALTER TABLE `${tableName}` MODIFY COLUMN amount DECIMAL(12, 2) NULL""" + sql """ALTER TABLE `${tableName}` MODIFY COLUMN required_value BIGINT NULL""" + + // Omitting DEFAULT and COMMENT in a full MODIFY definition removes + // their existing values. + sql """ALTER TABLE `${tableName}` MODIFY COLUMN added_after STRING NULL""" + qt_paimon_alter_modify_column_desc """DESC `${tableName}`""" + qt_paimon_alter_modify_column_schema """ + SELECT schema_id, fields + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + sql """ALTER TABLE `${tableName}` DROP COLUMN obsolete""" + assertColumnAbsent(tableName, "obsolete") + qt_paimon_alter_drop_column_desc """DESC `${tableName}`""" + + sql """ + ALTER TABLE `${tableName}` ORDER BY ( + id, display_name, score, required_value, small_col, + tiny_col, added_after, amount, profile, first_col + ) + """ + assertColumnOrder( + tableName, + [ + "id", "display_name", "score", "required_value", "small_col", + "tiny_col", "added_after", "amount", "profile", "first_col" + ]) + qt_paimon_alter_reorder_columns_desc """DESC `${tableName}`""" + qt_paimon_alter_final_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // A failed ADD COLUMNS must not publish the valid prefix of the batch. + beforeSchemaId = schemaId(tableName) + test { + sql """ + ALTER TABLE `${tableName}` ADD COLUMN ( + batch_ok INT NULL, + batch_bad INT NOT NULL DEFAULT '1' + ) + """ + exception "cannot specify NOT NULL" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + assertColumnAbsent(tableName, "batch_ok") + assertColumnAbsent(tableName, "batch_bad") + qt_paimon_alter_failed_batch_schema """ + SELECT schema_id, fields + FROM `${tableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + // Multiple Doris ALTER clauses cannot be committed atomically by an + // external catalog, so they are rejected before the first mutation. + beforeSchemaId = schemaId(tableName) + test { + sql """ + ALTER TABLE `${tableName}` + ADD COLUMN multi_a INT NULL, + ADD COLUMN multi_b INT NULL + """ + exception "External table does not support multiple ALTER clauses" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + assertColumnAbsent(tableName, "multi_a") + assertColumnAbsent(tableName, "multi_b") + + // Paimon SDK schema validation. + beforeSchemaId = schemaId(tableName) + test { + sql """ + ALTER TABLE `${tableName}` + ADD COLUMN required_col INT NOT NULL DEFAULT '1' + """ + exception "cannot specify NOT NULL" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` MODIFY COLUMN score INT NULL""" + exception "cannot be converted" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ + ALTER TABLE `${tableName}` + MODIFY COLUMN added_after STRING NOT NULL DEFAULT 'unknown' + """ + exception "Cannot update column type from nullable to non nullable" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` DROP COLUMN id""" + exception "Cannot drop partition key or primary key" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + // Doris/Paimon adapter validation which cannot be delegated to the SDK. + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN ID INT NULL""" + exception "conflicts with an existing Paimon column" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` RENAME COLUMN display_name ID""" + exception "conflicts with an existing Paimon column" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN agg_col INT SUM NULL""" + exception "does not support aggregation method" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN auto_col BIGINT AUTO_INCREMENT""" + exception "does not support AUTO_INCREMENT" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN generated_col INT AS (score + 1)""" + exception "cannot be a generated column in a Paimon table" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ALTER TABLE `${tableName}` ADD COLUMN bad_position INT NULL AFTER missing_col""" + exception "does not exist in Paimon table" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ + ALTER TABLE `${tableName}` ORDER BY ( + id, display_name, score + ) + """ + exception "must contain every Paimon column exactly once" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + test { + sql """ + ALTER TABLE `${tableName}` ORDER BY ( + id, display_name, score, required_value, small_col, + tiny_col, added_after, amount, profile, id + ) + """ + exception "Duplicate column in reorder columns" + } + assertEquals(beforeSchemaId, schemaId(tableName)) + + // Partition and primary-key constraints are delegated to Paimon. + sql """ + CREATE TABLE `${partitionTableName}` ( + id INT NOT NULL, + pt STRING NOT NULL, + payload INT NULL + ) ENGINE=paimon + PARTITION BY (pt) () + PROPERTIES ( + 'primary-key' = 'id,pt' + ) + """ + qt_paimon_alter_partition_initial_desc """DESC `${partitionTableName}`""" + qt_paimon_alter_partition_initial_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${partitionTableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + + long partitionSchemaId = schemaId(partitionTableName) + test { + sql """ALTER TABLE `${partitionTableName}` DROP COLUMN pt""" + exception "Cannot drop partition key or primary key" + } + assertEquals(partitionSchemaId, schemaId(partitionTableName)) + + test { + sql """ALTER TABLE `${partitionTableName}` RENAME COLUMN pt partition_col""" + exception "Cannot rename partition column" + } + assertEquals(partitionSchemaId, schemaId(partitionTableName)) + + test { + sql """ALTER TABLE `${partitionTableName}` MODIFY COLUMN pt INT NOT NULL""" + exception "Cannot update partition column" + } + assertEquals(partitionSchemaId, schemaId(partitionTableName)) + + // Non-key columns of a partitioned table can still evolve. + sql """ALTER TABLE `${partitionTableName}` MODIFY COLUMN payload BIGINT NULL""" + sql """ALTER TABLE `${partitionTableName}` ADD COLUMN extra STRING NULL""" + assertColumnOrder(partitionTableName, ["id", "pt", "payload", "extra"]) + qt_paimon_alter_partition_final_desc """DESC `${partitionTableName}`""" + qt_paimon_alter_partition_final_schema """ + SELECT schema_id, fields, partition_keys, primary_keys + FROM `${partitionTableName}\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + } finally { + sql """DROP TABLE IF EXISTS `${partitionTableName}`""" + sql """DROP TABLE IF EXISTS `${tableName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """SWITCH internal""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index 0783d60392813d..1a41eee698ff1f 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -615,11 +615,11 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { // Scenario TC08/S20: illegal PK/partition changes fail atomically. test { sql """alter table ${pkTable} drop column id""" - exception "Drop column operation is not supported" + exception "Cannot drop partition key or primary key" } test { sql """alter table ${partitionTable} drop column old_partition""" - exception "Drop column operation is not supported" + exception "Cannot drop partition key or primary key" } assertEquals([[1, "alpha-updated"], [3, "gamma"]], sql("""select id, full_name from ${pkTable} order by id""")) diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy index 2f41229debcc17..6ce490e239026d 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy @@ -64,20 +64,14 @@ suite("test_paimon_write_boundary", qt_before_rows """select id, score, note from write_boundary order by id""" qt_before_snapshots """select count(*) from write_boundary\$snapshots""" - // WB01-WB06 preserve the documented data-write boundary at analysis time. The source table - // and its snapshot list must stay unchanged after every rejected write shape. - test { - sql """insert into write_boundary values (3, 30, 'insert-values')""" - exception "PaimonExternalCatalog" - } - test { - sql """insert into write_boundary select 3, 30, 'insert-select'""" - exception "PaimonExternalCatalog" - } - test { - sql """insert overwrite table write_boundary values (3, 30, 'overwrite')""" - exception "PaimonExternalCatalog" - } + // Doris supports INSERT VALUES, INSERT SELECT and INSERT OVERWRITE for Paimon. + // Row-level UPDATE, DELETE and MERGE remain outside this write path. + sql """insert into write_boundary values (3, 30, 'insert-values')""" + sql """insert into write_boundary select 4, 40, 'insert-select'""" + sql """refresh table write_boundary""" + qt_after_append_rows """select id, score, note from write_boundary order by id""" + + sql """insert overwrite table write_boundary values (5, 50, 'overwrite')""" test { sql """update write_boundary set score = score + 1 where id = 1""" exception "target table in update command should be an olapTable" diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog_write.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog_write.groovy new file mode 100644 index 00000000000000..4830f0e7a985d0 --- /dev/null +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog_write.groovy @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_hms_catalog_write", "p2,external,paimon,new_catalog_property") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive3HmsPort") + String hdfsPort = context.config.otherConfigs.get("hive3HdfsPort") + String catalogName = "test_paimon_hms_catalog_write" + String dbName = "hdfs_db" + String appendTable = "test_paimon_hms_write_append" + String primaryKeyTable = "test_paimon_hms_write_pk" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'warehouse' = 'hdfs://${externalEnvIp}:${hdfsPort}/user/hive/warehouse', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """USE `${dbName}`""" + + try { + sql """DROP TABLE IF EXISTS `${appendTable}`""" + sql """DROP TABLE IF EXISTS `${primaryKeyTable}`""" + + sql """ + CREATE TABLE `${appendTable}` ( + id INT NULL, + name STRING NULL, + score DOUBLE NULL, + region STRING NULL + ) ENGINE=paimon + PARTITION BY (region) () + """ + sql """ + INSERT INTO `${appendTable}` VALUES + (1, 'alice', 95.5, 'east'), + (2, 'bob', 87.0, 'west') + """ + sql """ + INSERT INTO `${appendTable}` VALUES + (3, 'charlie', 92.3, 'east'), + (4, 'diana', 88.0, 'north') + """ + sql """ + INSERT INTO `${appendTable}` (region, score, name, id) + VALUES ('south', 86.5, 'erin', 5) + """ + sql """ + INSERT INTO `${appendTable}` (region, id) + VALUES ('east', 6) + """ + order_qt_hms_append """ + SELECT id, name, score, region + FROM `${appendTable}` + ORDER BY id + """ + + sql """ + CREATE TABLE `${primaryKeyTable}` ( + user_id INT NOT NULL, + event_time BIGINT NOT NULL, + event_type STRING NULL, + value DOUBLE NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'user_id,event_time', + 'bucket' = '2', + 'bucket-key' = 'user_id' + ) + """ + sql """ + INSERT INTO `${primaryKeyTable}` VALUES + (1, 100, 'click', 1.0), + (1, 200, 'view', 2.0), + (2, 100, 'click', 3.0), + (1, 100, 'click_updated', 99.0) + """ + order_qt_hms_pk """ + SELECT user_id, event_time, event_type, value + FROM `${primaryKeyTable}` + ORDER BY user_id, event_time + """ + } finally { + sql """DROP TABLE IF EXISTS `${appendTable}`""" + sql """DROP TABLE IF EXISTS `${primaryKeyTable}`""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_rest_catalog_write.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_rest_catalog_write.groovy new file mode 100644 index 00000000000000..cfb47d48b04f87 --- /dev/null +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_rest_catalog_write.groovy @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_rest_catalog_write", + "p2,external,paimon,external_remote,external_remote_paimon,new_catalog_property") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String catalogProperties = context.config.otherConfigs.get("paimonDlfRestCatalog") + String catalogName = "test_paimon_rest_catalog_write" + String dbName = "new_dlf_paimon_db" + String appendTable = "test_paimon_rest_write_append" + String primaryKeyTable = "test_paimon_rest_write_pk" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + ${catalogProperties} + ) + """ + sql """SWITCH `${catalogName}`""" + sql """USE `${dbName}`""" + + try { + sql """DROP TABLE IF EXISTS `${appendTable}`""" + sql """DROP TABLE IF EXISTS `${primaryKeyTable}`""" + + sql """ + CREATE TABLE `${appendTable}` ( + id INT NULL, + name STRING NULL, + score DOUBLE NULL, + region STRING NULL + ) ENGINE=paimon + PARTITION BY (region) () + """ + sql """ + INSERT INTO `${appendTable}` VALUES + (1, 'alice', 95.5, 'east'), + (2, 'bob', 87.0, 'west') + """ + sql """ + INSERT INTO `${appendTable}` VALUES + (3, 'charlie', 92.3, 'east'), + (4, 'diana', 88.0, 'north') + """ + sql """ + INSERT INTO `${appendTable}` (region, score, name, id) + VALUES ('south', 86.5, 'erin', 5) + """ + sql """ + INSERT INTO `${appendTable}` (region, id) + VALUES ('east', 6) + """ + order_qt_rest_append """ + SELECT id, name, score, region + FROM `${appendTable}` + ORDER BY id + """ + + sql """ + CREATE TABLE `${primaryKeyTable}` ( + user_id INT NOT NULL, + event_time BIGINT NOT NULL, + event_type STRING NULL, + value DOUBLE NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'user_id,event_time', + 'bucket' = '2', + 'bucket-key' = 'user_id' + ) + """ + sql """ + INSERT INTO `${primaryKeyTable}` VALUES + (1, 100, 'click', 1.0), + (1, 200, 'view', 2.0), + (2, 100, 'click', 3.0), + (1, 100, 'click_updated', 99.0) + """ + order_qt_rest_pk """ + SELECT user_id, event_time, event_type, value + FROM `${primaryKeyTable}` + ORDER BY user_id, event_time + """ + } finally { + sql """DROP TABLE IF EXISTS `${appendTable}`""" + sql """DROP TABLE IF EXISTS `${primaryKeyTable}`""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy index c4ea889f92307a..26241877c5e6fe 100644 --- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy @@ -258,12 +258,12 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori assertTrue(showNullPartitionsResult.toString().contains("p_null")) assertTrue(showNullPartitionsResult.toString().contains("p_NULL")) assertTrue(showNullPartitionsResult.toString().contains("p_bj")) + assertEquals(4, showNullPartitionsResult.size()) sql """ REFRESH MATERIALIZED VIEW ${mvName} auto; - """ + """ waitingMTMVTaskFinishedByMvName(mvName) - // Will lose null data - order_qt_null_partition "SELECT * FROM ${mvName} " + order_qt_null_partition "SELECT * FROM ${mvName} ORDER BY id" sql """drop materialized view if exists ${mvName};""" // date type will has problem @@ -305,4 +305,3 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori sql """drop catalog if exists ${catalogName}""" } - diff --git a/regression-test/suites/paimon_write/test_paimon_create_ddl_write_properties.groovy b/regression-test/suites/paimon_write/test_paimon_create_ddl_write_properties.groovy new file mode 100644 index 00000000000000..dcfbec3891c98d --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_create_ddl_write_properties.groovy @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_create_ddl_write_properties", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_create_props_catalog" + String dbName = "test_pw_create_props_db" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + spark_paimon """ + REFRESH TABLE paimon.${dbName}.${tableName} + """ + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """ + SELECT * FROM `${tableName}` ${orderBy} + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT MAX(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + assertEquals(1, rows.size()) + assertTrue(rows[0][0] != null) + return rows[0][0].toString() + } + + // Doris maps location to Paimon's path option. The filesystem catalog + // deliberately rejects custom table paths, and that SDK validation + // must be preserved instead of silently ignoring the property. + test { + sql """ + CREATE TABLE `t_create_custom_location` ( + id INT NULL + ) ENGINE=paimon + PROPERTIES ( + 'location' = + 's3://warehouse/wh/${dbName}.db/t_create_custom_location_data' + ) + """ + exception "does not support specifying the table path" + } + qt_create_custom_location_absent """ + SHOW TABLES LIKE 't_create_custom_location' + """ + + // Doris CREATE must preserve the primary/partition keys, table comment + // and storage/write options. Sequence ordering is verified with a + // lower-sequence update followed by a higher one. + sql """ + CREATE TABLE `t_create_sequence` ( + id INT NOT NULL, + seq BIGINT NOT NULL, + payload STRING NULL, + dt STRING NOT NULL + ) ENGINE=paimon + PARTITION BY (dt) () + PROPERTIES ( + 'primary-key' = 'id,dt', + 'bucket' = '2', + 'bucket-key' = 'id', + 'sequence.field' = 'seq', + 'file.format' = 'orc', + 'snapshot.num-retained.min' = '2', + 'snapshot.num-retained.max' = '5', + 'comment' = 'created by Doris with write properties' + ) + """ + sql """ + INSERT INTO `t_create_sequence` VALUES + (1, 100, 'newer', 'p1'), + (2, 10, 'initial-2', 'p1'), + (3, 5, 'initial-3', 'p2') + """ + sql """ + INSERT INTO `t_create_sequence` VALUES + (1, 50, 'older-must-not-win', 'p1'), + (2, 20, 'updated-2', 'p1') + """ + order_qt_create_sequence_result """ + SELECT id, seq, payload, dt + FROM `t_create_sequence` + ORDER BY dt, id + """ + qt_create_sequence_schema """ + SELECT partition_keys, primary_keys, comment + FROM `t_create_sequence\$schemas` + ORDER BY schema_id DESC + LIMIT 1 + """ + order_qt_create_sequence_file_format """ + SELECT DISTINCT file_format + FROM `t_create_sequence\$files` + ORDER BY file_format + """ + assertTableEquals("t_create_sequence", "ORDER BY dt, id") + + // Fixed bucket properties must be consumed by the writer, while + // partial-update accepts arbitrary value-column subsets. + sql """ + CREATE TABLE `t_create_partial` ( + id INT NOT NULL, + name STRING NULL, + score INT NULL, + note STRING NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'partial-update' + ) + """ + sql """ + INSERT INTO `t_create_partial` VALUES + (1, 'alice', 10, 'initial'), + (2, 'bob', 20, 'initial') + """ + sql """INSERT INTO `t_create_partial` (id, score) VALUES (1, 15)""" + sql """INSERT INTO `t_create_partial` (note, id) VALUES ('updated', 1)""" + order_qt_create_partial_result """ + SELECT id, name, score, note + FROM `t_create_partial` + ORDER BY id + """ + assertTableEquals("t_create_partial", "ORDER BY id") + + // First-row and aggregation semantics prove that CREATE forwarded the + // merge-engine and per-field aggregation properties. + sql """ + CREATE TABLE `t_create_first_row` ( + id INT NOT NULL, + name STRING NULL, + score INT NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'first-row' + ) + """ + sql """ + INSERT INTO `t_create_first_row` VALUES + (1, 'first-1', 10), + (2, 'first-2', 20) + """ + sql """ + INSERT INTO `t_create_first_row` VALUES + (1, 'second-1', 11), + (3, 'first-3', 30) + """ + order_qt_create_first_row_result """ + SELECT id, name, score + FROM `t_create_first_row` + ORDER BY id + """ + assertTableEquals("t_create_first_row", "ORDER BY id") + + sql """ + CREATE TABLE `t_create_aggregation` ( + id INT NOT NULL, + total BIGINT NULL, + highest INT NULL, + label STRING NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum', + 'fields.highest.aggregate-function' = 'max' + ) + """ + sql """ + INSERT INTO `t_create_aggregation` VALUES + (1, 10, 80, 'first'), + (2, 5, 70, 'stable') + """ + sql """ + INSERT INTO `t_create_aggregation` VALUES + (1, 7, 90, 'latest'), + (2, 3, 60, NULL), + (3, 4, 50, 'new') + """ + order_qt_create_aggregation_result """ + SELECT id, total, highest, label + FROM `t_create_aggregation` + ORDER BY id + """ + assertTableEquals("t_create_aggregation", "ORDER BY id") + + // Lookup changelog generation is checked independently from the final + // table contents, so merely storing the option is not sufficient. + sql """ + CREATE TABLE `t_create_lookup` ( + id INT NOT NULL, + name STRING NULL, + score INT NULL + ) ENGINE=paimon + PROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'changelog-producer' = 'lookup' + ) + """ + sql """ + INSERT INTO `t_create_lookup` VALUES + (1, 'old', 10), + (2, 'stable', 20) + """ + String lookupBefore = latestSnapshotId("t_create_lookup") + sql """ + INSERT INTO `t_create_lookup` VALUES + (1, 'new', 11), + (3, 'added', 30) + """ + String lookupAfter = latestSnapshotId("t_create_lookup") + def lookupChanges = spark_paimon """ + SELECT rowkind, id, name, score + FROM paimon_incremental_query( + 'paimon.${dbName}.`t_create_lookup\$audit_log`', + '${lookupBefore}', + '${lookupAfter}' + ) + ORDER BY id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END + """ + assertEquals([ + ["-U", 1, "old", 10], + ["+U", 1, "new", 11], + ["+I", 3, "added", 30] + ], lookupChanges) + order_qt_create_lookup_changelog """ + SELECT rowkind, id, name, score + FROM `t_create_lookup\$audit_log` + ORDER BY id, + CASE rowkind + WHEN '+I' THEN 0 + WHEN '-U' THEN 1 + WHEN '+U' THEN 2 + ELSE 3 + END + """ + order_qt_create_lookup_result """ + SELECT id, name, score + FROM `t_create_lookup` + ORDER BY id + """ + assertTableEquals("t_create_lookup", "ORDER BY id") + + // Dynamic bucket options must affect physical routing after a Doris + // INSERT, not only appear in metadata. + sql """ + CREATE TABLE `t_create_dynamic_bucket` ( + pt STRING NOT NULL, + id INT NOT NULL, + value STRING NULL + ) ENGINE=paimon + PARTITION BY (pt) () + PROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.initial-buckets' = '1', + 'dynamic-bucket.max-buckets' = '4' + ) + """ + sql """ + INSERT INTO `t_create_dynamic_bucket` + SELECT 'p1', CAST(number AS INT), concat('v', CAST(number AS STRING)) + FROM numbers("number" = "12") + """ + qt_create_dynamic_bucket_result """ + SELECT COUNT(*), MIN(id), MAX(id) + FROM `t_create_dynamic_bucket` + """ + order_qt_create_dynamic_bucket_rows """ + SELECT pt, id, value + FROM `t_create_dynamic_bucket` + ORDER BY pt, id + """ + order_qt_create_dynamic_bucket_files """ + SELECT DISTINCT bucket + FROM `t_create_dynamic_bucket\$files` + ORDER BY bucket + """ + def dynamicBuckets = spark_paimon """ + SELECT DISTINCT bucket + FROM paimon.${dbName}.`t_create_dynamic_bucket\$files` + ORDER BY bucket + """ + assertFalse(dynamicBuckets.isEmpty()) + assertTrue(dynamicBuckets.every { row -> + int bucket = row[0].toString().toInteger() + return bucket >= 0 && bucket < 4 + }) + assertTrue(dynamicBuckets.size() > 1) + assertTableEquals("t_create_dynamic_bucket", "ORDER BY pt, id") + + // Keep one deterministic view of all important CREATE options. This + // catches property loss or accidental key rewriting in the Doris DDL. + order_qt_create_write_options """ + SELECT 'aggregation' AS table_name, `key`, value + FROM `t_create_aggregation\$options` + WHERE `key` IN ( + 'bucket', 'fields.highest.aggregate-function', + 'fields.total.aggregate-function', 'merge-engine' + ) + UNION ALL + SELECT 'dynamic_bucket', `key`, value + FROM `t_create_dynamic_bucket\$options` + WHERE `key` IN ( + 'bucket', 'dynamic-bucket.initial-buckets', + 'dynamic-bucket.max-buckets', 'dynamic-bucket.target-row-num' + ) + UNION ALL + SELECT 'first_row', `key`, value + FROM `t_create_first_row\$options` + WHERE `key` IN ('bucket', 'merge-engine') + UNION ALL + SELECT 'lookup', `key`, value + FROM `t_create_lookup\$options` + WHERE `key` IN ('bucket', 'changelog-producer') + UNION ALL + SELECT 'partial_update', `key`, value + FROM `t_create_partial\$options` + WHERE `key` IN ('bucket', 'bucket-key', 'merge-engine') + UNION ALL + SELECT 'sequence', `key`, value + FROM `t_create_sequence\$options` + WHERE `key` IN ( + 'bucket', 'bucket-key', 'file.format', + 'sequence.field', 'snapshot.num-retained.max', + 'snapshot.num-retained.min' + ) + ORDER BY table_name, `key` + """ + } finally { + [ + "t_create_dynamic_bucket", + "t_create_lookup", + "t_create_aggregation", + "t_create_first_row", + "t_create_partial", + "t_create_sequence" + ].each { tableName -> + try { + sql """DROP TABLE IF EXISTS `${tableName}`""" + } catch (Exception e) { + logger.info("Failed to drop ${tableName}: ${e.getMessage()}") + } + } + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """SWITCH internal""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_append_only.groovy b/regression-test/suites/paimon_write/test_paimon_write_append_only.groovy new file mode 100644 index 00000000000000..e7a8ccb18f8a47 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_append_only.groovy @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_append_only", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_ao_catalog" + String dbName = "test_pw_ao_db" + + // Tables are created via Spark because Doris does not yet support + // Paimon DDL (CREATE TABLE ... engine=paimon). + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.t_append; + CREATE TABLE paimon.${dbName}.t_append ( + id INT, name STRING, score DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_part; + CREATE TABLE paimon.${dbName}.t_append_part ( + id INT, name STRING, score DOUBLE, region STRING + ) USING paimon + PARTITIONED BY (region) + ; + + DROP TABLE IF EXISTS paimon.${dbName}.t_auto_partition; + CREATE TABLE paimon.${dbName}.t_auto_partition ( + id INT, name STRING, dt STRING + ) USING paimon + PARTITIONED BY (dt); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_empty; + CREATE TABLE paimon.${dbName}.t_append_empty ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_default; + CREATE TABLE paimon.${dbName}.t_append_default ( + id INT, name STRING NOT NULL DEFAULT 'unknown' + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_required; + CREATE TABLE paimon.${dbName}.t_append_required ( + id INT, name STRING NOT NULL + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_partition_default; + CREATE TABLE paimon.${dbName}.t_partition_default ( + id INT, name STRING, dt STRING NOT NULL DEFAULT '2026-07-01' + ) USING paimon + PARTITIONED BY (dt); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-001: Append-only table — basic INSERT + sql """INSERT INTO t_append VALUES (1, 'alice', 95.5)""" + sql """INSERT INTO t_append VALUES (2, 'bob', 87.0), (3, 'charlie', 92.3)""" + order_qt_ao_basic """SELECT * FROM t_append ORDER BY id""" + + sql """INSERT INTO t_append VALUES (4, 'diana', 88.0), (5, 'eve', 91.0)""" + // Full-column and partial-column writes with columns in non-schema order + sql """INSERT INTO t_append (score, name, id) VALUES (93.0, 'frank', 6)""" + sql """INSERT INTO t_append (name, id) VALUES ('grace', 7)""" + assertTableEquals("t_append", "ORDER BY id") + + // FT-002: Partitioned append-only + sql """INSERT INTO t_append_part VALUES (1, 'alice', 95.5, 'east'), (2, 'bob', 87.0, 'west')""" + sql """INSERT INTO t_append_part VALUES (3, 'charlie', 92.3, 'east'), (4, 'diana', 88.0, 'north')""" + // Keep the partition column away from its schema position in both full and partial writes + sql """INSERT INTO t_append_part (region, score, name, id) + VALUES ('south', 86.5, 'erin', 5)""" + sql """INSERT INTO t_append_part (region, id) VALUES ('east', 6)""" + order_qt_ao_part """SELECT * FROM t_append_part ORDER BY id""" + assertTableEquals("t_append_part", "ORDER BY id") + + // Paimon partitions are implicit: writing a previously unseen partition-key + // value creates the physical partition without an ADD PARTITION operation. + sql """INSERT INTO t_auto_partition VALUES + (1, 'alpha', '2026-07-01'), + (2, 'beta', '2026-07-02'), + (3, 'gamma', '2026-07-01') + """ + sql """INSERT INTO t_auto_partition VALUES + (4, 'delta', '2026-07-01'), + (5, 'epsilon', '2026-07-03'), + (6, 'default_partition', NULL) + """ + order_qt_ao_auto_partition_data """ + SELECT id, name, dt FROM t_auto_partition ORDER BY id + """ + assertTableEquals("t_auto_partition", "ORDER BY id") + + def sparkPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_auto_partition\$partitions` + ORDER BY `partition` + """ + def dorisPartitions = sql """ + SELECT `partition`, record_count + FROM t_auto_partition\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + order_qt_ao_auto_partition_metadata """ + SELECT `partition`, record_count + FROM t_auto_partition\$partitions + ORDER BY `partition` + """ + + // FT-014: Empty INSERT — should succeed with 0 rows + sql """INSERT INTO t_append_empty SELECT 1, 'test' WHERE 1 = 0""" + sql """INSERT INTO t_append_empty (id) VALUES (1)""" + sql """INSERT INTO t_append_empty (name, id) VALUES ('reordered', 2)""" + order_qt_ao_empty """SELECT id, name FROM t_append_empty ORDER BY id""" + assertTableEquals("t_append_empty", "ORDER BY id") + + // FT-043: Only omitted fields are filled from the real Paimon schema. + sql """INSERT INTO t_append_default (id) VALUES (1)""" + order_qt_ao_default_value """SELECT id, name FROM t_append_default ORDER BY id""" + assertTableEquals("t_append_default", "ORDER BY id") + + // Explicit NULL remains an input value. Paimon checks the real NOT NULL + // schema before applying its writer-side default wrapper. + test { + sql """INSERT INTO t_append_default (name, id) VALUES (NULL, 2)""" + exception "Cannot write null to non-null column(name)" + } + order_qt_ao_default_after_explicit_null """ + SELECT id, name FROM t_append_default ORDER BY id + """ + + // Doris does not duplicate Paimon's nullability validation. An omitted + // NOT NULL field without a default remains NULL and is rejected by the + // writer against the real Paimon schema. + test { + sql """INSERT INTO t_append_required (id) VALUES (1)""" + exception "Cannot write null to non-null column(name)" + } + + // A defaulted partition field uses the schema default as its logical and + // physical partition value instead of the configured null-partition name. + sql """INSERT INTO t_partition_default (name, id) VALUES ('omitted-partition', 1)""" + order_qt_ao_partition_default_data """ + SELECT id, name, dt FROM t_partition_default ORDER BY id + """ + assertTableEquals("t_partition_default", "ORDER BY id") + def sparkDefaultPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_partition_default\$partitions` + ORDER BY `partition` + """ + def dorisDefaultPartitions = sql """ + SELECT `partition`, record_count + FROM t_partition_default\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkDefaultPartitions, dorisDefaultPartitions) + order_qt_ao_partition_default_metadata """ + SELECT `partition`, record_count + FROM t_partition_default\$partitions + ORDER BY `partition` + """ + test { + sql """INSERT INTO t_partition_default (id, name, dt) + VALUES (2, 'explicit-null-partition', NULL)""" + exception "Cannot write null to non-null column(dt)" + } + + // FT-044: Duplicate target columns are rejected case-insensitively. + test { + sql """INSERT INTO t_append (id, ID) VALUES (8, 9)""" + exception "Duplicate column" + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_bucket_modes.groovy b/regression-test/suites/paimon_write/test_paimon_write_bucket_modes.groovy new file mode 100644 index 00000000000000..08e9fd85b04bd2 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_bucket_modes.groovy @@ -0,0 +1,497 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_bucket_modes", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_bucket_catalog" + String dbName = "test_pw_bucket_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_fixed; + CREATE TABLE paimon.${dbName}.t_hash_fixed ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '4', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_dynamic; + CREATE TABLE paimon.${dbName}.t_hash_dynamic ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.initial-buckets' = '1', + 'dynamic-bucket.max-buckets' = '4' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_dynamic_partial; + CREATE TABLE paimon.${dbName}.t_hash_dynamic_partial ( + pt STRING, id INT, name STRING, score INT + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'partial-update' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_hash_dynamic_overwrite; + CREATE TABLE paimon.${dbName}.t_hash_dynamic_overwrite ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.max-buckets' = '4' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic; + CREATE TABLE paimon.${dbName}.t_key_dynamic ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.max-buckets' = '4' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_partial; + CREATE TABLE paimon.${dbName}.t_key_dynamic_partial ( + pt STRING, id INT, name STRING, score INT + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'partial-update' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_first_row; + CREATE TABLE paimon.${dbName}.t_key_dynamic_first_row ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'first-row' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_aggregation; + CREATE TABLE paimon.${dbName}.t_key_dynamic_aggregation ( + pt STRING, id INT, total BIGINT + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_key_dynamic_scale; + CREATE TABLE paimon.${dbName}.t_key_dynamic_scale ( + pt STRING, id BIGINT, payload STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '128', + 'dynamic-bucket.max-buckets' = '16' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_bucket_unaware; + CREATE TABLE paimon.${dbName}.t_bucket_unaware ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'bucket' = '-1' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_postpone; + CREATE TABLE paimon.${dbName}.t_postpone ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-2', + 'postpone.default-bucket-num' = '2' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def bucketIds = { String tableName -> + def rows = spark_paimon """ + SELECT DISTINCT bucket + FROM paimon.${dbName}.`${tableName}\$files` + ORDER BY bucket + """ + return rows.collect { row -> row[0].toString().toInteger() } + } + + def assertBucketsInRange = { String tableName, int minBucket, int maxBucket -> + def buckets = bucketIds(tableName) + assertFalse(buckets.isEmpty()) + assertTrue(buckets.every { bucket -> + bucket >= minBucket && bucket <= maxBucket + }) + return buckets + } + + // Dynamic bucket modes must gather into one fragment instance and one JNI writer. + def hashDynamicPlan = sql """ + EXPLAIN SHAPE PLAN + INSERT INTO t_hash_dynamic + SELECT 'plan_only', CAST(number AS INT), 'unused' + FROM numbers("number" = "8") + """ + assertTrue(hashDynamicPlan.flatten().join("\n").contains("DistributionSpecGather")) + + def keyDynamicPlan = sql """ + EXPLAIN SHAPE PLAN + INSERT INTO t_key_dynamic + SELECT 'plan_only', CAST(number AS INT), 'unused' + FROM numbers("number" = "8") + """ + assertTrue(keyDynamicPlan.flatten().join("\n").contains("DistributionSpecGather")) + + // HASH_FIXED: SDK computes the fixed bucket from bucket-key=id. + sql """ + INSERT INTO t_hash_fixed + SELECT concat('p', CAST(number % 2 AS STRING)), + CAST(number AS INT), + concat('fixed_', CAST(number AS STRING)) + FROM numbers("number" = "16") + """ + qt_bucket_hash_fixed """ + SELECT COUNT(*), MIN(id), MAX(id), COUNT(DISTINCT pt) + FROM t_hash_fixed + """ + assertTableEquals("t_hash_fixed", "ORDER BY pt, id") + def fixedBuckets = assertBucketsInRange("t_hash_fixed", 0, 3) + assertTrue(fixedBuckets.size() > 1) + + // HASH_DYNAMIC: new keys expand buckets independently per partition. + sql """INSERT INTO t_hash_dynamic VALUES + ('p1', 1, 'v1'), + ('p1', 2, 'v2'), + ('p1', 3, 'v3'), + ('p1', 4, 'v4'), + ('p1', 5, 'v5'), + ('p1', 6, 'v6'), + ('p2', 1, 'p2_v1'), + ('p2', 2, 'p2_v2') + """ + assertTableEquals("t_hash_dynamic", "ORDER BY pt, id") + def dynamicBucketsBeforeUpdate = + assertBucketsInRange("t_hash_dynamic", 0, 3) + assertTrue(dynamicBucketsBeforeUpdate.size() > 1) + + // A new Doris transaction must load the existing hash index. Updating only + // existing keys must not allocate another bucket. + sql """INSERT INTO t_hash_dynamic VALUES + ('p1', 1, 'v1_updated'), + ('p1', 4, 'v4_updated'), + ('p2', 2, 'p2_v2_updated') + """ + order_qt_bucket_hash_dynamic """ + SELECT pt, id, name FROM t_hash_dynamic ORDER BY pt, id + """ + assertTableEquals("t_hash_dynamic", "ORDER BY pt, id") + assertEquals(dynamicBucketsBeforeUpdate, bucketIds("t_hash_dynamic")) + + // Dynamic bucket and partial-update share the same normalized table row. + sql """INSERT INTO t_hash_dynamic_partial VALUES + ('p1', 1, 'alice', 10), + ('p1', 2, 'bob', 20) + """ + sql """INSERT INTO t_hash_dynamic_partial (pt, id, score) VALUES + ('p1', 1, 15), + ('p1', 3, 30) + """ + order_qt_bucket_hash_dynamic_partial """ + SELECT pt, id, name, score FROM t_hash_dynamic_partial ORDER BY pt, id + """ + assertTableEquals("t_hash_dynamic_partial", "ORDER BY pt, id") + assertBucketsInRange("t_hash_dynamic_partial", 0, Integer.MAX_VALUE) + + // HASH_DYNAMIC overwrite uses the SDK's overwrite assigner and replaces + // both data files and the dynamic hash index. + sql """INSERT INTO t_hash_dynamic_overwrite VALUES + (1, 'old_1'), (2, 'old_2'), (3, 'old_3'), (4, 'old_4') + """ + sql """INSERT OVERWRITE TABLE t_hash_dynamic_overwrite VALUES + (10, 'new_10'), (11, 'new_11'), (12, 'new_12') + """ + order_qt_bucket_hash_dynamic_overwrite """ + SELECT id, name FROM t_hash_dynamic_overwrite ORDER BY id + """ + assertTableEquals("t_hash_dynamic_overwrite", "ORDER BY id") + def overwriteRows = sql """ + SELECT id, name FROM t_hash_dynamic_overwrite ORDER BY id + """ + assertEquals([ + [10, "new_10"], + [11, "new_11"], + [12, "new_12"] + ], overwriteRows) + assertBucketsInRange("t_hash_dynamic_overwrite", 0, 3) + + // KEY_DYNAMIC: the second statement bootstraps the existing global index. + // Deduplicate moves an existing primary key to its new partition. + sql """INSERT INTO t_key_dynamic VALUES + ('p1', 1, 'id1_old'), + ('p2', 2, 'id2_stable'), + ('p1', 3, 'id3_old') + """ + sql """INSERT INTO t_key_dynamic VALUES + ('p2', 1, 'id1_moved'), + ('p3', 3, 'id3_moved'), + ('p2', 4, 'id4_added') + """ + order_qt_bucket_key_dynamic """ + SELECT pt, id, name FROM t_key_dynamic ORDER BY id + """ + assertTableEquals("t_key_dynamic", "ORDER BY id") + def keyDynamicRows = sql """ + SELECT pt, id, name FROM t_key_dynamic ORDER BY id + """ + assertEquals([ + ["p2", 1, "id1_moved"], + ["p2", 2, "id2_stable"], + ["p3", 3, "id3_moved"], + ["p2", 4, "id4_added"] + ], keyDynamicRows) + assertBucketsInRange("t_key_dynamic", 0, 3) + + // For cross-partition partial-update, the global index keeps the old + // partition and applies the new non-null fields there. + sql """INSERT INTO t_key_dynamic_partial VALUES + ('p1', 10, 'old_10', 10), + ('p2', 20, 'stable_20', 20) + """ + sql """INSERT INTO t_key_dynamic_partial (pt, id, score) VALUES + ('p9', 10, 15), + ('p3', 30, 30) + """ + order_qt_bucket_key_dynamic_partial """ + SELECT pt, id, name, score FROM t_key_dynamic_partial ORDER BY id + """ + assertTableEquals("t_key_dynamic_partial", "ORDER BY id") + def keyDynamicPartialRows = sql """ + SELECT pt, id, name, score FROM t_key_dynamic_partial ORDER BY id + """ + assertEquals([ + ["p1", 10, "old_10", 15], + ["p2", 20, "stable_20", 20], + ["p3", 30, null, 30] + ], keyDynamicPartialRows) + + // FIRST_ROW ignores a later value even if it arrives in another partition. + sql """INSERT INTO t_key_dynamic_first_row VALUES + ('p1', 1, 'first_1') + """ + sql """INSERT INTO t_key_dynamic_first_row VALUES + ('p2', 1, 'ignored_1'), + ('p2', 2, 'first_2') + """ + order_qt_bucket_key_dynamic_first_row """ + SELECT pt, id, name FROM t_key_dynamic_first_row ORDER BY id + """ + assertTableEquals("t_key_dynamic_first_row", "ORDER BY id") + def keyDynamicFirstRowRows = sql """ + SELECT pt, id, name FROM t_key_dynamic_first_row ORDER BY id + """ + assertEquals([ + ["p1", 1, "first_1"], + ["p2", 2, "first_2"] + ], keyDynamicFirstRowRows) + + // Aggregation also stays in the original partition and combines values. + sql """INSERT INTO t_key_dynamic_aggregation VALUES + ('p1', 1, 10) + """ + sql """INSERT INTO t_key_dynamic_aggregation VALUES + ('p9', 1, 7), + ('p2', 2, 20) + """ + order_qt_bucket_key_dynamic_aggregation """ + SELECT pt, id, total FROM t_key_dynamic_aggregation ORDER BY id + """ + assertTableEquals("t_key_dynamic_aggregation", "ORDER BY id") + def keyDynamicAggregationRows = sql """ + SELECT pt, id, total FROM t_key_dynamic_aggregation ORDER BY id + """ + assertEquals([ + ["p1", 1, 17L], + ["p2", 2, 20L] + ], keyDynamicAggregationRows) + + // Bootstrap a larger KEY_DYNAMIC global index across multiple partitions + // and transactions. REFRESH CATALOG forces the next statement to reopen + // table metadata and construct a new JNI writer before restoring the index. + sql """ + INSERT INTO t_key_dynamic_scale + SELECT concat('p', CAST(number % 16 AS STRING)), + number, + concat('txn1_', CAST(number AS STRING)) + FROM numbers("number" = "4096") + """ + sql """ + INSERT INTO t_key_dynamic_scale + SELECT concat('p', CAST((number + 3) % 16 AS STRING)), + number, + concat('txn2_', CAST(number AS STRING)) + FROM numbers("number" = "2048") + """ + sql """REFRESH CATALOG ${catalogName}""" + sql """SWITCH ${catalogName}""" + sql """USE ${dbName}""" + sql """ + INSERT INTO t_key_dynamic_scale + SELECT concat('p', CAST((number + 5) % 16 AS STRING)), + number + 2048, + concat('txn3_', CAST(number + 2048 AS STRING)) + FROM numbers("number" = "2048") + """ + def keyDynamicScaleSummary = sql """ + SELECT COUNT(*), COUNT(DISTINCT id), MIN(id), MAX(id), SUM(id), + COUNT(DISTINCT pt), + SUM(IF(payload LIKE 'txn2_%', 1, 0)), + SUM(IF(payload LIKE 'txn3_%', 1, 0)) + FROM t_key_dynamic_scale + """ + assertEquals([[4096L, 4096L, 0L, 4095L, 8386560L, 16L, 2048L, 2048L]], + keyDynamicScaleSummary) + assertEquals(3L, + (sql """SELECT COUNT(*) FROM t_key_dynamic_scale\$snapshots""")[0][0] as long) + assertBucketsInRange("t_key_dynamic_scale", 0, 15) + def sparkScaleSummary = spark_paimon """ + SELECT COUNT(*), COUNT(DISTINCT id), MIN(id), MAX(id), SUM(id), + COUNT(DISTINCT pt), + SUM(CASE WHEN payload LIKE 'txn2_%' THEN 1 ELSE 0 END), + SUM(CASE WHEN payload LIKE 'txn3_%' THEN 1 ELSE 0 END) + FROM paimon.${dbName}.t_key_dynamic_scale + """ + assertSparkDorisResultEquals(sparkScaleSummary, keyDynamicScaleSummary) + order_qt_bucket_key_dynamic_scale_samples """ + SELECT pt, id, payload + FROM t_key_dynamic_scale + WHERE id IN (0, 1023, 2047, 2048, 3071, 4095) + ORDER BY id + """ + + // BUCKET_UNAWARE: append-only writers remain parallel while all files use bucket 0. + sql """SET parallel_pipeline_task_num = 4""" + sql """ + INSERT INTO t_bucket_unaware + SELECT concat('p', CAST(number % 2 AS STRING)), + CAST(number AS INT), + concat('unaware_', CAST(number AS STRING)) + FROM numbers("number" = "32") + """ + sql """SET parallel_pipeline_task_num = 0""" + qt_bucket_unaware """ + SELECT COUNT(*), MIN(id), MAX(id), COUNT(DISTINCT pt) + FROM t_bucket_unaware + """ + assertTableEquals("t_bucket_unaware", "ORDER BY pt, id") + assertEquals([0], bucketIds("t_bucket_unaware")) + + // POSTPONE_MODE commits files to bucket -2. Paimon deliberately + // excludes those files from readers and the files system table until + // an external compaction job assigns final buckets. + sql """INSERT INTO t_postpone VALUES + ('p1', 1, 'old_1'), + ('p1', 2, 'stable_2'), + ('p2', 3, 'stable_3') + """ + assertEquals([], bucketIds("t_postpone")) + assertTableEquals("t_postpone", "ORDER BY pt, id") + def postponeSnapshots = spark_paimon """ + SELECT COUNT(*) + FROM paimon.${dbName}.`t_postpone\$snapshots` + """ + assertEquals(1, postponeSnapshots[0][0].toString().toInteger()) + + sql """INSERT INTO t_postpone VALUES + ('p1', 1, 'new_1'), + ('p2', 4, 'added_4') + """ + assertEquals([], bucketIds("t_postpone")) + assertTableEquals("t_postpone", "ORDER BY pt, id") + postponeSnapshots = spark_paimon """ + SELECT COUNT(*) + FROM paimon.${dbName}.`t_postpone\$snapshots` + """ + assertEquals(2, postponeSnapshots[0][0].toString().toInteger()) + qt_bucket_postpone """SELECT COUNT(*) FROM t_postpone""" + } finally { + sql """SET parallel_pipeline_task_num = 0""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_changelog_producer.groovy b/regression-test/suites/paimon_write/test_paimon_write_changelog_producer.groovy new file mode 100644 index 00000000000000..6744820e9dbe2c --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_changelog_producer.groovy @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_changelog_producer", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_changelog_catalog" + String dbName = "test_pw_changelog_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_input_partial; + CREATE TABLE paimon.${dbName}.t_input_partial ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'partial-update', + 'changelog-producer' = 'input' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_lookup; + CREATE TABLE paimon.${dbName}.t_lookup ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'changelog-producer' = 'lookup' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_lookup_aggregation; + CREATE TABLE paimon.${dbName}.t_lookup_aggregation ( + id INT, total BIGINT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum', + 'changelog-producer' = 'lookup' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_full_compaction; + CREATE TABLE paimon.${dbName}.t_full_compaction ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '1', + 'changelog-producer' = 'full-compaction', + 'changelog-producer.row-deduplicate' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_full_compaction_dynamic; + CREATE TABLE paimon.${dbName}.t_full_compaction_dynamic ( + pt STRING, id INT, name STRING + ) USING paimon + PARTITIONED BY (pt) + TBLPROPERTIES ( + 'primary-key' = 'pt,id', + 'bucket' = '-1', + 'dynamic-bucket.target-row-num' = '2', + 'dynamic-bucket.max-buckets' = '4', + 'changelog-producer' = 'full-compaction', + 'changelog-producer.row-deduplicate' = 'true' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def latestSnapshotId = { String tableName -> + def rows = spark_paimon """ + SELECT max(snapshot_id) + FROM paimon.${dbName}.`${tableName}\$snapshots` + """ + assertEquals(1, rows.size()) + assertTrue(rows[0][0] != null) + return rows[0][0].toString() + } + + def incrementalAuditLog = { tableName, columns, beforeSnapshot, afterSnapshot, orderBy -> + def rows = spark_paimon """ + SELECT ${columns} + FROM paimon_incremental_query( + 'paimon.${dbName}.`${tableName}\$audit_log`', + '${beforeSnapshot}', + '${afterSnapshot}' + ) + ${orderBy} + """ + return rows + } + + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Input producer preserves the incoming row kind and partial-update payload. + sql """INSERT INTO t_input_partial VALUES + (1, 'alice', 10), + (2, 'bob', 20) + """ + String inputBefore = latestSnapshotId("t_input_partial") + sql """INSERT INTO t_input_partial (id, score) VALUES + (1, 15), + (3, 30) + """ + String inputAfter = latestSnapshotId("t_input_partial") + def inputChanges = incrementalAuditLog( + "t_input_partial", "rowkind, id, name, score", inputBefore, inputAfter, + "ORDER BY id") + assertEquals([ + ["+I", 1, null, 15], + ["+I", 3, null, 30] + ], inputChanges) + order_qt_changelog_input_partial """ + SELECT id, name, score FROM t_input_partial ORDER BY id + """ + assertTableEquals("t_input_partial", "ORDER BY id") + + // Lookup producer resolves previous values and emits complete before/after rows. + sql """INSERT INTO t_lookup VALUES + (1, 'old', 10), + (2, 'stable', 20) + """ + String lookupBefore = latestSnapshotId("t_lookup") + sql """INSERT INTO t_lookup VALUES + (1, 'new', 11), + (3, 'added', 30) + """ + String lookupAfter = latestSnapshotId("t_lookup") + def lookupChanges = incrementalAuditLog( + "t_lookup", "rowkind, id, name, score", lookupBefore, lookupAfter, + """ORDER BY id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", 1, "old", 10], + ["+U", 1, "new", 11], + ["+I", 3, "added", 30] + ], lookupChanges) + order_qt_changelog_lookup """ + SELECT id, name, score FROM t_lookup ORDER BY id + """ + assertTableEquals("t_lookup", "ORDER BY id") + + // Lookup producer reports the values before and after aggregation. + sql """INSERT INTO t_lookup_aggregation VALUES + (1, 10), + (2, 20) + """ + String aggregationBefore = latestSnapshotId("t_lookup_aggregation") + sql """INSERT INTO t_lookup_aggregation VALUES + (1, 7), + (3, 30) + """ + String aggregationAfter = latestSnapshotId("t_lookup_aggregation") + def aggregationChanges = incrementalAuditLog( + "t_lookup_aggregation", "rowkind, id, total", + aggregationBefore, aggregationAfter, + """ORDER BY id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", 1, 10L], + ["+U", 1, 17L], + ["+I", 3, 30L] + ], aggregationChanges) + order_qt_changelog_lookup_aggregation """ + SELECT id, total FROM t_lookup_aggregation ORDER BY id + """ + assertTableEquals("t_lookup_aggregation", "ORDER BY id") + + // Full-compaction producer must compact every partition/bucket touched by the batch. + sql """INSERT INTO t_full_compaction VALUES + ('p1', 1, 'old'), + ('p2', 2, 'stable') + """ + String fullCompactionBefore = latestSnapshotId("t_full_compaction") + sql """INSERT INTO t_full_compaction VALUES + ('p1', 1, 'new'), + ('p2', 3, 'added') + """ + String fullCompactionAfter = latestSnapshotId("t_full_compaction") + def fullCompactionChanges = incrementalAuditLog( + "t_full_compaction", "rowkind, pt, id, name", + fullCompactionBefore, fullCompactionAfter, + """ORDER BY pt, id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", "p1", 1, "old"], + ["+U", "p1", 1, "new"], + ["+I", "p2", 3, "added"] + ], fullCompactionChanges) + order_qt_changelog_full_compaction """ + SELECT pt, id, name FROM t_full_compaction ORDER BY pt, id + """ + assertTableEquals("t_full_compaction", "ORDER BY pt, id") + + def fullCompactionSnapshot = spark_paimon """ + SELECT commit_kind, changelog_record_count + FROM paimon.${dbName}.`t_full_compaction\$snapshots` + WHERE snapshot_id = ${fullCompactionAfter} + """ + assertEquals([["COMPACT", 3L]], fullCompactionSnapshot) + + // HASH_DYNAMIC uses writer.write(row, assignedBucket). Combining it with + // full-compaction exercises the explicit-bucket writeAndReturn path and + // compacts every dynamically assigned partition/bucket touched by Doris. + sql """INSERT INTO t_full_compaction_dynamic VALUES + ('p1', 1, 'old_1'), + ('p1', 2, 'stable_2'), + ('p2', 3, 'old_3') + """ + String dynamicCompactionBefore = latestSnapshotId("t_full_compaction_dynamic") + sql """INSERT INTO t_full_compaction_dynamic VALUES + ('p1', 1, 'new_1'), + ('p1', 4, 'added_4'), + ('p2', 3, 'new_3'), + ('p2', 5, 'added_5') + """ + String dynamicCompactionAfter = latestSnapshotId("t_full_compaction_dynamic") + def dynamicCompactionChanges = incrementalAuditLog( + "t_full_compaction_dynamic", "rowkind, pt, id, name", + dynamicCompactionBefore, dynamicCompactionAfter, + """ORDER BY pt, id, + CASE rowkind WHEN '-U' THEN 0 WHEN '+U' THEN 1 ELSE 2 END""") + assertEquals([ + ["-U", "p1", 1, "old_1"], + ["+U", "p1", 1, "new_1"], + ["+I", "p1", 4, "added_4"], + ["-U", "p2", 3, "old_3"], + ["+U", "p2", 3, "new_3"], + ["+I", "p2", 5, "added_5"] + ], dynamicCompactionChanges) + order_qt_changelog_full_compaction_dynamic """ + SELECT pt, id, name + FROM t_full_compaction_dynamic + ORDER BY pt, id + """ + assertTableEquals("t_full_compaction_dynamic", "ORDER BY pt, id") + + def dynamicCompactionSnapshot = spark_paimon """ + SELECT commit_kind, changelog_record_count + FROM paimon.${dbName}.`t_full_compaction_dynamic\$snapshots` + WHERE snapshot_id = ${dynamicCompactionAfter} + """ + assertEquals([["COMPACT", 6L]], dynamicCompactionSnapshot) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_compaction.groovy b/regression-test/suites/paimon_write/test_paimon_write_compaction.groovy new file mode 100644 index 00000000000000..b94fd559838ebf --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_compaction.groovy @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_compaction", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_compaction_catalog" + String dbName = "test_pw_compaction_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_auto_compaction; + CREATE TABLE paimon.${dbName}.t_pk_auto_compaction ( + id INT, name STRING, score INT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'bucket-key' = 'id', + 'num-sorted-run.compaction-trigger' = '2', + 'target-file-size' = '1 gb' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_append_auto_compaction; + CREATE TABLE paimon.${dbName}.t_append_auto_compaction ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'bucket' = '1', + 'bucket-key' = 'id', + 'compaction.min.file-num' = '2', + 'target-file-size' = '1 gb' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """ + SELECT * FROM paimon.${dbName}.${tableName} ${orderBy} + """ + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def fetchFiles = { String tableName -> + def rows = spark_paimon """ + SELECT level, record_count, file_source + FROM paimon.${dbName}.`${tableName}\$files` + ORDER BY file_path + """ + return rows + } + + def fetchSnapshots = { String tableName -> + def rows = spark_paimon """ + SELECT snapshot_id, commit_kind + FROM paimon.${dbName}.`${tableName}\$snapshots` + ORDER BY snapshot_id + """ + return rows + } + + // A second primary-key write restores the existing L0 file. Two sorted + // runs trigger merge-tree compaction, which must merge the updated key + // and commit the compact increment produced by the JNI writer. + sql """INSERT INTO t_pk_auto_compaction VALUES + (1, 'old', 10), + (2, 'stable', 20) + """ + def pkFilesBefore = fetchFiles("t_pk_auto_compaction") + assertEquals(1, pkFilesBefore.size()) + assertEquals(0, pkFilesBefore[0][0].toString().toInteger()) + assertEquals(2L, pkFilesBefore[0][1].toString().toLong()) + assertEquals("APPEND", pkFilesBefore[0][2].toString()) + + sql """INSERT INTO t_pk_auto_compaction VALUES + (1, 'new', 11), + (3, 'added', 30) + """ + order_qt_compaction_pk """ + SELECT id, name, score FROM t_pk_auto_compaction ORDER BY id + """ + def pkRows = sql """SELECT id, name, score FROM t_pk_auto_compaction ORDER BY id""" + assertEquals([ + [1, "new", 11], + [2, "stable", 20], + [3, "added", 30] + ], pkRows) + assertTableEquals("t_pk_auto_compaction", "ORDER BY id") + + def pkFilesAfter = fetchFiles("t_pk_auto_compaction") + assertEquals(1, pkFilesAfter.size()) + assertTrue(pkFilesAfter[0][0].toString().toInteger() > 0) + assertEquals(3L, pkFilesAfter[0][1].toString().toLong()) + assertEquals("COMPACT", pkFilesAfter[0][2].toString()) + + def pkSnapshots = fetchSnapshots("t_pk_auto_compaction") + assertEquals(3, pkSnapshots.size()) + assertEquals(["APPEND", "APPEND", "COMPACT"], + pkSnapshots.collect { row -> row[1].toString() }) + + // Fixed-bucket append-only tables restore existing files for the bucket. + // The second write reaches compaction.min.file-num and rewrites both + // small APPEND files into one COMPACT file without losing duplicates. + sql """INSERT INTO t_append_auto_compaction VALUES + (1, 'a'), + (2, 'b') + """ + def appendFilesBefore = fetchFiles("t_append_auto_compaction") + assertEquals(1, appendFilesBefore.size()) + assertEquals(2L, appendFilesBefore[0][1].toString().toLong()) + assertEquals("APPEND", appendFilesBefore[0][2].toString()) + + sql """INSERT INTO t_append_auto_compaction VALUES + (3, 'c'), + (4, 'd') + """ + order_qt_compaction_append """ + SELECT id, name FROM t_append_auto_compaction ORDER BY id + """ + def appendRows = sql """SELECT id, name FROM t_append_auto_compaction ORDER BY id""" + assertEquals([ + [1, "a"], + [2, "b"], + [3, "c"], + [4, "d"] + ], appendRows) + assertTableEquals("t_append_auto_compaction", "ORDER BY id") + + def appendFilesAfter = fetchFiles("t_append_auto_compaction") + assertEquals(1, appendFilesAfter.size()) + assertEquals(4L, appendFilesAfter[0][1].toString().toLong()) + assertEquals("COMPACT", appendFilesAfter[0][2].toString()) + + def appendSnapshots = fetchSnapshots("t_append_auto_compaction") + assertEquals(3, appendSnapshots.size()) + assertEquals(["APPEND", "APPEND", "COMPACT"], + appendSnapshots.collect { row -> row[1].toString() }) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_complex_types.groovy b/regression-test/suites/paimon_write/test_paimon_write_complex_types.groovy new file mode 100644 index 00000000000000..42a63c17ac1dbd --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_complex_types.groovy @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_complex_types", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_cx_catalog" + String dbName = "test_pw_cx_db" + + spark_paimon_multi """ + SET spark.sql.binaryOutputStyle=HEX; + SET spark.sql.timestampType=TIMESTAMP_NTZ; + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_array; + CREATE TABLE paimon.${dbName}.t_array ( + id INT, + c_array_int ARRAY, + c_array_string ARRAY, + c_array_double ARRAY + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_map; + CREATE TABLE paimon.${dbName}.t_map ( + id INT, + c_map_str_int MAP, + c_map_int_str MAP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_struct; + CREATE TABLE paimon.${dbName}.t_struct ( + id INT, + c_struct STRUCT + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_nested; + CREATE TABLE paimon.${dbName}.t_nested ( + id INT, + c_map_arr MAP> + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_recursive; + CREATE TABLE paimon.${dbName}.t_recursive ( + id INT, + c_array_decimal ARRAY, + c_array_date ARRAY, + c_array_timestamp ARRAY, + c_map_decimal MAP, + c_struct_mixed STRUCT, + c_deep MAP>> + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_binary; + CREATE TABLE paimon.${dbName}.t_binary ( + id INT, + c_binary BINARY, + c_array_binary ARRAY, + c_map_binary MAP, + c_struct_binary STRUCT + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true', + 'enable.mapping.varbinary' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-028: ARRAY types — normal array, empty array, NULL array + sql """INSERT INTO t_array VALUES + (1, [1, 2, 3], ['a', 'b', 'c'], [1.1, 2.2]), + (2, [], [], []), + (3, [10, NULL, 30], ['x', NULL, 'z'], [NULL, 2.0]), + (4, NULL, NULL, NULL) + """ + order_qt_cx_array """SELECT id, c_array_int, c_array_string, c_array_double FROM t_array ORDER BY id""" + assertTableEquals("t_array", "ORDER BY id") + + // FT-029: MAP types — normal map, empty map, NULL value + sql """INSERT INTO t_map VALUES + (1, map('math', 90, 'eng', 95), map(1, 'one', 2, 'two')), + (2, map(), map()), + (3, map('science', NULL), map(3, NULL)), + (4, NULL, NULL) + """ + order_qt_cx_map """SELECT id, c_map_str_int, c_map_int_str FROM t_map ORDER BY id""" + assertTableEquals("t_map", "ORDER BY id") + + // FT-030: STRUCT types + sql """INSERT INTO t_struct VALUES + (1, named_struct('name', 'alice', 'age', 30)), + (2, named_struct('name', NULL, 'age', NULL)), + (3, NULL) + """ + order_qt_cx_struct """SELECT id, c_struct FROM t_struct ORDER BY id""" + assertTableEquals("t_struct", "ORDER BY id") + + // Nested: MAP> + sql """INSERT INTO t_nested VALUES + (1, map('group1', [1, 2], 'group2', [3, 4, 5])), + (2, map('empty', [])), + (3, NULL) + """ + order_qt_cx_nested """SELECT id, c_map_arr FROM t_nested ORDER BY id""" + assertTableEquals("t_nested", "ORDER BY id") + + // Recursive conversion covers the non-trivial Arrow child vectors which + // cannot use the primitive column fast path in PaimonArrowConverter. + sql """INSERT INTO t_recursive VALUES + ( + 1, + array(CAST(1.250000 AS DECIMAL(18, 6)), CAST(-2.500000 AS DECIMAL(18, 6))), + array(DATE '2024-01-01', DATE '2024-12-31'), + array(TIMESTAMP '2024-01-01 01:02:03.123456', + TIMESTAMP '2024-12-31 23:59:59.654321'), + map(CAST(1.25 AS DECIMAL(8, 2)), CAST(2.50 AS DECIMAL(8, 2)), + CAST(-3.75 AS DECIMAL(8, 2)), CAST(4.00 AS DECIMAL(8, 2))), + named_struct( + 'flag', true, + 'amount', CAST(123.456789 AS DECIMAL(18, 6)), + 'event_date', DATE '2024-02-29', + 'event_time', TIMESTAMP '2024-02-29 12:34:56.000001'), + map('term', array( + named_struct('score', 90, 'label', 'good'), + named_struct('score', 95, 'label', 'better') + )) + ), + ( + 2, + array(CAST(NULL AS DECIMAL(18, 6)), CAST(0.000001 AS DECIMAL(18, 6))), + array(CAST(NULL AS DATE), DATE '1970-01-01'), + array(CAST(NULL AS DATETIME(6)), TIMESTAMP '1970-01-01 00:00:00.000001'), + map(CAST(5.25 AS DECIMAL(8, 2)), CAST(NULL AS DECIMAL(8, 2))), + named_struct( + 'flag', CAST(NULL AS BOOLEAN), + 'amount', CAST(NULL AS DECIMAL(18, 6)), + 'event_date', CAST(NULL AS DATE), + 'event_time', CAST(NULL AS DATETIME(6))), + map('nullable', array( + named_struct('score', CAST(NULL AS INT), 'label', CAST(NULL AS STRING)) + )) + ), + (3, [], [], [], map(), named_struct( + 'flag', false, + 'amount', CAST(0 AS DECIMAL(18, 6)), + 'event_date', DATE '1970-01-01', + 'event_time', TIMESTAMP '1970-01-01 00:00:00'), map()) + """ + + // Every projected field is deliberately in reverse table order. This + // verifies that target type conversion follows Doris input order while + // PaimonWriteSchema restores canonical table-schema order. + sql """INSERT INTO t_recursive ( + c_deep, c_struct_mixed, c_map_decimal, c_array_timestamp, + c_array_date, c_array_decimal, id + ) VALUES ( + map('reverse', array(named_struct('score', 88, 'label', 'reordered'))), + named_struct( + 'flag', true, + 'amount', CAST(8.800000 AS DECIMAL(18, 6)), + 'event_date', DATE '2025-01-01', + 'event_time', TIMESTAMP '2025-01-01 08:08:08.000008'), + map(CAST(8.80 AS DECIMAL(8, 2)), CAST(9.90 AS DECIMAL(8, 2))), + array(TIMESTAMP '2025-01-01 00:00:00.000008'), + array(DATE '2025-01-01'), + array(CAST(8.800008 AS DECIMAL(18, 6))), + 4 + ) + """ + + // A reordered subset expands to a full table row with NULL in every + // omitted nullable field. + sql """INSERT INTO t_recursive (c_deep, c_array_date, id) VALUES ( + map('partial', array(named_struct('score', 77, 'label', 'subset'))), + array(DATE '2026-01-01'), + 5 + )""" + order_qt_cx_recursive """SELECT * FROM t_recursive ORDER BY id""" + assertTableEquals("t_recursive", "ORDER BY id") + + // Top-level and recursively nested BINARY values exercise both the Arrow + // VarBinaryVector fast path and nested convertVectorValue branches. + sql """INSERT INTO t_binary VALUES + ( + 1, + X'0001FEFF', + [X'41', X'00FF'], + map('payload', X'102030'), + named_struct('label', 'binary_1', 'payload', X'DEADBEEF') + ), + ( + 2, + NULL, + [], + map(), + named_struct('label', 'binary_2', 'payload', CAST(NULL AS VARBINARY)) + ), + (3, X'E4B8ADE69687', NULL, NULL, NULL) + """ + // Reordering binary and nested columns also verifies that their target + // Paimon types are resolved by projected column name rather than position. + sql """INSERT INTO t_binary ( + c_struct_binary, c_map_binary, c_array_binary, c_binary, id + ) VALUES ( + named_struct('label', 'reordered', 'payload', X'ABCD'), + map('payload', X'0102'), + [X'03', X'0405'], + X'060708', + 4 + ) + """ + order_qt_cx_binary """ + SELECT id, + HEX(c_binary), + SIZE(c_array_binary), + HEX(ELEMENT_AT(c_array_binary, 1)), + SIZE(c_map_binary), + HEX(ELEMENT_AT(c_map_binary, 'payload')), + c_struct_binary.label, + HEX(c_struct_binary.payload) + FROM t_binary + ORDER BY id + """ + def sparkBinaryRows = spark_paimon """ + SELECT id, + HEX(c_binary), + CASE WHEN SIZE(c_array_binary) >= 1 + THEN HEX(ELEMENT_AT(c_array_binary, 1)) END, + CASE WHEN SIZE(c_array_binary) >= 2 + THEN HEX(ELEMENT_AT(c_array_binary, 2)) END, + HEX(ELEMENT_AT(c_map_binary, 'payload')), + c_struct_binary.label, + HEX(c_struct_binary.payload) + FROM paimon.${dbName}.t_binary + ORDER BY id + """ + def dorisBinaryRows = sql """ + SELECT id, + HEX(c_binary), + CASE WHEN SIZE(c_array_binary) >= 1 + THEN HEX(ELEMENT_AT(c_array_binary, 1)) END, + CASE WHEN SIZE(c_array_binary) >= 2 + THEN HEX(ELEMENT_AT(c_array_binary, 2)) END, + HEX(ELEMENT_AT(c_map_binary, 'payload')), + c_struct_binary.label, + HEX(c_struct_binary.payload) + FROM t_binary + ORDER BY id + """ + assertSparkDorisResultEquals(sparkBinaryRows, dorisBinaryRows) + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_edge_cases.groovy b/regression-test/suites/paimon_write/test_paimon_write_edge_cases.groovy new file mode 100644 index 00000000000000..167ae149ed0bfa --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_edge_cases.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_edge_cases", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_edge_catalog" + String dbName = "test_pw_edge_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_str; + CREATE TABLE paimon.${dbName}.t_edge_str ( + id INT, c_string STRING, c_varchar VARCHAR(10) + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_numeric; + CREATE TABLE paimon.${dbName}.t_edge_numeric ( + id INT, + c_tiny TINYINT, + c_small SMALLINT, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_bool; + CREATE TABLE paimon.${dbName}.t_edge_bool ( + id INT, c_bool BOOLEAN + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_edge_pk_null; + CREATE TABLE paimon.${dbName}.t_edge_pk_null ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ('primary-key' = 'id', 'bucket' = '1', 'bucket-key' = 'id'); + + DROP TABLE IF EXISTS paimon.${dbName}.t_mixed_write; + CREATE TABLE paimon.${dbName}.t_mixed_write ( + id INT, name STRING, score DOUBLE + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String columns, String orderBy -> + def sparkRows = spark_paimon """SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-042: Empty string and boundary VARCHAR + sql """INSERT INTO t_edge_str VALUES + (1, '', ''), + (2, 'hello world', 'short_str'), + (3, 'x', 'abcdefghij'), + (4, 'very long string over 100 chars: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890', 'max10chars') + """ + order_qt_edge_str """SELECT id, c_varchar FROM t_edge_str ORDER BY id""" + assertTableEquals("t_edge_str", "*", "ORDER BY id") + + // FT-021: Numeric boundary values (INT_MIN, INT_MAX, etc.) + sql """INSERT INTO t_edge_numeric VALUES + (1, CAST(127 AS TINYINT), CAST(32767 AS SMALLINT), 2147483647, + CAST(9223372036854775807 AS BIGINT), + CAST(3.4028235E38 AS FLOAT), CAST(1.7976931348623157E308 AS DOUBLE)), + (2, CAST(-128 AS TINYINT), CAST(-32768 AS SMALLINT), -2147483648, + CAST(-9223372036854775808 AS BIGINT), + CAST(-3.4028235E38 AS FLOAT), CAST(-1.7976931348623157E308 AS DOUBLE)), + (3, CAST(0 AS TINYINT), CAST(0 AS SMALLINT), 0, + CAST(0 AS BIGINT), + CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE)) + """ + order_qt_edge_numeric """SELECT id, c_tiny, c_small, c_int, c_bigint FROM t_edge_numeric ORDER BY id""" + assertTableEquals("t_edge_numeric", """ + id, c_tiny, c_small, c_int, c_bigint, + c_float / 1.0E38, + c_double / 1.0E308 + """, "ORDER BY id") + + // BOOLEAN with NULL and both true/false + sql """INSERT INTO t_edge_bool VALUES (1, true), (2, false), (3, NULL)""" + order_qt_edge_bool """SELECT id, c_bool FROM t_edge_bool ORDER BY id""" + assertTableEquals("t_edge_bool", "*", "ORDER BY id") + + // FT-041: PK table — insert NULL values, then update with non-NULL + sql """INSERT INTO t_edge_pk_null VALUES (1, 'first'), (2, NULL)""" + assertTableEquals("t_edge_pk_null", "*", "ORDER BY id") + + sql """INSERT INTO t_edge_pk_null VALUES (2, 'updated'), (3, 'third')""" + order_qt_edge_pk_null """SELECT id, name FROM t_edge_pk_null ORDER BY id""" + assertTableEquals("t_edge_pk_null", "*", "ORDER BY id") + + // Mixed INSERT patterns: VALUES, then SELECT from self, then single-row VALUES + sql """INSERT INTO t_mixed_write VALUES (1, 'a', 10.0), (2, 'b', 20.0)""" + sql """INSERT INTO t_mixed_write VALUES (3, 'c', 30.0)""" + sql """INSERT INTO t_mixed_write SELECT id + 3, concat(name, '_copy'), score + 30.0 FROM t_mixed_write""" + order_qt_edge_mixed """SELECT id, name, score FROM t_mixed_write ORDER BY id""" + assertTableEquals("t_mixed_write", "*", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_failures.groovy b/regression-test/suites/paimon_write/test_paimon_write_failures.groovy new file mode 100644 index 00000000000000..12c7b1316cb9a0 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_failures.groovy @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_failures", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_failure_catalog" + String dbName = "test_pw_failure_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_atomic_append; + CREATE TABLE paimon.${dbName}.t_atomic_append ( + id INT NOT NULL, + payload STRING NOT NULL, + dt STRING NOT NULL + ) USING paimon + PARTITIONED BY (dt); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_not_null; + CREATE TABLE paimon.${dbName}.t_pk_not_null ( + id INT NOT NULL, + payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertAtomicAppendState = { long expectedRows, long expectedSnapshots -> + assertEquals(expectedRows, + (sql """SELECT COUNT(*) FROM t_atomic_append""")[0][0] as long) + assertEquals(expectedSnapshots, + (sql """SELECT COUNT(*) FROM t_atomic_append\$snapshots""")[0][0] as long) + } + + // A failure after an earlier row has already entered the JNI writer must + // abort the whole statement, including data for a different partition. + sql """INSERT INTO t_atomic_append VALUES (1, 'baseline', 'p0')""" + order_qt_failure_atomic_before """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_atomic_snapshot_before """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + + test { + sql """INSERT INTO t_atomic_append VALUES + (2, 'accepted_before_error', 'p1'), + (3, NULL, 'p2')""" + exception "Cannot write null to non-null column(payload)" + } + assertAtomicAppendState(1L, 1L) + order_qt_failure_atomic_after """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_atomic_snapshot_after """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + + // An omitted field without a Paimon default remains NULL and is validated + // against the real Paimon schema by the Paimon writer. + test { + sql """INSERT INTO t_atomic_append (id, dt) VALUES (4, 'p4')""" + exception "Cannot write null to non-null column(payload)" + } + + // Partition columns follow the same Paimon nullability contract. + test { + sql """INSERT INTO t_atomic_append VALUES (4, 'bad_partition', NULL)""" + exception "Cannot write null to non-null column(dt)" + } + assertAtomicAppendState(1L, 1L) + + // These errors are rejected during target-column and partition binding and + // therefore must not create a writer or a new Paimon snapshot. + test { + sql """INSERT INTO t_atomic_append (id, payload, dt, missing) + VALUES (5, 'unknown_column', 'p5', 1)""" + exception "Unknown column 'missing' in target table" + } + test { + sql """INSERT INTO t_atomic_append (id, payload, dt) + VALUES (5, 'too_few_values')""" + exception "Column count doesn't match value count" + } + test { + sql """INSERT OVERWRITE TABLE t_atomic_append + PARTITION (payload = 'not_a_partition') VALUES (5, 'p5')""" + exception "is not a partition column of Paimon table" + } + assertAtomicAppendState(1L, 1L) + + // A successful statement after several failures verifies that failed JNI + // writers and transactions do not poison subsequent writes. + sql """INSERT INTO t_atomic_append VALUES (5, 'recovered', 'p5')""" + assertAtomicAppendState(2L, 2L) + order_qt_failure_recovered """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_recovered_snapshot """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + + // A failed overwrite must not publish its replacement files or remove the + // data referenced by the previous committed snapshot. + test { + sql """INSERT OVERWRITE TABLE t_atomic_append VALUES + (10, 'would_replace', 'p10'), + (11, NULL, 'p11')""" + exception "Cannot write null to non-null column(payload)" + } + assertAtomicAppendState(2L, 2L) + order_qt_failure_overwrite_after """ + SELECT id, payload, dt FROM t_atomic_append ORDER BY id + """ + qt_failure_overwrite_snapshot_after """ + SELECT COUNT(*) FROM t_atomic_append\$snapshots + """ + assertTableEquals("t_atomic_append", "ORDER BY id") + + // Primary-key nullability is checked before bucket routing. A rejected row + // must publish no snapshot, and the table remains writable afterwards. + test { + sql """INSERT INTO t_pk_not_null VALUES (NULL, 'invalid_key')""" + exception "Cannot write null to non-null column(id)" + } + assertEquals(0L, + (sql """SELECT COUNT(*) FROM t_pk_not_null\$snapshots""")[0][0] as long) + + sql """INSERT INTO t_pk_not_null VALUES (1, 'valid_after_failure')""" + order_qt_failure_pk_recovered """ + SELECT id, payload FROM t_pk_not_null ORDER BY id + """ + qt_failure_pk_snapshot """ + SELECT COUNT(*) FROM t_pk_not_null\$snapshots + """ + assertTableEquals("t_pk_not_null", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_merge_engine.groovy b/regression-test/suites/paimon_write/test_paimon_write_merge_engine.groovy new file mode 100644 index 00000000000000..a0dbdd944c4870 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_merge_engine.groovy @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_merge_engine", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_merge_engine_catalog" + String dbName = "test_pw_merge_engine_db" + + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_partial_update; + CREATE TABLE paimon.${dbName}.t_partial_update ( + id INT, name STRING, score DOUBLE, note STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'partial-update' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_first_row; + CREATE TABLE paimon.${dbName}.t_first_row ( + id INT, name STRING, score DOUBLE + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'first-row' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_aggregation; + CREATE TABLE paimon.${dbName}.t_aggregation ( + id INT, total BIGINT, highest DOUBLE, label STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id', + 'merge-engine' = 'aggregation', + 'fields.total.aggregate-function' = 'sum', + 'fields.highest.aggregate-function' = 'max' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // Partial-update accepts both full rows and arbitrary value-column subsets. + sql """INSERT INTO t_partial_update VALUES + (1, 'alice', 10.0, 'created'), + (2, 'bob', 20.0, 'created') + """ + sql """INSERT INTO t_partial_update (score, id) VALUES (15.5, 1)""" + sql """INSERT INTO t_partial_update (note, id) VALUES ('score_updated', 1)""" + sql """INSERT INTO t_partial_update (id, name) VALUES (1, NULL)""" + sql """INSERT INTO t_partial_update (name, id) VALUES ('charlie', 3)""" + sql """INSERT INTO t_partial_update VALUES (2, 'bob_full', 25.0, 'full_update')""" + order_qt_partial_update """SELECT id, name, score, note + FROM t_partial_update ORDER BY id""" + assertTableEquals("t_partial_update", "ORDER BY id") + + // An omitted primary key reaches the SDK as NULL in the complete table row. + // Let Paimon's real NOT NULL schema enforce the primary-key requirement. + test { + sql """INSERT INTO t_partial_update (name, score) VALUES ('missing_pk', 1.0)""" + exception "Cannot write null to non-null column(id)" + } + + // First-row keeps the first value observed for each primary key across writes. + sql """INSERT INTO t_first_row VALUES + (1, 'first_1', 10.0), + (2, 'first_2', 20.0) + """ + sql """INSERT INTO t_first_row VALUES + (2, 'second_2', 21.0), + (1, 'second_1', 11.0), + (3, 'first_3', 30.0) + """ + sql """INSERT INTO t_first_row VALUES (1, 'third_1', 12.0)""" + order_qt_first_row """SELECT id, name, score FROM t_first_row ORDER BY id""" + assertTableEquals("t_first_row", "ORDER BY id") + + test { + sql """INSERT INTO t_first_row (id, name) VALUES (4, 'partial')""" + exception "table uses merge-engine=first-row" + } + + // Aggregation applies the configured function per value field. Fields without + // an explicit function use Paimon's default last_non_null_value aggregation. + sql """INSERT INTO t_aggregation VALUES + (1, 10, 90.0, 'first_1'), + (2, 5, 70.0, 'first_2') + """ + sql """INSERT INTO t_aggregation VALUES + (1, 20, 85.0, NULL), + (2, 3, 80.0, NULL), + (3, 7, 60.0, 'first_3') + """ + sql """INSERT INTO t_aggregation VALUES (1, 7, 95.0, 'latest_1')""" + order_qt_aggregation """SELECT id, total, highest, label + FROM t_aggregation ORDER BY id""" + assertTableEquals("t_aggregation", "ORDER BY id") + + test { + sql """INSERT INTO t_aggregation (id, total) VALUES (4, 100)""" + exception "table uses merge-engine=aggregation" + } + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_pk.groovy b/regression-test/suites/paimon_write/test_paimon_write_pk.groovy new file mode 100644 index 00000000000000..39496118f0e420 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_pk.groovy @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_pk", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_pk_catalog" + String dbName = "test_pw_pk_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_dedup; + CREATE TABLE paimon.${dbName}.t_pk_dedup ( + id INT, name STRING, score DOUBLE, ts BIGINT + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '2', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_bucket4; + CREATE TABLE paimon.${dbName}.t_pk_bucket4 ( + id INT, name STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '4', + 'bucket-key' = 'id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_composite; + CREATE TABLE paimon.${dbName}.t_pk_composite ( + user_id INT, event_time BIGINT, event_type STRING, value DOUBLE + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'user_id,event_time', + 'bucket' = '2', + 'bucket-key' = 'user_id' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_string_bucket; + CREATE TABLE paimon.${dbName}.t_pk_string_bucket ( + user_key STRING, event_id INT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'user_key,event_id', + 'bucket' = '4', + 'bucket-key' = 'user_key' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_pk_writer_scaling; + CREATE TABLE paimon.${dbName}.t_pk_writer_scaling ( + id INT, version BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'bucket-key' = 'id' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + // Prepare an internal OLAP source table for INSERT INTO ... SELECT + sql """create database if not exists internal.${dbName}""" + sql """drop table if exists internal.${dbName}.t_source""" + sql """ + CREATE TABLE internal.${dbName}.t_source ( + id INT, name STRING, score DOUBLE, ts BIGINT + ) ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES('replication_num'='1'); + """ + sql """INSERT INTO internal.${dbName}.t_source VALUES + (1, 'alice', 95.5, 1000), + (1, 'alice_updated', 99.0, 2000), + (2, 'bob', 87.0, 1000), + (3, 'charlie', 92.3, 1000), + (3, 'charlie_v2', 88.0, 1500), + (4, 'diana', 91.0, 1000), + (5, 'eve', 85.0, 1000) + """ + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-004: PK table, deduplicate — duplicate keys merged by Paimon SDK + sql """INSERT INTO t_pk_dedup SELECT id, name, score, ts FROM internal.${dbName}.t_source""" + order_qt_pk_dedup """SELECT id, name, score FROM t_pk_dedup ORDER BY id""" + assertTableEquals("t_pk_dedup", "ORDER BY id") + + // FT-005: Interleaved duplicate keys verify that SDK routing preserves the + // input order within each bucket and the last row for each key wins. + sql """INSERT INTO t_pk_dedup VALUES + (100, 'key100_v1', 10.0, 1000), + (200, 'key200_v1', 20.0, 1000), + (100, 'key100_v2', 11.0, 2000), + (200, 'key200_v2', 21.0, 2000), + (100, 'key100_v3', 12.0, 3000) + """ + order_qt_pk_interleaved """SELECT id, name, score, ts FROM t_pk_dedup + WHERE id >= 100 ORDER BY id""" + assertTableEquals("t_pk_dedup", "ORDER BY id") + + // FT-003: Fixed bucket table with multiple buckets + for (int i = 0; i < 20; i++) { + sql """INSERT INTO t_pk_bucket4 VALUES (${i}, 'row${i}')""" + } + order_qt_pk_bucket """SELECT id, name FROM t_pk_bucket4 ORDER BY id""" + assertTableEquals("t_pk_bucket4", "ORDER BY id") + + // PK table with composite primary key + sql """INSERT INTO t_pk_composite VALUES + (1, 100, 'click', 1.0), + (1, 200, 'view', 2.0), + (2, 100, 'click', 3.0), + (1, 100, 'click_updated', 99.0) + """ + // (1,100) duplicated → 3 unique PKs: (1,100), (1,200), (2,100) + order_qt_pk_composite """SELECT user_id, event_time, event_type, value FROM t_pk_composite ORDER BY user_id, event_time""" + assertTableEquals("t_pk_composite", "ORDER BY user_id, event_time") + + // FT-006: A string bucket key must use Paimon's string hash and preserve + // UTF-8 values while routing rows to fixed buckets. + sql """INSERT INTO t_pk_string_bucket VALUES + ('alpha', 1, 'alpha_v1'), + ('beta', 2, '中文_payload'), + ('emoji_😀', 3, 'emoji_payload'), + ('alpha', 1, 'alpha_v2') + """ + order_qt_pk_string_bucket """SELECT user_key, event_id, payload + FROM t_pk_string_bucket ORDER BY event_id""" + assertTableEquals("t_pk_string_bucket", "ORDER BY event_id") + + // FT-045: A fixed-bucket PK table uses one writer until Doris has a + // bucket-aware exchange, so all versions owned by one bucket stay together. + sql """SET parallel_pipeline_task_num = 4""" + sql """SET enable_strict_consistency_dml = false""" + qt_pk_writer_scaling_plan """EXPLAIN SHAPE PLAN + INSERT INTO t_pk_writer_scaling + SELECT 1, number, repeat('x', 4096) + FROM numbers("number" = "10000")""" + sql """INSERT INTO t_pk_writer_scaling + SELECT 1, number, repeat('x', 4096) + FROM numbers("number" = "10000") + ORDER BY number""" + qt_pk_writer_scaling """SELECT COUNT(*), MIN(id), MAX(id), + MIN(LENGTH(payload)), MAX(LENGTH(payload)) FROM t_pk_writer_scaling""" + assertTableEquals("t_pk_writer_scaling", "ORDER BY id") + sql """SET parallel_pipeline_task_num = 0""" + sql """SET enable_strict_consistency_dml = true""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_schema_change.groovy b/regression-test/suites/paimon_write/test_paimon_write_schema_change.groovy new file mode 100644 index 00000000000000..97951822fcdb52 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_schema_change.groovy @@ -0,0 +1,1097 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_schema_change", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String catalogName = "test_pw_schema_change_catalog" + String dbName = "test_pw_schema_change_db" + String appendTable = "t_schema_change_append" + String typeTable = "t_schema_change_types" + String explicitTypeTable = "t_schema_change_explicit_types" + String primaryKeyTable = "t_schema_change_pk" + + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + sql """ + CREATE CATALOG `${catalogName}` PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ) + """ + sql """SWITCH `${catalogName}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """CREATE DATABASE `${dbName}`""" + sql """USE `${dbName}`""" + sql """SET show_column_comment_in_describe = true""" + + try { + def assertTableEquals = { String table, String columns, String orderBy -> + spark_paimon """ + REFRESH TABLE paimon.${dbName}.${table} + """ + def sparkRows = spark_paimon """ + SELECT ${columns} + FROM paimon.${dbName}.${table} + ${orderBy} + """ + def dorisRows = sql """ + SELECT ${columns} + FROM `${table}` + ${orderBy} + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // ------------------------------------------------------------------ + // Append-only partitioned table: every supported column evolution is + // followed by reading historical rows and writing with the new schema. + // ------------------------------------------------------------------ + sql """ + CREATE TABLE `${appendTable}` ( + id INT NULL, + required_value BIGINT NOT NULL, + name STRING NULL, + score INT NULL, + amount DECIMAL(8, 2) NULL, + obsolete STRING NULL, + dt STRING NULL + ) ENGINE=paimon + PARTITION BY (dt) () + PROPERTIES ( + 'disable-explicit-type-casting' = 'true' + ) + """ + + sql """ + INSERT INTO `${appendTable}` VALUES + (1, 100, 'alice', 10, 1.10, 'old-a', '2026-07-01'), + (2, 200, 'bob', 20, 2.20, 'old-b', '2026-07-02') + """ + order_qt_sc_append_initial """ + SELECT id, required_value, name, score, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, required_value, name, score, amount, obsolete, dt", + "ORDER BY id") + + // ADD COLUMN with DEFAULT, COMMENT and AFTER. Historical rows remain + // readable and explicit values can immediately be written. + sql """ + ALTER TABLE `${appendTable}` + ADD COLUMN added_after STRING NULL DEFAULT 'unknown' + COMMENT 'added after score' AFTER score + """ + order_qt_sc_add_after_before_insert """ + SELECT id, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (added_after, dt, id, obsolete, amount, name, required_value, score) + VALUES + ('added-3', '2026-07-03', 3, 'old-c', 3.30, 'carol', 300, 30), + (NULL, '2026-07-01', 4, 'old-d', 4.40, 'dave', 400, 40) + """ + order_qt_sc_add_after_after_insert """ + SELECT id, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, required_value, name, score, amount, obsolete, dt) + VALUES (100, 10000, 'default-value', 100, 100.00, 'old-default', '2026-07-10') + """ + order_qt_sc_add_default_omitted """ + SELECT id, added_after FROM `${appendTable}` WHERE id = 100 + """ + + // ADD COLUMN FIRST. All historical rows expose NULL for the new column. + sql """ALTER TABLE `${appendTable}` ADD COLUMN first_col BIGINT NULL FIRST""" + order_qt_sc_add_first_before_insert """ + SELECT id, first_col, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, first_col, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (first_col, id, required_value, name, score, added_after, amount, obsolete, dt) + VALUES + (5000, 5, 500, 'erin', 50, 'added-first', 5.50, 'old-e', '2026-07-04') + """ + order_qt_sc_add_first_after_insert """ + SELECT id, first_col, required_value, name, score, added_after, amount, obsolete, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, first_col, required_value, name, score, added_after, amount, obsolete, dt", + "ORDER BY id") + + // ADD COLUMNS validates that a batch of columns is visible atomically + // to both the reader and writer. + sql """ + ALTER TABLE `${appendTable}` ADD COLUMN ( + tiny_col TINYINT NULL, + small_col SMALLINT NULL COMMENT 'small integer' + ) + """ + order_qt_sc_add_columns_before_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (small_col, tiny_col, dt, id, first_col, required_value, + name, score, added_after, amount, obsolete) + VALUES + (600, 6, '2026-07-05', 6, 6000, 600, + 'frank', 60, 'added-columns', 6.60, 'old-f') + """ + // Omit the newly added nullable columns while keeping added_after + // explicit because its omitted-default behavior is tracked above. + sql """ + INSERT INTO `${appendTable}` + (id, required_value, name, score, added_after, amount, obsolete, dt) + VALUES + (60, 6000, 'partial-columns', 600, 'explicit-default-column', + 60.60, 'old-partial', '2026-07-05') + """ + order_qt_sc_add_columns_after_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, obsolete, tiny_col, small_col, dt + """, + "ORDER BY id") + + // DROP a populated non-key column. + sql """ALTER TABLE `${appendTable}` DROP COLUMN obsolete""" + order_qt_sc_drop_before_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (tiny_col, small_col, dt, id, first_col, required_value, + name, score, added_after, amount) + VALUES + (7, 700, '2026-07-06', 7, 7000, 700, + 'grace', 70, 'after-drop', 7.70) + """ + order_qt_sc_drop_after_insert """ + SELECT id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + // RENAME resolves the old name case-insensitively while preserving the + // Paimon field id and all historical values. + sql """ALTER TABLE `${appendTable}` RENAME COLUMN NAME full_name""" + order_qt_sc_rename_before_insert """ + SELECT id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (full_name, id, score, added_after, dt, amount, + required_value, first_col, tiny_col, small_col) + VALUES + ('heidi', 8, 80, 'after-rename', '2026-07-02', 8.80, + 800, 8000, 8, 800) + """ + order_qt_sc_rename_after_insert """ + SELECT id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, first_col, required_value, full_name, score, added_after, + amount, tiny_col, small_col, dt + """, + "ORDER BY id") + + // MODIFY type: INT -> BIGINT. The new value exceeds the INT range. + sql """ALTER TABLE `${appendTable}` MODIFY COLUMN score BIGINT NULL""" + order_qt_sc_modify_bigint_before_insert """ + SELECT id, full_name, score, amount, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, required_value, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (9, 9000, 900, 'ivan', CAST(3000000000 AS BIGINT), + 'after-bigint', 9.90, 9, 900, '2026-07-07') + """ + order_qt_sc_modify_bigint_after_insert """ + SELECT id, full_name, score, amount, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, required_value, dt", + "ORDER BY id") + + // MODIFY type: widen DECIMAL precision without changing the scale. + sql """ALTER TABLE `${appendTable}` MODIFY COLUMN amount DECIMAL(12, 2) NULL""" + order_qt_sc_modify_decimal_before_insert """ + SELECT id, full_name, score, amount, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (10, 10000, 1000, 'judy', 100, + 'after-decimal', 1234567890.12, 10, 1000, '2026-07-08') + """ + order_qt_sc_modify_decimal_after_insert """ + SELECT id, full_name, score, amount, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, amount, dt", + "ORDER BY id") + + // MODIFY nullability: NOT NULL -> NULL, then actually write NULL. + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN required_value BIGINT NULL + """ + order_qt_sc_modify_nullable_before_insert """ + SELECT id, full_name, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, required_value, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (11, 11000, NULL, 'kate', 110, + 'after-nullable', 11.11, 11, 1100, '2026-07-09') + """ + order_qt_sc_modify_nullable_after_insert """ + SELECT id, full_name, required_value, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, required_value, dt", + "ORDER BY id") + + // MODIFY DEFAULT, COMMENT and position together. DESC checks metadata; + // data checks field-id preservation after moving the column. + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN added_after STRING NULL DEFAULT 'changed-default' + COMMENT 'changed comment' FIRST + """ + qt_sc_modify_metadata_desc """DESC `${appendTable}`""" + order_qt_sc_modify_metadata_before_insert """ + SELECT id, full_name, added_after, score, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, added_after, score, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (added_after, id, first_col, required_value, full_name, + score, amount, tiny_col, small_col, dt) + VALUES + ('after-metadata', 12, 12000, 1200, 'leo', + 120, 12.12, 12, 1200, '2026-07-10') + """ + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, + score, amount, tiny_col, small_col, dt) + VALUES + (120, 120000, 12000, 'modified-default', + 1200, 120.00, 12, 1200, '2026-07-10') + """ + order_qt_sc_modify_default_omitted """ + SELECT id, added_after FROM `${appendTable}` WHERE id = 120 + """ + order_qt_sc_modify_metadata_after_insert """ + SELECT id, full_name, added_after, score, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, added_after, score, dt", + "ORDER BY id") + + // Omitting DEFAULT and COMMENT removes both, while AFTER moves the same + // field again. Explicit writes must continue to map to the correct id. + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN added_after STRING NULL AFTER score + """ + qt_sc_modify_remove_metadata_desc """DESC `${appendTable}`""" + order_qt_sc_modify_remove_metadata_before_insert """ + SELECT id, full_name, score, added_after, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, added_after, dt", + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + added_after, amount, tiny_col, small_col, dt) + VALUES + (13, 13000, 1300, 'mallory', 130, + 'after-remove-metadata', 13.13, 13, 1300, '2026-07-11') + """ + sql """ + INSERT INTO `${appendTable}` + (id, first_col, required_value, full_name, score, + amount, tiny_col, small_col, dt) + VALUES + (130, 130000, 13000, 'removed-default', 1300, + 130.00, 13, 1300, '2026-07-11') + """ + order_qt_sc_remove_default_omitted """ + SELECT id, added_after FROM `${appendTable}` WHERE id = 130 + """ + order_qt_sc_modify_remove_metadata_after_insert """ + SELECT id, full_name, score, added_after, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + "id, full_name, score, added_after, dt", + "ORDER BY id") + + // Reorder every column, then use INSERT VALUES without a target list. + // This catches stale FE and JNI writer physical-column ordering. + sql """ + ALTER TABLE `${appendTable}` ORDER BY ( + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + ) + """ + order_qt_sc_reorder_before_insert """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + sql """ + INSERT INTO `${appendTable}` VALUES + (14, 'nick', 140, 14.14, 1400, + 'after-reorder', 14000, 14, 1400, '2026-07-12') + """ + order_qt_sc_reorder_after_insert """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + // Failed schema changes must be atomic and must not poison a subsequent + // writer created from the still-current schema. + test { + sql """ + ALTER TABLE `${appendTable}` ADD COLUMN ( + batch_ok INT NULL, + batch_bad INT NOT NULL DEFAULT '1' + ) + """ + exception "cannot specify NOT NULL" + } + test { + sql """ALTER TABLE `${appendTable}` ADD COLUMN ID INT NULL""" + exception "conflicts with an existing Paimon column" + } + test { + sql """ + ALTER TABLE `${appendTable}` + ADD COLUMN bad_position INT NULL AFTER missing_col + """ + exception "does not exist in Paimon table" + } + test { + sql """ + ALTER TABLE `${appendTable}` + ADD COLUMN multi_a INT NULL, + ADD COLUMN multi_b INT NULL + """ + exception "External table does not support multiple ALTER clauses" + } + test { + sql """ALTER TABLE `${appendTable}` DROP COLUMN missing_col""" + exception "does not exist in Paimon table" + } + test { + sql """ + ALTER TABLE `${appendTable}` + RENAME COLUMN full_name id + """ + exception "conflicts with an existing Paimon column" + } + test { + sql """ALTER TABLE `${appendTable}` MODIFY COLUMN score INT NULL""" + exception "cannot be converted" + } + test { + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN required_value BIGINT NOT NULL + """ + exception "nullable to non nullable" + } + test { + sql """ALTER TABLE `${appendTable}` DROP COLUMN dt""" + exception "Cannot drop partition key or primary key" + } + test { + sql """ + ALTER TABLE `${appendTable}` + RENAME COLUMN dt partition_col + """ + exception "Cannot rename partition column" + } + test { + sql """ + ALTER TABLE `${appendTable}` + MODIFY COLUMN dt INT NULL + """ + exception "Cannot update partition column" + } + test { + sql """ + ALTER TABLE `${appendTable}` ORDER BY ( + id, full_name, score + ) + """ + exception "must contain every Paimon column exactly once" + } + test { + sql """ + ALTER TABLE `${appendTable}` ORDER BY ( + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, id + ) + """ + exception "Duplicate column in reorder columns" + } + + sql """ + INSERT INTO `${appendTable}` VALUES + (15, 'olivia', 150, 15.15, NULL, + 'after-failed-alters', 15000, 15, 1500, '2026-07-13') + """ + order_qt_sc_after_failed_alters """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + // Paimon 1.3.1 has no partition-key evolution in SchemaChange. + // Doris therefore rejects ADD, DROP and REPLACE before catalog mutation. + test { + sql """ + ALTER TABLE `${appendTable}` + ADD PARTITION KEY bucket(4, id) AS id_bucket + """ + exception "ADD PARTITION KEY is only supported for Iceberg tables" + } + test { + sql """ALTER TABLE `${appendTable}` DROP PARTITION KEY dt""" + exception "DROP PARTITION KEY is only supported for Iceberg tables" + } + test { + sql """ + ALTER TABLE `${appendTable}` + REPLACE PARTITION KEY dt WITH bucket(4, id) AS id_bucket + """ + exception "REPLACE PARTITION KEY is only supported for Iceberg tables" + } + + // Rejected partition evolution must not mutate the current schema or + // prevent a new writer from committing another physical partition. + sql """ + INSERT INTO `${appendTable}` VALUES + (16, 'peggy', 160, 16.16, 1600, + 'after-partition-evolution-failures', 16000, 16, 1600, '2026-07-14') + """ + order_qt_sc_after_partition_evolution_failures """ + SELECT id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + FROM `${appendTable}` + ORDER BY id + """ + assertTableEquals( + appendTable, + """ + id, full_name, score, amount, required_value, + added_after, first_col, tiny_col, small_col, dt + """, + "ORDER BY id") + + // Regular writes create new partition values before and after schema + // evolution. This is dynamic partition creation, not partition evolution. + def sparkPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`${appendTable}\$partitions` + ORDER BY `partition` + """ + def dorisPartitions = sql """ + SELECT `partition`, record_count + FROM `${appendTable}\$partitions` + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + order_qt_sc_append_partitions """ + SELECT `partition`, record_count + FROM `${appendTable}\$partitions` + ORDER BY `partition` + """ + + // ------------------------------------------------------------------ + // Type-evolution matrix: verify widening conversions on existing data, + // then write values which cannot fit in the original types. + // ------------------------------------------------------------------ + sql """ + CREATE TABLE `${typeTable}` ( + id INT NULL, + c_tiny TINYINT NULL, + c_small SMALLINT NULL, + c_int INT NULL, + c_float FLOAT NULL, + c_decimal DECIMAL(8, 2) NULL + ) ENGINE=paimon + PROPERTIES ( + 'disable-explicit-type-casting' = 'true' + ) + """ + sql """ + INSERT INTO `${typeTable}` VALUES + (1, 100, 30000, 2000000000, 1.5, 123456.78) + """ + order_qt_sc_types_initial """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_tiny SMALLINT NULL""" + order_qt_sc_types_tiny_to_small_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (2, 200, 30001, 2000000001, 2.5, 123456.79) + """ + order_qt_sc_types_tiny_to_small_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_small INT NULL""" + order_qt_sc_types_small_to_int_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (3, 201, 40000, 2000000002, 3.5, 123456.80) + """ + order_qt_sc_types_small_to_int_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_int BIGINT NULL""" + order_qt_sc_types_int_to_bigint_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (4, 202, 40001, 3000000000, 4.5, 123456.81) + """ + order_qt_sc_types_int_to_bigint_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_float DOUBLE NULL""" + order_qt_sc_types_float_to_double_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (5, 203, 40002, 3000000001, 1.0E40, 123456.82) + """ + order_qt_sc_types_float_to_double_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + ALTER TABLE `${typeTable}` + MODIFY COLUMN c_decimal DECIMAL(12, 2) NULL + """ + order_qt_sc_types_decimal_widen_before_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${typeTable}` VALUES + (6, 204, 40003, 3000000002, 2.0E40, 1234567890.12) + """ + order_qt_sc_types_decimal_widen_after_insert """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + // A narrowing conversion fails with data present; the old writer schema + // remains usable after the failed ALTER. + test { + sql """ALTER TABLE `${typeTable}` MODIFY COLUMN c_int INT NULL""" + exception "cannot be converted" + } + sql """ + INSERT INTO `${typeTable}` VALUES + (7, 205, 40004, 3000000003, 3.0E40, 1234567890.13) + """ + order_qt_sc_types_after_failed_narrow """ + SELECT * FROM `${typeTable}` ORDER BY id + """ + assertTableEquals(typeTable, "*", "ORDER BY id") + + // By default Paimon also permits explicit casts. Use values which fit + // in the target type to cover BIGINT -> INT and then INT -> STRING. + sql """ + CREATE TABLE `${explicitTypeTable}` ( + id INT NULL, + explicit_value BIGINT NULL + ) ENGINE=paimon + """ + sql """ + INSERT INTO `${explicitTypeTable}` VALUES + (1, 100), + (2, 200) + """ + order_qt_sc_explicit_types_initial """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + ALTER TABLE `${explicitTypeTable}` + MODIFY COLUMN explicit_value INT NULL + """ + order_qt_sc_explicit_bigint_to_int_before_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${explicitTypeTable}` VALUES + (3, 300) + """ + order_qt_sc_explicit_bigint_to_int_after_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + ALTER TABLE `${explicitTypeTable}` + MODIFY COLUMN explicit_value STRING NULL + """ + order_qt_sc_explicit_int_to_string_before_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + sql """ + INSERT INTO `${explicitTypeTable}` VALUES + (4, 'after-explicit-cast') + """ + order_qt_sc_explicit_int_to_string_after_insert """ + SELECT * FROM `${explicitTypeTable}` ORDER BY id + """ + assertTableEquals(explicitTypeTable, "*", "ORDER BY id") + + // ------------------------------------------------------------------ + // Primary-key table: repeat the core evolutions around merge-tree data + // and verify key/partition-column restrictions do not affect later writes. + // ------------------------------------------------------------------ + sql """ + CREATE TABLE `${primaryKeyTable}` ( + id INT NOT NULL, + dt STRING NOT NULL, + metric INT NULL, + legacy STRING NULL + ) ENGINE=paimon + PARTITION BY (dt) () + PROPERTIES ( + 'primary-key' = 'id,dt', + 'bucket' = '2', + 'merge-engine' = 'partial-update' + ) + """ + sql """ + INSERT INTO `${primaryKeyTable}` VALUES + (1, '2026-08-01', 10, 'pk-a'), + (2, '2026-08-01', 20, 'pk-b') + """ + order_qt_sc_pk_initial """ + SELECT * FROM `${primaryKeyTable}` ORDER BY dt, id + """ + assertTableEquals(primaryKeyTable, "*", "ORDER BY dt, id") + + sql """ + ALTER TABLE `${primaryKeyTable}` + ADD COLUMN note STRING NULL DEFAULT 'default-note' AFTER metric + """ + order_qt_sc_pk_add_before_insert """ + SELECT id, dt, metric, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric, note, legacy", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` VALUES + (1, '2026-08-01', 11, 'updated-after-add', 'pk-a2'), + (3, '2026-08-02', 30, 'new-after-add', 'pk-c') + """ + order_qt_sc_pk_add_after_insert """ + SELECT id, dt, metric, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric, note, legacy", + "ORDER BY dt, id") + + sql """ + ALTER TABLE `${primaryKeyTable}` + RENAME COLUMN metric metric_value + """ + order_qt_sc_pk_rename_before_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value, note, legacy) + VALUES + (2, '2026-08-01', 22, 'updated-after-rename', 'pk-b2'), + (4, '2026-08-02', 40, 'new-after-rename', 'pk-d') + """ + order_qt_sc_pk_rename_after_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ + ALTER TABLE `${primaryKeyTable}` + MODIFY COLUMN metric_value BIGINT NULL + """ + order_qt_sc_pk_type_before_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value, note, legacy) + VALUES + (5, '2026-08-03', 3000000000, 'new-after-type', 'pk-e') + """ + order_qt_sc_pk_type_after_insert """ + SELECT id, dt, metric_value, note, legacy + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note, legacy", + "ORDER BY dt, id") + + sql """ALTER TABLE `${primaryKeyTable}` DROP COLUMN legacy""" + order_qt_sc_pk_drop_before_insert """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + + // A PK partial-update writer also distinguishes omitted fields from + // explicit NULL using the evolved remote schema. A later partial row + // which omits note applies its schema default again. + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value) + VALUES + (8, '2026-08-05', 80) + """ + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, note) + VALUES + (8, '2026-08-05', 'explicit-note'), + (9, '2026-08-05', NULL) + """ + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value) + VALUES + (8, '2026-08-05', 81) + """ + order_qt_sc_pk_partial_default """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + WHERE id IN (8, 9) + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + + sql """ + INSERT INTO `${primaryKeyTable}` + (note, metric_value, dt, id) + VALUES + ('new-after-drop', 60, '2026-08-03', 6) + """ + order_qt_sc_pk_drop_after_insert """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + + // Primary and partition keys cannot be dropped, renamed or have their + // types changed. All failures must leave the merge-tree writer usable. + test { + sql """ALTER TABLE `${primaryKeyTable}` DROP COLUMN id""" + exception "Cannot drop partition key or primary key" + } + test { + sql """ + ALTER TABLE `${primaryKeyTable}` + RENAME COLUMN dt partition_col + """ + exception "Cannot rename partition column" + } + test { + sql """ + ALTER TABLE `${primaryKeyTable}` + MODIFY COLUMN id BIGINT NOT NULL + """ + exception "Cannot update primary key" + } + + sql """ + INSERT INTO `${primaryKeyTable}` + (id, dt, metric_value, note) + VALUES + (7, '2026-08-04', 70, 'after-key-failures') + """ + order_qt_sc_pk_after_key_failures """ + SELECT id, dt, metric_value, note + FROM `${primaryKeyTable}` + ORDER BY dt, id + """ + assertTableEquals( + primaryKeyTable, + "id, dt, metric_value, note", + "ORDER BY dt, id") + } finally { + sql """DROP TABLE IF EXISTS `${primaryKeyTable}`""" + sql """DROP TABLE IF EXISTS `${explicitTypeTable}`""" + sql """DROP TABLE IF EXISTS `${typeTable}`""" + sql """DROP TABLE IF EXISTS `${appendTable}`""" + sql """DROP DATABASE IF EXISTS `${dbName}` FORCE""" + sql """SWITCH internal""" + sql """DROP CATALOG IF EXISTS `${catalogName}`""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_transaction.groovy b/regression-test/suites/paimon_write/test_paimon_write_transaction.groovy new file mode 100644 index 00000000000000..b4ce0a942f3ae5 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_transaction.groovy @@ -0,0 +1,466 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_transaction", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_txn_catalog" + String dbName = "test_pw_txn_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_commit; + CREATE TABLE paimon.${dbName}.t_commit ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_commit_batch; + CREATE TABLE paimon.${dbName}.t_commit_batch ( + id INT, val DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite; + CREATE TABLE paimon.${dbName}.t_overwrite ( + id INT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite_part; + CREATE TABLE paimon.${dbName}.t_overwrite_part ( + id INT, name STRING, region STRING + ) USING paimon + PARTITIONED BY (region); + + DROP TABLE IF EXISTS paimon.${dbName}.t_overwrite_part_case; + CREATE TABLE paimon.${dbName}.t_overwrite_part_case ( + id INT, name STRING, Region STRING + ) USING paimon + PARTITIONED BY (Region); + + DROP TABLE IF EXISTS paimon.${dbName}.t_static_multi; + CREATE TABLE paimon.${dbName}.t_static_multi ( + id INT, name STRING, pt0 INT, pt1 STRING + ) USING paimon + PARTITIONED BY (pt0, pt1) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_static_default; + CREATE TABLE paimon.${dbName}.t_static_default ( + id INT, name STRING, region STRING + ) USING paimon + PARTITIONED BY (region) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true', + 'partition.default-name' = '__CUSTOM_DEFAULT_PARTITION__' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_static_boundary; + CREATE TABLE paimon.${dbName}.t_static_boundary ( + id INT, name STRING, region STRING, dt DATE + ) USING paimon + PARTITIONED BY (region, dt) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true', + 'partition.default-name' = '__CUSTOM_DEFAULT_PARTITION__' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_dynamic_multi; + CREATE TABLE paimon.${dbName}.t_dynamic_multi ( + id INT, name STRING, region STRING + ) USING paimon + PARTITIONED BY (region) + TBLPROPERTIES ( + 'dynamic-partition-overwrite' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_multi; + CREATE TABLE paimon.${dbName}.t_multi ( + id INT, name STRING, score DOUBLE + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_parallel; + CREATE TABLE paimon.${dbName}.t_parallel ( + id BIGINT, name STRING + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_multi_block; + CREATE TABLE paimon.${dbName}.t_multi_block ( + id BIGINT, + group_id INT, + payload STRING, + nullable_value BIGINT, + pt STRING + ) USING paimon + PARTITIONED BY (pt); + + DROP TABLE IF EXISTS paimon.${dbName}.t_spill; + CREATE TABLE paimon.${dbName}.t_spill ( + id BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-buffer-size' = '256 kb', + 'page-size' = '64 kb', + 'write-buffer-spillable' = 'true' + ); + + DROP TABLE IF EXISTS paimon.${dbName}.t_failed_write; + CREATE TABLE paimon.${dbName}.t_failed_write ( + id BIGINT, payload STRING + ) USING paimon + TBLPROPERTIES ( + 'primary-key' = 'id', + 'bucket' = '1', + 'write-buffer-size' = '256 kb', + 'page-size' = '64 kb', + 'write-buffer-spillable' = 'true' + ); + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + try { + def assertTableEquals = { String tableName, String orderBy -> + def sparkRows = spark_paimon """SELECT * FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT * FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-010: Basic commit — INSERT INTO, commit, read back + sql """INSERT INTO t_commit VALUES (1, 'alice'), (2, 'bob')""" + assertTableEquals("t_commit", "ORDER BY id") + + sql """INSERT INTO t_commit VALUES (3, 'charlie'), (4, 'diana')""" + order_qt_txn_commit """SELECT id, name FROM t_commit ORDER BY id""" + assertTableEquals("t_commit", "ORDER BY id") + + // FT-011: Batch INSERT — 10 rows, then self-copy via SELECT + sql """INSERT INTO t_commit_batch VALUES + (1, 1.0), (2, 2.0), (3, 3.0), (4, 4.0), (5, 5.0), + (6, 6.0), (7, 7.0), (8, 8.0), (9, 9.0), (10, 10.0)""" + sql """INSERT INTO t_commit_batch SELECT id + 10, val + 10.0 FROM t_commit_batch""" + order_qt_txn_batch """SELECT id, val FROM t_commit_batch ORDER BY id""" + assertTableEquals("t_commit_batch", "ORDER BY id") + + // FT-012: Full-table overwrite must remove every row from the previous snapshot. + sql """INSERT INTO t_overwrite VALUES (1, 'old1'), (2, 'old2'), (3, 'old3')""" + assertTableEquals("t_overwrite", "ORDER BY id") + + sql """INSERT OVERWRITE TABLE t_overwrite VALUES (10, 'new1'), (20, 'new2')""" + order_qt_txn_overwrite """SELECT id, name FROM t_overwrite ORDER BY id""" + assertTableEquals("t_overwrite", "ORDER BY id") + + // FT-013: An overwrite with an empty input still commits an empty snapshot. + sql """INSERT OVERWRITE TABLE t_overwrite SELECT 1, 'unused' WHERE 1 = 0""" + qt_txn_empty_overwrite """SELECT COUNT(*) FROM t_overwrite""" + assertTableEquals("t_overwrite", "ORDER BY id") + + // A direct PhysicalEmptyRelation must still publish the empty overwrite snapshot. + sql """INSERT INTO t_overwrite VALUES (30, 'old_for_limit_zero')""" + sql """INSERT OVERWRITE TABLE t_overwrite SELECT 1, 'unused' LIMIT 0""" + qt_txn_empty_overwrite_limit_zero """SELECT COUNT(*) FROM t_overwrite""" + assertTableEquals("t_overwrite", "ORDER BY id") + + // FT-015: Static partition overwrite replaces only the requested partition. + sql """INSERT INTO t_overwrite_part VALUES + (1, 'east_old', 'east'), (2, 'west_old', 'west')""" + sql """INSERT OVERWRITE TABLE t_overwrite_part + PARTITION (region = 'east') VALUES (10, 'east_new')""" + order_qt_txn_static_partition """SELECT id, name, region FROM t_overwrite_part ORDER BY id""" + assertTableEquals("t_overwrite_part", "ORDER BY id") + + // Static partition names are case-insensitive in Doris and canonicalized + // to the exact Paimon schema field name before commit. + sql """INSERT INTO t_overwrite_part_case VALUES + (1, 'east_old', 'east'), (2, 'west_old', 'west')""" + sql """INSERT OVERWRITE TABLE t_overwrite_part_case + PARTITION (region = 'east') VALUES (10, 'east_new')""" + order_qt_txn_static_partition_case """SELECT id, name, Region + FROM t_overwrite_part_case ORDER BY id""" + assertTableEquals("t_overwrite_part_case", "ORDER BY id") + + test { + sql """INSERT OVERWRITE TABLE t_overwrite_part + PARTITION (region = 'east', REGION = 'west') VALUES (20, 'ambiguous')""" + exception "Duplicate partition column: REGION" + } + + // A static partial spec uses static overwrite semantics even when the + // table default is dynamic partition overwrite. It replaces every + // matching subpartition, including those absent from the new input. + sql """INSERT INTO t_static_multi VALUES + (1, 'old_a', 1, 'A'), (2, 'old_b', 1, 'B'), (3, 'keep', 2, 'C')""" + sql """INSERT OVERWRITE TABLE t_static_multi + PARTITION (pt0 = 1) VALUES (10, 'new_a', 'A')""" + order_qt_txn_static_partial """SELECT id, name, pt0, pt1 + FROM t_static_multi ORDER BY id""" + assertTableEquals("t_static_multi", "ORDER BY id") + + // Empty static overwrite still removes the complete matching spec. + sql """INSERT OVERWRITE TABLE t_static_multi + PARTITION (pt0 = 1) SELECT 1, 'unused', 'A' LIMIT 0""" + order_qt_txn_static_partial_empty """SELECT id, name, pt0, pt1 + FROM t_static_multi ORDER BY id""" + assertTableEquals("t_static_multi", "ORDER BY id") + + // NULL must use the table's actual configurable default partition + // name, while the literal string "null" and blank strings remain + // distinct typed partition values. + sql """INSERT INTO t_static_default VALUES + (1, 'null_old', NULL), + (2, 'literal_null', 'null'), + (3, 'blank_old', ''), + (4, 'east_old', 'east')""" + sql """INSERT OVERWRITE TABLE t_static_default + PARTITION (region = NULL) VALUES (10, 'null_new')""" + order_qt_txn_static_null """SELECT id, name, + IF(region = '', '', region) AS region + FROM t_static_default ORDER BY id""" + assertTableEquals("t_static_default", "ORDER BY id") + + sql """INSERT OVERWRITE TABLE t_static_default + PARTITION (region = 'east') SELECT 1, 'unused' LIMIT 0""" + order_qt_txn_static_empty """SELECT id, name, + IF(region = '', '', region) AS region + FROM t_static_default ORDER BY id""" + assertTableEquals("t_static_default", "ORDER BY id") + + sql """INSERT OVERWRITE TABLE t_static_default + PARTITION (region = '') VALUES (30, 'blank_new')""" + order_qt_txn_static_blank """SELECT id, name, + IF(region = '', '', region) AS region + FROM t_static_default ORDER BY id""" + assertTableEquals("t_static_default", "ORDER BY id") + + // A partial static specification must use typed partition identity. + // NULL, blank, the literal "null", escaped path characters, and DATE + // subpartitions must remain distinct even when display paths overlap. + sql """INSERT INTO t_static_boundary VALUES + (1, 'null_d1', NULL, '2026-07-01'), + (2, 'null_d2', NULL, '2026-07-02'), + (3, 'blank', '', '2026-07-01'), + (4, 'literal_null', 'null', '2026-07-01'), + (5, 'special_d1', 'a/b=c%20', '2026-07-01'), + (6, 'special_d2', 'a/b=c%20', '2026-07-02'), + (7, 'keep', 'keep', '2026-07-01') + """ + sql """INSERT OVERWRITE TABLE t_static_boundary + PARTITION (region = NULL) + VALUES (10, 'null_new', '2026-07-03')""" + def staticNullRows = sql """ + SELECT id, name, IF(region = '', '', region), CAST(dt AS STRING) + FROM t_static_boundary ORDER BY id + """ + assertEquals([ + [3, "blank", "", "2026-07-01"], + [4, "literal_null", "null", "2026-07-01"], + [5, "special_d1", "a/b=c%20", "2026-07-01"], + [6, "special_d2", "a/b=c%20", "2026-07-02"], + [7, "keep", "keep", "2026-07-01"], + [10, "null_new", null, "2026-07-03"] + ], staticNullRows) + + sql """INSERT OVERWRITE TABLE t_static_boundary + PARTITION (region = 'a/b=c%20') + VALUES (50, 'special_new', '2026-07-04')""" + order_qt_txn_static_typed_boundaries """ + SELECT id, name, IF(region = '', '', region) AS region, dt + FROM t_static_boundary ORDER BY id + """ + assertTableEquals("t_static_boundary", "ORDER BY id") + def sparkBoundaryPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_static_boundary\$partitions` + ORDER BY `partition` + """ + def dorisBoundaryPartitions = sql """ + SELECT `partition`, record_count + FROM t_static_boundary\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkBoundaryPartitions, dorisBoundaryPartitions) + + // Dynamic overwrite replaces all partitions present in one input batch, + // preserves untouched partitions, and publishes one overwrite snapshot. + sql """INSERT INTO t_dynamic_multi VALUES + (1, 'p1_old_a', 'p1'), (2, 'p1_old_b', 'p1'), + (3, 'p2_old', 'p2'), (4, 'p3_keep', 'p3'), (5, 'p4_keep', 'p4') + """ + sql """INSERT OVERWRITE TABLE t_dynamic_multi VALUES + (10, 'p1_new', 'p1'), + (20, 'p2_new_a', 'p2'), + (21, 'p2_new_b', 'p2') + """ + order_qt_txn_dynamic_multi """ + SELECT id, name, region FROM t_dynamic_multi ORDER BY id + """ + assertTableEquals("t_dynamic_multi", "ORDER BY id") + assertEquals(2L, + (sql """SELECT COUNT(*) FROM t_dynamic_multi\$snapshots""")[0][0] as long) + def sparkDynamicPartitions = spark_paimon """ + SELECT `partition`, record_count + FROM paimon.${dbName}.`t_dynamic_multi\$partitions` + ORDER BY `partition` + """ + def dorisDynamicPartitions = sql """ + SELECT `partition`, record_count + FROM t_dynamic_multi\$partitions + ORDER BY `partition` + """ + assertSparkDorisResultEquals(sparkDynamicPartitions, dorisDynamicPartitions) + + // FT-016: Dynamic partition overwrite replaces the partitions present in the + // input while preserving existing partitions that are not touched. + sql """INSERT OVERWRITE TABLE t_overwrite_part VALUES (30, 'south_new', 'south')""" + order_qt_txn_dynamic_partition """SELECT id, name, region FROM t_overwrite_part ORDER BY id""" + assertTableEquals("t_overwrite_part", "ORDER BY id") + + test { + sql """INSERT OVERWRITE TABLE t_overwrite_part + PARTITION (region) VALUES (40, 'bare_partition', 'east')""" + exception "Paimon tables do not support PARTITION name lists" + } + test { + sql """INSERT OVERWRITE TABLE t_overwrite_part + TEMPORARY PARTITION (region) VALUES (40, 'temporary_partition', 'east')""" + exception "Paimon tables do not support temporary partitions" + } + order_qt_txn_unsupported_partition_syntax """ + SELECT id, name, region FROM t_overwrite_part ORDER BY id + """ + assertTableEquals("t_overwrite_part", "ORDER BY id") + + // FT-017: Multiple pipeline tasks create multiple LocalState-scoped writers. + // FE must aggregate every writer's commit payload into one Paimon snapshot. + sql """SET parallel_pipeline_task_num = 4""" + sql """INSERT INTO t_parallel + SELECT number, concat('row_', CAST(number AS STRING)) + FROM numbers("number" = "256")""" + qt_txn_parallel_writers """SELECT COUNT(*), MIN(id), MAX(id), SUM(id) FROM t_parallel""" + assertTableEquals("t_parallel", "ORDER BY id") + sql """SET parallel_pipeline_task_num = 0""" + + // A single JNI writer receives many native Blocks, each serialized as an + // independent Arrow stream. Every row must survive repeated write() calls, + // partition routing and prepareCommit(). + sql """SET parallel_pipeline_task_num = 1""" + sql """INSERT INTO t_multi_block + SELECT number, + CAST(number % 97 AS INT), + concat('payload_', CAST(number AS STRING)), + IF(number % 11 = 0, NULL, number * 3), + concat('p', CAST(number % 8 AS STRING)) + FROM numbers("number" = "16384")""" + + // Opening a fresh writer in the next Doris transaction must append to the + // existing snapshot without losing or duplicating the first transaction. + sql """INSERT INTO t_multi_block + SELECT number + 16384, + CAST((number + 16384) % 97 AS INT), + concat('payload_', CAST(number + 16384 AS STRING)), + IF((number + 16384) % 11 = 0, NULL, (number + 16384) * 3), + concat('p', CAST((number + 16384) % 8 AS STRING)) + FROM numbers("number" = "4096")""" + sql """SET parallel_pipeline_task_num = 0""" + + qt_txn_multi_block """ + SELECT COUNT(*), MIN(id), MAX(id), SUM(id), + COUNT(nullable_value), COUNT(DISTINCT pt) + FROM t_multi_block + """ + order_qt_txn_multi_block_samples """ + SELECT id, group_id, payload, nullable_value, pt + FROM t_multi_block + WHERE id IN (0, 4095, 4096, 8191, 8192, 16383, 16384, 20479) + ORDER BY id + """ + qt_txn_multi_block_snapshots """ + SELECT COUNT(*) FROM t_multi_block\$snapshots + """ + def sparkMultiBlock = spark_paimon """ + SELECT COUNT(*), MIN(id), MAX(id), SUM(id), + COUNT(nullable_value), COUNT(DISTINCT pt) + FROM paimon.${dbName}.t_multi_block + """ + def dorisMultiBlock = sql """ + SELECT COUNT(*), MIN(id), MAX(id), SUM(id), + COUNT(nullable_value), COUNT(DISTINCT pt) + FROM t_multi_block + """ + assertSparkDorisResultEquals(sparkMultiBlock, dorisMultiBlock) + + // FT-018: The payload is larger than the 256 KB write buffer while each + // individual row still fits, forcing Paimon's spillable buffer path. + sql """INSERT INTO t_spill + SELECT number, repeat('spill_payload_', 80) + FROM numbers("number" = "2048")""" + qt_txn_spill """SELECT COUNT(*), MIN(id), MAX(id), SUM(id) FROM t_spill""" + assertTableEquals("t_spill", "ORDER BY id") + + // FT-019: A row larger than the complete write buffer fails inside the + // Paimon writer. The failed statement must not publish rows or a snapshot. + sql """INSERT INTO t_failed_write VALUES (1, 'committed_before_failure')""" + qt_txn_failed_write_before """SELECT id, payload FROM t_failed_write ORDER BY id""" + qt_txn_failed_snapshot_before """SELECT COUNT(*) FROM t_failed_write\$snapshots""" + test { + sql """INSERT INTO t_failed_write VALUES + (2, 'accepted_before_error'), + (3, repeat('x', 1048576))""" + exception "The record exceeds the maximum size of a sort buffer" + } + qt_txn_failed_write_after """SELECT id, payload FROM t_failed_write ORDER BY id""" + qt_txn_failed_snapshot_after """SELECT COUNT(*) FROM t_failed_write\$snapshots""" + assertTableEquals("t_failed_write", "ORDER BY id") + + // Multi-row VALUES — verify all 20 rows committed + sql """ + INSERT INTO t_multi VALUES + (1, 'a', 10.0), (2, 'b', 20.0), (3, 'c', 30.0), (4, 'd', 40.0), (5, 'e', 50.0), + (6, 'f', 60.0), (7, 'g', 70.0), (8, 'h', 80.0), (9, 'i', 90.0), (10, 'j', 100.0), + (11, 'k', 110.0),(12, 'l', 120.0),(13, 'm', 130.0),(14, 'n', 140.0),(15, 'o', 150.0), + (16, 'p', 160.0),(17, 'q', 170.0),(18, 'r', 180.0),(19, 's', 190.0),(20, 't', 200.0) + """ + order_qt_txn_multi """SELECT id, name, score FROM t_multi ORDER BY id""" + assertTableEquals("t_multi", "ORDER BY id") + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/paimon_write/test_paimon_write_types.groovy b/regression-test/suites/paimon_write/test_paimon_write_types.groovy new file mode 100644 index 00000000000000..5d9d8f80e42481 --- /dev/null +++ b/regression-test/suites/paimon_write/test_paimon_write_types.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_types", "p0,external,paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + + String catalogName = "test_pw_types_catalog" + String dbName = "test_pw_types_db" + + // Create Paimon tables via Spark + spark_paimon_multi """ + CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types; + CREATE TABLE paimon.${dbName}.t_types ( + c_boolean BOOLEAN, + c_int INT, + c_bigint BIGINT, + c_float FLOAT, + c_double DOUBLE, + c_decimal DECIMAL(10,2), + c_string STRING, + c_varchar VARCHAR(100), + c_date DATE, + c_datetime TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_null; + CREATE TABLE paimon.${dbName}.t_types_null ( + id INT, + c_int INT, + c_string STRING, + c_double DOUBLE, + c_boolean BOOLEAN + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_decimal; + CREATE TABLE paimon.${dbName}.t_types_decimal ( + id INT, + d2 DECIMAL(2,1), + d10 DECIMAL(10,2), + d18 DECIMAL(18,6), + d38 DECIMAL(38,10) + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_dt; + CREATE TABLE paimon.${dbName}.t_types_dt ( + d DATE, dt TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_timezone; + CREATE TABLE paimon.${dbName}.t_types_timezone ( + id INT, event_time TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_ltz_schema; + CREATE TABLE paimon.${dbName}.t_types_ltz_schema ( + event_time TIMESTAMP + ) USING paimon; + + DROP TABLE IF EXISTS paimon.${dbName}.t_types_ntz; + CREATE TABLE paimon.${dbName}.t_types_ntz ( + id INT, event_time TIMESTAMP_NTZ + ) USING paimon; + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + 'type' = 'paimon', + 'paimon.catalog.type' = 'filesystem', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch ${catalogName}""" + sql """use ${dbName}""" + def originalTimeZone = sql """SELECT @@time_zone""" + + try { + def assertTableEquals = { String tableName, String columns, String orderBy -> + def sparkRows = spark_paimon """SELECT ${columns} FROM paimon.${dbName}.${tableName} ${orderBy}""" + def dorisRows = sql """SELECT ${columns} FROM ${tableName} ${orderBy}""" + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + // FT-020~027: Basic types with boundary values + sql """ + INSERT INTO t_types VALUES + (true, 1, 100, CAST(1.5 AS FLOAT), CAST(2.71828 AS DOUBLE), + CAST(99.99 AS DECIMAL(10,2)), 'hello', 'short', + DATE '2024-01-15', TIMESTAMP '2024-01-15 10:30:00'), + (false, 2147483647, 9223372036854775807, + CAST(3.4E38 AS FLOAT), CAST(1.7E308 AS DOUBLE), + CAST(0.00 AS DECIMAL(10,2)), '', '', + DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), + (false, -2147483648, -9223372036854775808, + CAST(-3.4E38 AS FLOAT), CAST(-1.7E308 AS DOUBLE), + CAST(-1.50 AS DECIMAL(10,2)), 'long_string_1234567890', 'max_varchar', + DATE '2099-12-31', TIMESTAMP '2099-12-31 23:59:59'), + (true, 0, 0, CAST(0.0 AS FLOAT), CAST(0.0 AS DOUBLE), + CAST(12345678.90 AS DECIMAL(10,2)), 'hello', 'fixed_len', + DATE '2024-06-15', TIMESTAMP '2024-06-15 12:00:00.123456') + """ + order_qt_types_basic """SELECT * FROM t_types ORDER BY c_int""" + assertTableEquals("t_types", """ + c_boolean, c_int, c_bigint, + c_float / 1.0E38, + c_double / 1.0E308, + c_decimal, c_string, c_varchar, c_date, c_datetime + """, "ORDER BY c_int") + + // FT-040: NULL handling + sql """INSERT INTO t_types_null VALUES (1, 100, 'data', 1.5, true)""" + sql """INSERT INTO t_types_null VALUES (2, NULL, NULL, NULL, NULL)""" + sql """INSERT INTO t_types_null VALUES (3, NULL, 'partial', 2.0, false)""" + order_qt_types_null """SELECT id, c_int, c_string, c_double, c_boolean FROM t_types_null ORDER BY id""" + assertTableEquals("t_types_null", "*", "ORDER BY id") + + // FT-043: Decimal precision + sql """ + INSERT INTO t_types_decimal VALUES + (1, CAST(1.5 AS DECIMAL(2,1)), CAST(12345678.90 AS DECIMAL(10,2)), + CAST(123456789012.123456 AS DECIMAL(18,6)), + CAST(1234567890123456789012345678.1234567890 AS DECIMAL(38,10))), + (2, CAST(-1.5 AS DECIMAL(2,1)), CAST(-0.01 AS DECIMAL(10,2)), + CAST(-1.000001 AS DECIMAL(18,6)), + CAST(0.0000000001 AS DECIMAL(38,10))), + (3, CAST(0.0 AS DECIMAL(2,1)), CAST(0.00 AS DECIMAL(10,2)), + CAST(0.000000 AS DECIMAL(18,6)), + CAST(0.0000000000 AS DECIMAL(38,10))) + """ + order_qt_types_decimal """SELECT id, d2, d10, d18, d38 FROM t_types_decimal ORDER BY id""" + assertTableEquals("t_types_decimal", "*", "ORDER BY id") + + // DATE / DATETIME boundary + sql """ + INSERT INTO t_types_dt VALUES + (DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00'), + (DATE '2024-06-15', TIMESTAMP '2024-06-15 12:00:00'), + (DATE '2099-12-31', TIMESTAMP '2099-12-31 23:59:59.999999') + """ + order_qt_types_dt """SELECT d, dt FROM t_types_dt ORDER BY d""" + assertTableEquals("t_types_dt", "*", "ORDER BY d") + + // FT-027: Spark TIMESTAMP maps to Paimon's local-zoned timestamp. Values + // written in different Doris session timezones must represent the same instant + // in UTC while preserving their independent microsecond fractions. + qt_desc_types_timezone """DESC t_types_ltz_schema""" + sql """SET time_zone = 'Asia/Shanghai'""" + sql """INSERT INTO t_types_timezone VALUES + (1, TIMESTAMP '2024-01-15 10:30:00.123456')""" + sql """SET time_zone = 'UTC'""" + sql """INSERT INTO t_types_timezone VALUES + (2, TIMESTAMP '2024-01-15 02:30:00.654321')""" + order_qt_types_timezone_utc """SELECT id, event_time FROM t_types_timezone ORDER BY id""" + + // Reading the same snapshot in Asia/Shanghai must apply the session offset + // to both rows without losing their microsecond fractions. + sql """SET time_zone = 'Asia/Shanghai'""" + order_qt_types_timezone_shanghai """SELECT id, event_time FROM t_types_timezone ORDER BY id""" + + // Paimon TIMESTAMP_NTZ stores civil fields. A value in a DST gap and a + // Doris-supported short timezone alias must therefore survive without + // an instant conversion or Java ZoneId parsing. + sql """SET time_zone = 'America/Los_Angeles'""" + sql """INSERT INTO t_types_ntz VALUES + (1, TIMESTAMP '2024-03-10 02:30:00.123456')""" + sql """SET time_zone = 'CST'""" + sql """INSERT INTO t_types_ntz VALUES + (2, TIMESTAMP '2024-01-15 10:30:00.654321')""" + sql """SET time_zone = 'UTC'""" + order_qt_types_ntz """SELECT id, event_time FROM t_types_ntz ORDER BY id""" + assertTableEquals("t_types_ntz", "*", "ORDER BY id") + } finally { + sql """SET time_zone = '${originalTimeZone[0][0]}'""" + sql """drop catalog if exists ${catalogName}""" + } +}