diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index a4b117889862de..9b029b83c4b46c 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -104,7 +104,6 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, static constexpr int64_t MICROS_PER_MINUTE = MICROS_PER_SECOND * 60; static constexpr int64_t MICROS_PER_HOUR = MICROS_PER_MINUTE * 60; static constexpr int64_t MICROS_PER_DAY = MICROS_PER_HOUR * 24; - static const int64_t EPOCH_DAYNR = calc_daynr(1970, 1, 1); int64_t days_since_epoch = timestamp_micros / MICROS_PER_DAY; int64_t micros_of_day = timestamp_micros % MICROS_PER_DAY; @@ -113,11 +112,16 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, --days_since_epoch; } - const int64_t daynr = EPOCH_DAYNR + days_since_epoch; - if (daynr <= 0) { + // A local Parquet timestamp carries a civil value whose day ordinal is defined in the + // proleptic Gregorian calendar, while Doris numbers days in MySQL's calendar, where year 0 is + // not a leap year. Adding `calc_daynr(1970, 1, 1)` directly conflates the two and shifts + // 0000-01-01 .. 0000-02-28 one day early. `epoch_days_to_daynr()` bridges them and returns 0 + // for everything Doris cannot represent, including the proleptic-only 0000-02-29. + const int64_t daynr = epoch_days_to_daynr(days_since_epoch); + if (daynr == 0) { return Status::DataQualityError( - "Decoded DATETIMEV2 timestamp is out of range: micros={}, daynr={}", - timestamp_micros, daynr); + "Decoded DATETIMEV2 timestamp is out of range: micros={}, epoch_days={}", + timestamp_micros, days_since_epoch); } DateV2Value datetime_value; @@ -154,6 +158,11 @@ Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data DateV2Value datetime_value; datetime_value.from_unixtime(epoch_seconds, timezone); datetime_value.set_microsecond(static_cast(micros_of_second)); + // The civil range has to be checked here, after the offset is applied, not on the raw instant: + // a local 0001-01-01 00:00:00 east of UTC is an instant below the civil minimum, and a local + // 9999-12-31 23:59:59 west of it is an instant above the maximum. cctz resolves year 0 (it + // uses the proleptic Gregorian calendar), and a year outside 0..9999 fails the check below + // because `from_unixtime()` narrows it into a `uint16_t`. if (!datetime_value.is_valid_date()) { return Status::DataQualityError( "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}", diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp index 02059045469285..91417a17be738d 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/config.h" @@ -41,13 +42,8 @@ namespace doris { -// This number represents the number of days from 0000-01-01 to 1970-01-01. -static constexpr int32_t date_threshold = 719528; - namespace { -constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528; - Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column, const OrcDecodedColumnView& orc_view) { const auto* orc_batch = dynamic_cast(orc_view.batch); @@ -64,22 +60,44 @@ Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column, orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows); std::vector date_values; date_values.resize(output_rows); - auto& date_dict = date_day_offset_dict::get(); for (size_t row = 0; row < output_rows; ++row) { const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows); - const auto date = date_dict[cast_set(orc_batch->data[source_row])]; - date_values[row] = cast_set(date.daynr() - DORIS_DATE_EPOCH_DAYNR); + // The payload of a null slot is undefined, so it is zeroed rather than range-checked; + // `read_column_from_decoded_values` never decodes it. + if (view.null_map != nullptr && view.null_map[row] != 0) { + date_values[row] = 0; + continue; + } + // ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, the same encoding + // `decode_epoch_days()` expects, but ORC hands it over in an int64 batch. Reject a value + // that does not fit the int32 view here, while the real file value is still available for + // the message, instead of narrowing it blindly. + const int64_t file_days = orc_batch->data[source_row]; + if (file_days < std::numeric_limits::min() || + file_days > std::numeric_limits::max()) { + return Status::DataQualityError( + "DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, " + "0000-02-29 excluded)", + file_days); + } + date_values[row] = static_cast(file_days); } view.values = reinterpret_cast(date_values.data()); RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view)); return Status::OK(); } -Status decode_parquet_date(int32_t encoded_date, DateV2Value* value) { +// Shared by every "days since 1970-01-01" source: Parquet, ORC and Arrow date32/date64. The +// message must therefore not name one format, and it carries the value so a broken file can be +// identified without re-reading it. +Status decode_epoch_days(int64_t encoded_date, DateV2Value* value) { DORIS_CHECK(value != nullptr); - const int64_t day_number = static_cast(encoded_date) + date_threshold; - if (day_number < 0 || !value->get_date_from_daynr(static_cast(day_number))) { - return Status::DataQualityError("Parquet DATE value is out of range"); + const int64_t day_number = epoch_days_to_daynr(encoded_date); + if (day_number == 0 || !value->get_date_from_daynr(static_cast(day_number))) { + return Status::DataQualityError( + "DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, " + "0000-02-29 excluded)", + encoded_date); } return Status::OK(); } @@ -99,7 +117,7 @@ class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { _data.resize(old_size + num_values); for (size_t row = 0; row < num_values; ++row) { DateV2Value value; - const auto status = decode_parquet_date( + const auto status = decode_epoch_days( unaligned_load(values + row * sizeof(int32_t)), &value); if (!status.ok()) { if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { @@ -216,13 +234,12 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const IColumn& column, const N const auto& col_data = static_cast(column).get_data(); auto& date32_builder = assert_cast(*array_builder); for (size_t i = start; i < end; ++i) { - auto daynr = col_data[i].daynr() - date_threshold; if (null_map && (*null_map)[i]) { RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(), column, *array_builder)); } else { - RETURN_IF_ERROR( - checkArrowStatus(date32_builder.Append(cast_set(daynr)), - column, *array_builder)); + RETURN_IF_ERROR(checkArrowStatus( + date32_builder.Append(daynr_to_epoch_days(col_data[i].daynr())), column, + *array_builder)); } } return Status::OK(); @@ -257,10 +274,9 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow: "Arrow Date64 value must contain whole days: row={}, milliseconds={}", value_i, milliseconds); } - const int64_t daynr = milliseconds / MILLISECONDS_PER_DAY + date_threshold; DateV2Value value; - if (daynr <= 0 || daynr > DATE_MAX_DAYNR || - !value.get_date_from_daynr(static_cast(daynr))) { + if (const auto status = decode_epoch_days(milliseconds / MILLISECONDS_PER_DAY, &value); + !status.ok()) { return Status::InvalidArgument( "Arrow Date64 value is outside the Doris DATE range: " "row={}, milliseconds={}", @@ -281,7 +297,11 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow: auto date_value = unaligned_load(raw_byte_ptr); DateV2Value v; - v.get_date_from_daynr(date_value + date_threshold); + if (const auto status = decode_epoch_days(date_value, &v); !status.ok()) { + return Status::InvalidArgument( + "Arrow Date32 value is outside the Doris DATE range: row={}, days={}", + value_i, date_value); + } col_data.emplace_back(v); } } else { @@ -309,7 +329,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, continue; } DateV2Value date_v2; - const auto status = decode_parquet_date(values[row], &date_v2); + const auto status = decode_epoch_days(values[row], &date_v2); if (!status.ok()) { // Decoded values back both metadata conversion and native materialization. Preserve // strict errors while allowing nullable non-strict scans to mark only the bad row. @@ -401,7 +421,7 @@ Status DataTypeDateV2SerDe::write_column_to_orc(const std::string& timezone, con if (cur_batch->notNull[row_id] == 0) { continue; } - cur_batch->data[row_id] = col_data[row_id].daynr() - date_threshold; + cur_batch->data[row_id] = daynr_to_epoch_days(col_data[row_id].daynr()); } cur_batch->numElements = end - start; return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index d83bc0745d932d..8f65c9fd53c54a 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -89,9 +89,13 @@ Status append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& da TimestampTzValue timestamp_tz; timestamp_tz.from_unixtime(epoch_seconds, UTC); timestamp_tz.set_microsecond(static_cast(micros_of_second)); + // `from_unixtime()` splits the instant with cctz (proleptic Gregorian, so year 0 exists) and + // narrows the civil year into a `uint16_t`. This is the exact range check for the target + // type: a year below 0 wraps above 9999 and a year of 10000 exceeds it, so both are rejected + // here, as is the proleptic-only 0000-02-29, which Doris's calendar does not have. if (!timestamp_tz.is_valid_date()) { return Status::DataQualityError( - "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", + "Decoded TIMESTAMPTZ is outside the Doris 0000-9999 range: micros={}", timestamp_micros); } data.push_back(timestamp_tz); diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h index ba2aa686ad4272..0c6b5aabfb542c 100644 --- a/be/src/core/data_type_serde/parquet_timestamp.h +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -33,14 +33,35 @@ struct ParquetInt96Timestamp { #pragma pack() static_assert(sizeof(ParquetInt96Timestamp) == 12); -inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL; -inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL; +// Doris DATETIMEV2 and TIMESTAMPTZ both start at 0000-01-01 00:00:00 (`MIN_DATE_V2`) and end at +// 9999-12-31 23:59:59.999999. Parquet timestamps count from 1970-01-01 in the proleptic Gregorian +// calendar, where year 0 IS a leap year, so 0000-01-01 is 366 days below 0001-01-01 rather than +// 365. Pinning the lower bound at 0001-01-01 made the reader stricter than the type it +// materializes into: every year-zero value Doris itself writes came back as a conversion failure. +inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62167219200000000LL; // 0000-01-01 00:00:00 +inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = + 253402300799999999LL; // 9999-12-31 23:59:59.999999 + +// A UTC-adjusted Parquet timestamp is an instant, not a civil value: which Doris DATETIME it +// becomes depends on the reader's timezone, so the raw instant must not be measured against the +// civil range. A local 0001-01-01 00:00:00 in Asia/Shanghai is an instant *below* the civil +// minimum, and a local 9999-12-31 23:59:59 in America/New_York is one *above* the maximum; both +// are representable and must survive. What stays here is only a coarse guard keeping the value in +// the domain where the cctz conversion and the narrowing to a `uint16_t` year behave. No timezone +// offset has ever exceeded 16 hours, so one day of slack admits every instant that can still land +// inside the civil range. The exact range is enforced per target type after the conversion: +// `epoch_days_to_daynr()` for a civil timestamp, `is_valid_date()` for an instant. +inline constexpr int64_t DORIS_TIMESTAMP_OFFSET_SLACK_MICROS = 86400000000LL; +inline constexpr int64_t MIN_DORIS_INSTANT_MICROS = + MIN_DORIS_TIMESTAMP_MICROS - DORIS_TIMESTAMP_OFFSET_SLACK_MICROS; +inline constexpr int64_t MAX_DORIS_INSTANT_MICROS = + MAX_DORIS_TIMESTAMP_MICROS + DORIS_TIMESTAMP_OFFSET_SLACK_MICROS; inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { - if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || - timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + if (timestamp_micros < MIN_DORIS_INSTANT_MICROS || + timestamp_micros > MAX_DORIS_INSTANT_MICROS) { return Status::DataQualityError( - "Parquet timestamp is outside the Doris 0001-9999 range: micros={}", + "Parquet timestamp is outside the Doris 0000-9999 range: micros={}", timestamp_micros); } return Status::OK(); @@ -83,10 +104,10 @@ inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value, const __int128 days = static_cast(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; const __int128 timestamp_micros = days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND; - if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || - timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + if (timestamp_micros < MIN_DORIS_INSTANT_MICROS || + timestamp_micros > MAX_DORIS_INSTANT_MICROS) { return Status::DataQualityError( - "Parquet INT96 timestamp is outside the Doris 0001-9999 range"); + "Parquet INT96 timestamp is outside the Doris 0000-9999 range"); } *result = static_cast(timestamp_micros); return Status::OK(); diff --git a/be/src/core/value/timestamptz_value.h b/be/src/core/value/timestamptz_value.h index a3e3861ac2be50..5186bdd8ab1064 100644 --- a/be/src/core/value/timestamptz_value.h +++ b/be/src/core/value/timestamptz_value.h @@ -35,8 +35,9 @@ struct CastParameters; // TIMESTAMPTZ can be understood as a DATETIME type with timezone conversion functionality. // Doris automatically handles timezone conversions internally. -// The storage format of TIMESTAMPTZ is the same as DATETIMEV2, both are 8-byte integers -// representing microseconds from 0001-01-01 00:00:00.000000 to 9999-12-31 23:59:59.999999. +// The storage format of TIMESTAMPTZ is the same as DATETIMEV2: both are 8-byte packed civil +// values spanning 0000-01-01 00:00:00.000000 (`MIN_DATETIME_V2`, also the default) to +// 9999-12-31 23:59:59.999999. Year 0 is inside the domain, so readers must not floor at year 1. // TIMESTAMPTZ does not store timezone information; conversions are performed during read and write // operations according to the specified timezone. // This requires that both reading and writing operations need a timezone parameter. diff --git a/be/src/core/value/vdatetime_value.h b/be/src/core/value/vdatetime_value.h index d7929bb101c167..0543d7ca0647a3 100644 --- a/be/src/core/value/vdatetime_value.h +++ b/be/src/core/value/vdatetime_value.h @@ -1694,8 +1694,6 @@ class date_day_offset_dict { static constexpr int START_YEAR = 1900; // 1900-01-01 static constexpr int END_YEAR = 2039; // 2039-10-24 - static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR = - 719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1) static std::array, DICT_DAYS> DATE_DAY_OFFSET_ITEMS; static std::array, 12>, 140> DATE_DAY_OFFSET_DICT; @@ -1710,6 +1708,9 @@ class date_day_offset_dict { date_day_offset_dict& operator=(const date_day_offset_dict&) = default; public: + static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR = + 719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1) + static bool can_speed_up_calc_daynr(int year) { return year >= START_YEAR && year <= END_YEAR; } static int get_offset_by_daynr(int daynr) { return daynr - DAY_OFFSET_CAL_START_POINT_DAYNR; } @@ -1770,6 +1771,47 @@ inline uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) { return delsum + y / 4 - y / 100 + y / 400; } +// Doris follows MySQL's calendar, in which year 0 is NOT a leap year (see `is_leap()`): 0000-02-29 +// does not exist and `calc_daynr()` therefore numbers 0000-01-01 .. 0000-02-28 one day ahead of the +// proleptic Gregorian calendar. Arrow `date32`, Parquet/ORC `DATE` and the Iceberg spec all define +// their day ordinal in the proleptic Gregorian calendar, where year 0 IS a leap year. The two +// numberings coincide from 0000-03-01 (daynr 60) onwards, so the whole difference is the missing +// 0000-02-29. Every conversion between a Doris DATE and an external "days since 1970-01-01" value +// must go through the two helpers below instead of adding/subtracting the epoch daynr directly. +inline constexpr int64_t DAYNR_OF_UNIX_EPOCH = + date_day_offset_dict::DAY_OFFSET_CAL_START_POINT_DAYNR; +inline constexpr int64_t DAYNR_OF_0000_03_01 = 60; // first daynr shared by both calendars + +// Doris daynr -> days since 1970-01-01 in the proleptic Gregorian calendar. +inline constexpr int32_t daynr_to_epoch_days(int64_t daynr) { + return static_cast(daynr - DAYNR_OF_UNIX_EPOCH - + (daynr < DAYNR_OF_0000_03_01 ? 1 : 0)); +} + +// The three boundaries of the external ordinal domain, derived from the daynr domain rather than +// typed out, so a change to either calendar constant cannot leave them behind. +inline constexpr int64_t EPOCH_DAYS_MIN = daynr_to_epoch_days(1); // 0000-01-01, smallest Doris DATE +inline constexpr int64_t EPOCH_DAYS_0000_02_29 = + daynr_to_epoch_days(DAYNR_OF_0000_03_01) - 1; // exists in proleptic Gregorian only +inline constexpr int64_t EPOCH_DAYS_MAX = + daynr_to_epoch_days(DATE_MAX_DAYNR); // 9999-12-31, the largest Doris DATE + +static_assert(DAYNR_OF_UNIX_EPOCH == 719528, "calc_daynr(1970, 1, 1)"); +static_assert(EPOCH_DAYS_MIN == -719528); +static_assert(EPOCH_DAYS_0000_02_29 == -719469); +static_assert(EPOCH_DAYS_MAX == 2932896); + +// Inverse of `daynr_to_epoch_days()`. Returns 0, which is never a valid daynr, when the value has +// no Doris DATE representation: before 0000-01-01, after 9999-12-31, or the proleptic-only +// 0000-02-29. Callers must reject 0 instead of feeding it to `get_date_from_daynr()`. +inline constexpr int64_t epoch_days_to_daynr(int64_t epoch_days) { + if (epoch_days < EPOCH_DAYS_MIN || epoch_days > EPOCH_DAYS_MAX || + epoch_days == EPOCH_DAYS_0000_02_29) { + return 0; + } + return epoch_days + DAYNR_OF_UNIX_EPOCH + (epoch_days < EPOCH_DAYS_0000_02_29 ? 1 : 0); +} + // DAY / WEEK fast path. Real workloads hold dates that cluster in a narrow range, so the two // dictionary lookups below (`daynr(y, m, d)` and `daynr -> date`) are L1-resident; measured about // 3x faster than the generic `date_add_interval` (no TimeInterval, no second-level diff --git a/be/src/exec/sink/writer/iceberg/partition_transformers.h b/be/src/exec/sink/writer/iceberg/partition_transformers.h index 51bca7000ad5bb..c05be337c0286d 100644 --- a/be/src/exec/sink/writer/iceberg/partition_transformers.h +++ b/be/src/exec/sink/writer/iceberg/partition_transformers.h @@ -20,9 +20,10 @@ #include "core/column/column.h" #include "core/column/column_nullable.h" #include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" #include "exec/common/stringop_substring.h" -#include "exprs/function/cast/cast_to_datetimev2_impl.hpp" -#include "exprs/function/cast/cast_to_datev2_impl.hpp" +#include "exprs/function/function_helpers.h" #include "util/bit_util.h" namespace doris { @@ -44,35 +45,16 @@ class PartitionColumnTransforms { const doris::iceberg::PartitionField& field, const DataTypePtr& source_type); }; +// Iceberg time transforms are all defined relative to 1970-01-01 (spec: Partition Transforms). +inline constexpr int ICEBERG_EPOCH_YEAR = 1970; + class PartitionColumnTransformUtils { public: - static DateV2Value& epoch_date() { - static DateV2Value epoch_date; - static bool initialized = false; - if (!initialized) { - CastParameters params; - DORIS_CHECK((CastToDateV2::from_string_strict_mode( - {"1970-01-01 00:00:00", 19}, epoch_date, nullptr, params))); - initialized = true; - } - return epoch_date; - } - - static DateV2Value& epoch_datetime() { - static DateV2Value epoch_datetime; - static bool initialized = false; - if (!initialized) { - CastParameters params; - DORIS_CHECK((CastToDatetimeV2::from_string_strict_mode( - {"1970-01-01 00:00:00", 19}, epoch_datetime, nullptr, -1, params))); - initialized = true; - } - return epoch_datetime; - } - static std::string human_year(int year_ordinal) { auto ymd = std::chrono::year_month_day {EPOCH} + std::chrono::years(year_ordinal); - return std::to_string(static_cast(ymd.year())); + // iceberg-api's TransformUtil.humanYear is String.format("%04d", ...): year 0 has to be + // "0000", not "0", or the partition directory differs from the one Spark writes. + return fmt::format("{:04d}", static_cast(ymd.year())); } static std::string human_month(int month_ordinal) { @@ -89,13 +71,15 @@ class PartitionColumnTransformUtils { } static std::string human_hour(int hour_ordinal) { - int day_value = hour_ordinal / 24; - int housr_value = hour_ordinal % 24; + // Hour ordinals are negative before 1970-01-01, so the split must floor rather than + // truncate towards zero: -1 is 1969-12-31-23, not 1970-01-01 minus one hour. + int day_value = (hour_ordinal >= 0 ? hour_ordinal : hour_ordinal - 23) / 24; + int hour_value = hour_ordinal - day_value * 24; auto ymd = std::chrono::year_month_day(std::chrono::sys_days( std::chrono::floor(EPOCH + std::chrono::days(day_value)))); return fmt::format("{:04d}-{:02d}-{:02d}-{:02d}", static_cast(ymd.year()), static_cast(ymd.month()), static_cast(ymd.day()), - housr_value); + hour_value); } private: @@ -615,7 +599,7 @@ class DateBucketPartitionColumnTransform : public PartitionColumnTransform { DateV2Value value = binary_cast>(*(UInt32*)p_in); - int64_t days_from_unix_epoch = value.daynr() - 719528; + int64_t days_from_unix_epoch = daynr_to_epoch_days(value.daynr()); uint32_t hash_value = HashUtil::murmur_hash3_32(&days_from_unix_epoch, sizeof(days_from_unix_epoch), 0); @@ -686,7 +670,12 @@ class TimestampBucketPartitionColumnTransform : public PartitionColumnTransform LOG(WARNING) << "Failed to call unix_timestamp :" << value.debug_string(); timestamp = 0; } - Int64 long_value = static_cast(timestamp) * 1000000; + // Iceberg hashes the full microsecond value of a timestamp (spec: Partition + // Transforms, `bucket` over `timestamp`), so the sub-second part must be carried + // along; dropping it puts a DATETIME(6) row in a different bucket than Spark does. + // `unix_timestamp()` returns whole seconds floored towards negative infinity, so + // adding the wall-clock microseconds is exact before the epoch as well. + Int64 long_value = static_cast(timestamp) * 1000000 + value.microsecond(); uint32_t hash_value = HashUtil::murmur_hash3_32(&long_value, sizeof(long_value), 0); *p_out = (hash_value & INT32_MAX) % _bucket_num; @@ -820,9 +809,10 @@ class DateYearPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt32*)p_in); - // datetime_diff actually returns int - *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_date(), value)); + // Iceberg's `year` transform counts whole calendar years from 1970 and floors: + // 1969-06-15 is -1, not the 0 that `datetime_diff` returns by rounding towards + // zero. Taking the calendar year directly is exactly that floor. + *p_out = value.year() - ICEBERG_EPOCH_YEAR; ++p_in; ++p_out; } @@ -889,9 +879,10 @@ class TimestampYearPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt64*)p_in); - // datetime_diff actually returns int - *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_datetime(), value)); + // Iceberg's `year` transform counts whole calendar years from 1970 and floors: + // 1969-06-15 is -1, not the 0 that `datetime_diff` returns by rounding towards + // zero. Taking the calendar year directly is exactly that floor. + *p_out = value.year() - ICEBERG_EPOCH_YEAR; ++p_in; ++p_out; } @@ -958,9 +949,10 @@ class DateMonthPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt32*)p_in); - // datetime_diff actually returns int - *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_date(), value)); + // Iceberg's `month` transform counts whole calendar months from 1970-01 and floors, + // so 1969-06-15 is -7 rather than the -6 that `datetime_diff` produces by + // rounding towards zero. The day of month never participates. + *p_out = (value.year() - ICEBERG_EPOCH_YEAR) * 12 + (value.month() - 1); ++p_in; ++p_out; } @@ -1027,9 +1019,10 @@ class TimestampMonthPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt64*)p_in); - // datetime_diff actually returns int - *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_datetime(), value)); + // Iceberg's `month` transform counts whole calendar months from 1970-01 and floors, + // so 1969-06-15 is -7 rather than the -6 that `datetime_diff` produces by + // rounding towards zero. The day of month never participates. + *p_out = (value.year() - ICEBERG_EPOCH_YEAR) * 12 + (value.month() - 1); ++p_in; ++p_out; } @@ -1096,9 +1089,10 @@ class DateDayPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt32*)p_in); - // datetime_diff actually returns int - *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_date(), value)); + // Iceberg's `day` transform is "days from 1970-01-01" in the proleptic Gregorian + // calendar (spec: Partition Transforms). `datetime_diff` cannot be used here: it + // counts in Doris's MySQL calendar, which is one day ahead for year-zero dates. + *p_out = daynr_to_epoch_days(value.daynr()); ++p_in; ++p_out; } @@ -1170,9 +1164,10 @@ class TimestampDayPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt64*)p_in); - // datetime_diff actually returns int - *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_datetime(), value)); + // Same as the DATE variant: the partition value is the proleptic-Gregorian ordinal + // of the calendar day. Note this is a floor, not a truncation towards the epoch, so + // `datetime_diff` (which rounds towards zero by the time part) is not usable. + *p_out = daynr_to_epoch_days(value.daynr()); ++p_in; ++p_out; } @@ -1243,9 +1238,12 @@ class TimestampHourPartitionColumnTransform : public PartitionColumnTransform { while (p_in < end_in) { DateV2Value value = binary_cast>(*(UInt64*)p_in); - // hour diff would't overflow int32 + // Iceberg's `hour` transform floors to the hour boundary. Deriving it from the + // proleptic day ordinal plus the wall-clock hour is exact for every representable + // timestamp and, unlike `datetime_diff`, floors instead of truncating towards + // the epoch. The product cannot overflow int32: 2932896 * 24 + 23 == 70389527. *p_out = cast_set( - datetime_diff(PartitionColumnTransformUtils::epoch_datetime(), value)); + static_cast(daynr_to_epoch_days(value.daynr())) * 24 + value.hour()); ++p_in; ++p_out; } diff --git a/be/src/format/arrow/arrow_stream_reader.cpp b/be/src/format/arrow/arrow_stream_reader.cpp index 7000c55507b63c..aebe3fe47e2fe7 100644 --- a/be/src/format/arrow/arrow_stream_reader.cpp +++ b/be/src/format/arrow/arrow_stream_reader.cpp @@ -114,10 +114,15 @@ Status ArrowStreamReader::_do_get_next_block(Block* block, size_t* read_rows, bo column_name_in_block, column_name); } - RETURN_IF_ERROR( + auto status = columns_guard.get_datatype_by_position(c) ->get_serde() - ->read_column_from_arrow(*columns[c], column, 0, num_rows, _ctzz)); + ->read_column_from_arrow(*columns[c], column, 0, num_rows, _ctzz); + if (!status.ok()) { + // The SerDe sees only a value buffer, so it cannot name the column itself. + return status.prepend( + fmt::format("Failed to read arrow column '{}': ", column_name)); + } } catch (Exception& e) { return Status::InternalError("Failed to convert from arrow to block: {}", e.what()); } diff --git a/be/src/format/table/remote_doris_reader.cpp b/be/src/format/table/remote_doris_reader.cpp index 0e2184d65b62f5..ba30c047805999 100644 --- a/be/src/format/table/remote_doris_reader.cpp +++ b/be/src/format/table/remote_doris_reader.cpp @@ -86,10 +86,15 @@ Status RemoteDorisReader::_do_get_next_block(Block* block, size_t* read_rows, bo try { auto block_pos = (*_col_name_to_block_idx)[column_name]; - RETURN_IF_ERROR(columns_guard.get_datatype_by_position(block_pos) - ->get_serde() - ->read_column_from_arrow(*columns[block_pos], column, 0, - num_rows, _ctzz)); + auto status = columns_guard.get_datatype_by_position(block_pos) + ->get_serde() + ->read_column_from_arrow(*columns[block_pos], column, 0, num_rows, + _ctzz); + if (!status.ok()) { + // The SerDe sees only a value buffer, so it cannot name the column itself. + return status.prepend( + fmt::format("Failed to read arrow column '{}': ", column_name)); + } } catch (Exception& e) { return Status::InternalError( "Failed to convert from arrow to block, column_name: {}, e: {}", column_name, diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 83cb929e18bccc..6cfafe4f44286b 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -486,10 +486,26 @@ bool set_date_zone_map(const ::orc::ColumnStatistics& statistics, segment_v2::Zo !date_statistics->hasMaximum()) { return false; } - auto& date_dict = date_day_offset_dict::get(); - return set_validated_zone_map( - Field::create_field(date_dict[date_statistics->getMinimum()]), - Field::create_field(date_dict[date_statistics->getMaximum()]), zone_map); + // ORC DATE statistics are proleptic-Gregorian day ordinals, the same domain the row decoder + // (DataTypeDateV2SerDe::read_column_from_orc) interprets. Converting them through + // `date_day_offset_dict` instead would put the year-zero window one day off the rows and let a + // pushed-down MIN/MAX report a value no row holds. A bound with no Doris DATE disables the + // statistics, so MIN/MAX falls back to scanning rows. + const auto to_date = [](int64_t epoch_days) -> std::optional> { + const int64_t daynr = epoch_days_to_daynr(epoch_days); + DateV2Value value; + if (daynr == 0 || !value.get_date_from_daynr(static_cast(daynr))) { + return std::nullopt; + } + return value; + }; + const auto min_value = to_date(date_statistics->getMinimum()); + const auto max_value = to_date(date_statistics->getMaximum()); + if (!min_value.has_value() || !max_value.has_value()) { + return false; + } + return set_validated_zone_map(Field::create_field(*min_value), + Field::create_field(*max_value), zone_map); } std::optional> datetime_v2_from_orc_millis( @@ -2247,8 +2263,15 @@ Status OrcReader::_decode_column_into_block(const ::orc::StructVectorBatch& stru const auto* selected_type = _state->selected_type->getSubtype(selected_batch_idx); DORIS_CHECK(selected_type != nullptr); auto column = file_block->get_by_position(block_position.value()).column->assert_mutable(); - RETURN_IF_ERROR(_decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx], - column, rows, selected_rows)); + auto status = _decode_column(*type, *selected_type, *struct_batch.fields[selected_batch_idx], + column, rows, selected_rows); + if (!status.ok()) { + // The decoders work on a bare value buffer and cannot name the column themselves; without + // this the user only learns that some DATE/TIMESTAMP in some file is unrepresentable. + return status.prepend(fmt::format( + "Failed to decode ORC column '{}': ", + _state->root_type->getFieldName(static_cast(file_column_id.value())))); + } file_block->replace_by_position(block_position.value(), std::move(column)); return Status::OK(); } diff --git a/be/test/core/data_type_serde/data_type_datetimev2_serde_calendar_test.cpp b/be/test/core/data_type_serde/data_type_datetimev2_serde_calendar_test.cpp new file mode 100644 index 00000000000000..cb5df175afafc8 --- /dev/null +++ b/be/test/core/data_type_serde/data_type_datetimev2_serde_calendar_test.cpp @@ -0,0 +1,555 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Doris DATETIMEV2 starts at 0000-01-01 00:00:00, but the Parquet reader used to reject anything +// below 0001-01-01: a value Doris accepts, stores and exports could not be read back out of +// Doris's own file. Two independent defects sit behind that. +// +// 1. The range gate was a whole year narrower than the target type, and it was applied to the +// raw instant *before* the timezone offset, so it also lost values near year 1 east of UTC +// and near year 9999 west of it. +// 2. The local-timestamp materialization added `calc_daynr(1970, 1, 1)` to a proleptic +// Gregorian day ordinal. Doris follows MySQL's calendar, in which year 0 is not a leap year, +// so the two numberings differ for 0000-01-01 .. 0000-02-28 and the whole window decoded one +// day early. +// +// These tests pin both, against an oracle (cctz) that is independent of Doris's own arithmetic. +// See https://github.com/apache/doris/issues/67447 + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "core/value/vdatetime_value.h" + +namespace doris { +namespace { + +#pragma pack(1) +struct TestInt96 { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(TestInt96) == 12); + +constexpr int32_t JULIAN_UNIX_EPOCH = 2440588; +constexpr int64_t MICROS_PER_SECOND = 1000000LL; +constexpr int64_t MICROS_PER_DAY = 86400000000LL; + +// Independent oracle: cctz uses the proleptic Gregorian calendar, which is exactly what Parquet +// timestamps are defined in. Nothing here goes through Doris's own day arithmetic. +int64_t utc_micros(int year, int month, int day, int hour = 0, int minute = 0, int second = 0, + int64_t microsecond = 0) { + const auto tp = cctz::convert(cctz::civil_second(year, month, day, hour, minute, second), + cctz::utc_time_zone()); + return tp.time_since_epoch().count() * MICROS_PER_SECOND + microsecond; +} + +std::string civil_day_string(int64_t epoch_days) { + const cctz::civil_day day = cctz::civil_day(1970, 1, 1) + epoch_days; + char buffer[32]; + snprintf(buffer, sizeof(buffer), "%04d-%02d-%02d", static_cast(day.year()), + static_cast(day.month()), static_cast(day.day())); + return buffer; +} + +class VectorDecodeSource final : public ParquetDecodeSource { +public: + template + void set_values(const std::vector& values) { + _width = sizeof(T); + _values.resize(values.size() * sizeof(T)); + memcpy(_values.data(), values.data(), _values.size()); + } + + template + void set_dictionary(const std::vector& values, std::vector indices) { + _dictionary_width = sizeof(T); + _dictionary.resize(values.size() * sizeof(T)); + memcpy(_dictionary.data(), values.data(), _dictionary.size()); + _indices = std::move(indices); + _index_offset = 0; + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + const uint8_t* begin = _values.data() + _offset * _width; + _offset += num_values; + return consumer.consume(begin, num_values, _width); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + return Status::NotSupported("binary values are not part of these tests"); + } + + Status skip_values(size_t num_values) override { + _offset += num_values; + _index_offset += num_values; + return Status::OK(); + } + + bool has_dictionary() const override { return !_dictionary.empty(); } + uint64_t dictionary_generation() const override { return 1; } + size_t dictionary_size() const override { + return _dictionary_width == 0 ? 0 : _dictionary.size() / _dictionary_width; + } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return fixed_consumer.consume(_dictionary.data(), dictionary_size(), _dictionary_width); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + indices->assign(_indices.begin() + _index_offset, + _indices.begin() + _index_offset + num_values); + _index_offset += num_values; + return Status::OK(); + } + +private: + std::vector _values; + std::vector _dictionary; + std::vector _indices; + size_t _width = 0; + size_t _dictionary_width = 0; + size_t _offset = 0; + size_t _index_offset = 0; +}; + +struct CivilCase { + const char* rendered; + int year; + int month; + int day; + int hour; + int minute; + int second; + int64_t microsecond; +}; + +// Brackets both edges of the 0000-01-01 .. 0000-02-28 window where Doris's day numbering and the +// proleptic Gregorian ordinal disagree, plus the two range limits and the rows from the report. +const std::vector& civil_cases() { + static const std::vector cases = { + {"0000-01-01 00:00:00.000000", 0, 1, 1, 0, 0, 0, 0}, + {"0000-01-01 12:34:56.000000", 0, 1, 1, 12, 34, 56, 0}, + {"0000-01-02 00:00:00.000000", 0, 1, 2, 0, 0, 0, 0}, + {"0000-01-31 23:59:59.999999", 0, 1, 31, 23, 59, 59, 999999}, + {"0000-02-28 23:59:59.999999", 0, 2, 28, 23, 59, 59, 999999}, + {"0000-03-01 00:00:00.000000", 0, 3, 1, 0, 0, 0, 0}, + {"0000-12-31 00:00:00.000000", 0, 12, 31, 0, 0, 0, 0}, + {"0001-01-01 00:00:00.000000", 1, 1, 1, 0, 0, 0, 0}, + {"1969-12-31 23:59:59.000000", 1969, 12, 31, 23, 59, 59, 0}, + {"1970-01-01 00:00:00.000000", 1970, 1, 1, 0, 0, 0, 0}, + {"2024-01-01 12:00:00.000000", 2024, 1, 1, 12, 0, 0, 0}, + {"9999-12-31 23:59:59.999999", 9999, 12, 31, 23, 59, 59, 999999}, + }; + return cases; +} + +int64_t case_micros(const CivilCase& c) { + return utc_micros(c.year, c.month, c.day, c.hour, c.minute, c.second, c.microsecond); +} + +// Materializes `values` as DATETIMEV2 through the plain Parquet path. `timezone` is only consulted +// when `adjusted_to_utc` is set, matching the reader. +Status materialize_datetime(const std::vector& values, bool adjusted_to_utc, + const cctz::time_zone* timezone, MutableColumnPtr* column, + IColumn::Filter* null_map = nullptr, + ParquetTimeUnit unit = ParquetTimeUnit::MICROS) { + static DataTypeDateTimeV2 type(6); + VectorDecodeSource source; + source.set_values(values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = unit, + .timestamp_is_adjusted_to_utc = adjusted_to_utc, + .timezone = timezone}; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, source, context, values.size(), + state); +} + +std::string rendered(const IColumn& column, size_t row) { + static DataTypeDateTimeV2 type(6); + return type.to_string(column, row); +} + +} // namespace + +class DataTypeDateTimeV2SerDeCalendarTest : public ::testing::Test {}; + +// The gate must not be stricter than the type it materializes into. Deriving both ends from the +// type's own limits keeps that true if either side ever moves. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, RangeBoundsMatchTheDateTimeType) { + const DateV2Value min_date(static_cast(MIN_DATE_V2)); + const DateV2Value max_date(static_cast(MAX_DATE_V2)); + ASSERT_EQ(0, min_date.year()); + ASSERT_EQ(1, min_date.month()); + ASSERT_EQ(1, min_date.day()); + ASSERT_EQ(9999, max_date.year()); + + EXPECT_EQ(utc_micros(min_date.year(), min_date.month(), min_date.day()), + MIN_DORIS_TIMESTAMP_MICROS); + EXPECT_EQ(utc_micros(max_date.year(), max_date.month(), max_date.day(), 23, 59, 59, 999999), + MAX_DORIS_TIMESTAMP_MICROS); + // Year 0 is a leap year in the proleptic Gregorian calendar, so the old 0001-01-01 floor was + // 366 days -- not 365 -- above the type's minimum. + EXPECT_EQ(366 * MICROS_PER_DAY, utc_micros(1, 1, 1) - MIN_DORIS_TIMESTAMP_MICROS); +} + +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, LocalTimestampKeepsTheCivilValue) { + std::vector values; + for (const auto& c : civil_cases()) { + values.push_back(case_micros(c)); + } + + MutableColumnPtr column; + ASSERT_TRUE(materialize_datetime(values, false, nullptr, &column).ok()); + ASSERT_EQ(civil_cases().size(), column->size()); + for (size_t i = 0; i < civil_cases().size(); ++i) { + EXPECT_EQ(civil_cases()[i].rendered, rendered(*column, i)) << "row " << i; + } +} + +// A UTC-adjusted timestamp is an instant. Read in UTC it must land on the same civil value as the +// local encoding of the same wall clock. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, UtcTimestampKeepsTheCivilValueInUtc) { + std::vector values; + for (const auto& c : civil_cases()) { + values.push_back(case_micros(c)); + } + + const auto utc = cctz::utc_time_zone(); + MutableColumnPtr column; + ASSERT_TRUE(materialize_datetime(values, true, &utc, &column).ok()); + ASSERT_EQ(civil_cases().size(), column->size()); + for (size_t i = 0; i < civil_cases().size(); ++i) { + EXPECT_EQ(civil_cases()[i].rendered, rendered(*column, i)) << "row " << i; + } +} + +// The offset has to be applied before the civil range is judged. A local 0001-01-01 00:00:00 east +// of UTC is an instant *below* the civil minimum and a local 9999-12-31 23:59:59 west of it is an +// instant *above* the maximum; both are representable DATETIME values and must survive. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, UtcTimestampSurvivesOffsetsAtTheRangeEdges) { + const auto plus_eight = cctz::fixed_time_zone(std::chrono::hours(8)); + const int64_t offset_eight = 8 * 3600 * MICROS_PER_SECOND; + + // Below the floor the gate used to sit at, which is why this value was lost even though + // year 1 was supposed to be inside the accepted range. + const int64_t year_one_instant = utc_micros(1, 1, 1) - offset_eight; + ASSERT_LT(year_one_instant, utc_micros(1, 1, 1)) << "test no longer covers the old floor"; + MutableColumnPtr year_one_column; + ASSERT_TRUE(materialize_datetime({year_one_instant}, true, &plus_eight, &year_one_column).ok()); + EXPECT_EQ("0001-01-01 00:00:00.000000", rendered(*year_one_column, 0)); + + // Below the current floor as well: the civil range can only be judged after the offset, or + // widening the constant would simply move the same defect down by one year. + const int64_t year_zero_instant = MIN_DORIS_TIMESTAMP_MICROS - offset_eight; + ASSERT_LT(year_zero_instant, MIN_DORIS_TIMESTAMP_MICROS) + << "test no longer covers the low edge"; + MutableColumnPtr year_zero_column; + ASSERT_TRUE( + materialize_datetime({year_zero_instant}, true, &plus_eight, &year_zero_column).ok()); + EXPECT_EQ("0000-01-01 00:00:00.000000", rendered(*year_zero_column, 0)); + + // The same asymmetry at the top, west of UTC. + const auto minus_five = cctz::fixed_time_zone(std::chrono::hours(-5)); + const int64_t high_instant = MAX_DORIS_TIMESTAMP_MICROS + 5 * 3600 * MICROS_PER_SECOND; + ASSERT_GT(high_instant, MAX_DORIS_TIMESTAMP_MICROS) << "test no longer covers the high edge"; + MutableColumnPtr high_column; + ASSERT_TRUE(materialize_datetime({high_instant}, true, &minus_five, &high_column).ok()); + EXPECT_EQ("9999-12-31 23:59:59.999999", rendered(*high_column, 0)); + + // An offset only shifts the window; it does not widen it. One microsecond further out on + // either side still has no DATETIME representation in that timezone. + MutableColumnPtr rejected; + EXPECT_FALSE(materialize_datetime({year_zero_instant - 1}, true, &plus_eight, &rejected).ok()); + EXPECT_FALSE(materialize_datetime({high_instant + 1}, true, &minus_five, &rejected).ok()); +} + +// Every representable day of year zero, checked against cctz rather than against Doris's own +// day arithmetic. 0000-02-29 exists in the proleptic Gregorian calendar but not in Doris's, so it +// is the one day in the window that must fail instead of colliding with 0000-02-28 or 0000-03-01. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, LocalTimestampCoversEveryDayOfYearZero) { + constexpr int64_t FIRST_DAY = -719528; // 0000-01-01 + constexpr int64_t LAST_DAY = -719163; // 0000-12-31 + constexpr int64_t LEAP_DAY = -719469; // 0000-02-29, proleptic Gregorian only + const auto row_count = static_cast(LAST_DAY - FIRST_DAY + 1); + ASSERT_EQ(366, row_count); + + std::vector values; + values.reserve(row_count); + for (int64_t day = FIRST_DAY; day <= LAST_DAY; ++day) { + values.push_back(day * MICROS_PER_DAY); + } + + IColumn::Filter null_map(row_count, 0); + MutableColumnPtr column; + ASSERT_TRUE(materialize_datetime(values, false, nullptr, &column, &null_map).ok()); + ASSERT_EQ(row_count, column->size()); + + size_t rejected = 0; + for (size_t i = 0; i < row_count; ++i) { + const int64_t day = FIRST_DAY + static_cast(i); + if (day == LEAP_DAY) { + EXPECT_EQ(1, null_map[i]) << "0000-02-29 must not materialize"; + ++rejected; + continue; + } + ASSERT_EQ(0, null_map[i]) << "day " << day << " was rejected"; + EXPECT_EQ(civil_day_string(day) + " 00:00:00.000000", rendered(*column, i)) + << "day " << day; + } + EXPECT_EQ(1, rejected); +} + +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, RejectsValuesOutsideTheDateTimeRange) { + const auto utc = cctz::utc_time_zone(); + const auto expect_rejected = [&](int64_t micros, bool adjusted_to_utc, const char* why) { + MutableColumnPtr column; + const auto status = materialize_datetime({micros}, adjusted_to_utc, + adjusted_to_utc ? &utc : nullptr, &column); + EXPECT_FALSE(status.ok()) << why << " (micros=" << micros << ")"; + EXPECT_EQ(0, column->size()) << why; + }; + + for (bool adjusted : {false, true}) { + expect_rejected(MIN_DORIS_TIMESTAMP_MICROS - 1, adjusted, "one micro before 0000-01-01"); + expect_rejected(MIN_DORIS_TIMESTAMP_MICROS - MICROS_PER_DAY, adjusted, + "one day before 0000-01-01"); + expect_rejected(MAX_DORIS_TIMESTAMP_MICROS + 1, adjusted, + "one micro after 9999-12-31 23:59:59.999999"); + expect_rejected(utc_micros(10000, 1, 1), adjusted, "year 10000"); + } + // The proleptic-only leap day has no Doris representation even though it is inside the range. + expect_rejected(-719469 * MICROS_PER_DAY, false, "0000-02-29 as a local timestamp"); + expect_rejected(-719469 * MICROS_PER_DAY, true, "0000-02-29 as an instant"); +} + +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, MillisUnitKeepsYearZero) { + // INT64 nanos cannot reach year zero at all (it only spans 1677..2262), so millis is the only + // other unit that needs covering here. + const std::vector millis {utc_micros(0, 1, 1, 12, 34, 56) / 1000, + utc_micros(0, 2, 28) / 1000, utc_micros(0, 3, 1) / 1000}; + MutableColumnPtr column; + ASSERT_TRUE( + materialize_datetime(millis, false, nullptr, &column, nullptr, ParquetTimeUnit::MILLIS) + .ok()); + ASSERT_EQ(3, column->size()); + EXPECT_EQ("0000-01-01 12:34:56.000000", rendered(*column, 0)); + EXPECT_EQ("0000-02-28 00:00:00.000000", rendered(*column, 1)); + EXPECT_EQ("0000-03-01 00:00:00.000000", rendered(*column, 2)); +} + +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, Int96KeepsYearZero) { + // INT96 is always an instant. 0000-01-01 is Julian day 1721060. + const std::vector values { + {45296LL * 1000000000LL, JULIAN_UNIX_EPOCH - 719528}, // 0000-01-01 12:34:56 + {0, JULIAN_UNIX_EPOCH - 719468}, // 0000-03-01 00:00:00 + }; + VectorDecodeSource source; + source.set_values(values); + const auto utc = cctz::utc_time_zone(); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP, + .timezone = &utc}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + ASSERT_EQ(2, column->size()); + EXPECT_EQ("0000-01-01 12:34:56.000000", rendered(*column, 0)); + EXPECT_EQ("0000-03-01 00:00:00.000000", rendered(*column, 1)); +} + +// Dictionary-encoded pages convert each entry once and then fan the result out over the row +// indices, a different code path from the plain decoder above. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, DictionaryEncodedYearZeroMaterializes) { + const std::vector dictionary {utc_micros(0, 1, 1, 12, 34, 56), utc_micros(0, 3, 1), + utc_micros(2024, 1, 1, 12)}; + VectorDecodeSource source; + source.set_dictionary(dictionary, {2, 0, 1, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS}; + IColumn::Filter null_map(4, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 4, state).ok()); + ASSERT_EQ(4, column->size()); + EXPECT_EQ(null_map, IColumn::Filter({0, 0, 0, 0})); + EXPECT_EQ("2024-01-01 12:00:00.000000", rendered(*column, 0)); + EXPECT_EQ("0000-01-01 12:34:56.000000", rendered(*column, 1)); + EXPECT_EQ("0000-03-01 00:00:00.000000", rendered(*column, 2)); + EXPECT_EQ("0000-01-01 12:34:56.000000", rendered(*column, 3)); +} + +// The decoded-value path is a second copy of the same conversion, used when a reader hands Doris +// pre-decoded values instead of an encoded page. It shares the helpers, so it shares the bug. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, DecodedValuesKeepYearZero) { + const std::vector values {utc_micros(0, 1, 1, 12, 34, 56), utc_micros(0, 1, 31), + utc_micros(0, 3, 1)}; + NullMap conversion_failures(values.size(), 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT64, + .time_unit = DecodedTimeUnit::MICROS, + .row_count = static_cast(values.size()), + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + ASSERT_EQ(values.size(), column->size()); + EXPECT_EQ(conversion_failures, NullMap({0, 0, 0})); + EXPECT_EQ("0000-01-01 12:34:56.000000", rendered(*column, 0)); + EXPECT_EQ("0000-01-31 00:00:00.000000", rendered(*column, 1)); + EXPECT_EQ("0000-03-01 00:00:00.000000", rendered(*column, 2)); +} + +// TIMESTAMPTZ shares the same helper and the same storage minimum as DATETIMEV2, so the reader +// must not be narrower than that type either. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, TimestampTzKeepsYearZero) { + const std::vector values {MIN_DORIS_TIMESTAMP_MICROS, utc_micros(0, 1, 1, 12, 34, 56), + utc_micros(0, 3, 1), MAX_DORIS_TIMESTAMP_MICROS}; + VectorDecodeSource source; + source.set_values(values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, values.size(), state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(values.size(), data.size()); + EXPECT_EQ(0, data[0].year()); + EXPECT_EQ(1, data[0].month()); + EXPECT_EQ(1, data[0].day()); + EXPECT_EQ(0, data[1].year()); + EXPECT_EQ(34, data[1].minute()); + EXPECT_EQ(0, data[2].year()); + EXPECT_EQ(3, data[2].month()); + EXPECT_EQ(9999, data[3].year()); + EXPECT_EQ(999999, data[3].microsecond()); +} + +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, TimestampTzStillRejectsUnrepresentableValues) { + const auto expect_rejected = [&](int64_t micros, const char* why) { + VectorDecodeSource source; + source.set_values(std::vector {micros}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + EXPECT_FALSE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()) + << why; + EXPECT_EQ(0, column->size()) << why; + }; + + expect_rejected(MIN_DORIS_TIMESTAMP_MICROS - 1, "one micro before 0000-01-01"); + expect_rejected(MAX_DORIS_TIMESTAMP_MICROS + 1, "one micro after 9999-12-31 23:59:59.999999"); + expect_rejected(utc_micros(10000, 1, 1), "year 10000"); +} + +// Predicate pushdown converts the values through a separate consumer before the filter runs, so a +// value the reader refuses is compared as a conversion failure instead of as itself and the row +// silently drops out of the result. Year zero has to reach the predicate as a real value. +TEST_F(DataTypeDateTimeV2SerDeCalendarTest, RawPredicateKeepsYearZero) { + class CapturingConsumer final : public ParquetLogicalValueConsumer { + public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width, + const uint8_t* conversion_nulls) override { + width = value_width; + bytes.assign(values, values + num_values * value_width); + nulls.clear(); + nulls.resize_fill(num_values, 0); + if (conversion_nulls != nullptr) { + memcpy(nulls.data(), conversion_nulls, num_values); + } + return Status::OK(); + } + + std::vector bytes; + IColumn::Filter nulls; + size_t width = 0; + }; + + const std::vector values {utc_micros(0, 1, 1, 12, 34, 56), utc_micros(0, 3, 1), + utc_micros(2024, 1, 1, 12)}; + const auto utc = cctz::utc_time_zone(); + for (bool adjusted_to_utc : {false, true}) { + VectorDecodeSource source; + source.set_values(values); + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = adjusted_to_utc, + .timezone = &utc}; + CapturingConsumer consumer; + DataTypeDateTimeV2 type(6); + ASSERT_TRUE(type.get_serde() + ->read_parquet_raw_predicate(source, context, values.size(), false, + consumer) + .ok()) + << "adjusted_to_utc=" << adjusted_to_utc; + EXPECT_EQ(consumer.nulls, IColumn::Filter(values.size(), 0)) + << "adjusted_to_utc=" << adjusted_to_utc; + ASSERT_EQ(sizeof(DateV2Value), consumer.width); + + auto column = ColumnDateTimeV2::create(); + column->get_data().resize(values.size()); + memcpy(column->get_data().data(), consumer.bytes.data(), consumer.bytes.size()); + EXPECT_EQ("0000-01-01 12:34:56.000000", rendered(*column, 0)); + EXPECT_EQ("0000-03-01 00:00:00.000000", rendered(*column, 1)); + EXPECT_EQ("2024-01-01 12:00:00.000000", rendered(*column, 2)); + } +} + +} // namespace doris diff --git a/be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp b/be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp new file mode 100644 index 00000000000000..a4aa6e5520f4fd --- /dev/null +++ b/be/test/core/data_type_serde/data_type_datev2_serde_calendar_test.cpp @@ -0,0 +1,328 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Doris DATE values are numbered in MySQL's calendar, where year 0 is not a leap year. Arrow +// `date32`, Parquet `DATE` and ORC `DATE` are all days since 1970-01-01 in the proleptic +// Gregorian calendar, where year 0 IS a leap year. These tests pin the boundary so the two +// numberings cannot drift apart again: 0000-01-01 must leave Doris as -719528, not -719527. + +#include +#include +#include + +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_array.h" +#include "core/column/column_nullable.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type_serde/data_type_datev2_serde.h" +#include "core/data_type_serde/decoded_column_view.h" +#include "core/value/vdatetime_value.h" + +namespace doris { + +namespace { + +struct DateCase { + int year; + int month; + int day; + int32_t epoch_days; +}; + +// 0000-01-01 .. 0000-02-28 are the only dates where Doris's daynr and the proleptic Gregorian +// ordinal disagree, so the set brackets that window on both sides. +const std::vector& boundary_cases() { + static const std::vector cases = { + {0, 1, 1, -719528}, {0, 2, 28, -719470}, {0, 3, 1, -719468}, {1, 1, 1, -719162}, + {1969, 12, 31, -1}, {1970, 1, 1, 0}, {2024, 1, 1, 19723}, {9999, 12, 31, 2932896}}; + return cases; +} + +DateV2Value make_date(int year, int month, int day) { + DateV2Value value; + value.unchecked_set_time(year, month, day, 0, 0, 0, 0); + return value; +} + +ColumnDateV2::MutablePtr boundary_column() { + auto column = ColumnDateV2::create(); + auto& data = column->get_data(); + for (const auto& c : boundary_cases()) { + auto value = make_date(c.year, c.month, c.day); + data.push_back(*reinterpret_cast(&value)); + } + return column; +} + +std::shared_ptr build_date32(const std::vector& days) { + arrow::Date32Builder builder; + for (int32_t d : days) { + EXPECT_TRUE(builder.Append(d).ok()); + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + return array; +} + +} // namespace + +class DataTypeDateV2SerDeCalendarTest : public ::testing::Test { +protected: + DataTypeDateV2SerDe serde; + cctz::time_zone tz = cctz::utc_time_zone(); +}; + +TEST_F(DataTypeDateV2SerDeCalendarTest, WriteArrowUsesProlepticGregorian) { + auto column = boundary_column(); + arrow::Date32Builder builder; + ASSERT_TRUE(serde.write_column_to_arrow(*column, nullptr, &builder, 0, + static_cast(column->size()), tz) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + const auto& date32 = assert_cast(*array); + ASSERT_EQ(boundary_cases().size(), static_cast(date32.length())); + for (size_t i = 0; i < boundary_cases().size(); ++i) { + EXPECT_EQ(boundary_cases()[i].epoch_days, date32.Value(static_cast(i))) + << "row " << i; + } +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, WriteArrowHonoursNullMap) { + auto column = boundary_column(); + NullMap null_map; + null_map.resize_fill(column->size(), 0); + null_map[0] = 1; // 0000-01-01 + null_map[2] = 1; // 0000-03-01 + + arrow::Date32Builder builder; + ASSERT_TRUE(serde.write_column_to_arrow(*column, &null_map, &builder, 0, + static_cast(column->size()), tz) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + const auto& date32 = assert_cast(*array); + for (size_t i = 0; i < boundary_cases().size(); ++i) { + if (null_map[i]) { + EXPECT_TRUE(date32.IsNull(static_cast(i))) << "row " << i; + } else { + EXPECT_EQ(boundary_cases()[i].epoch_days, date32.Value(static_cast(i))) + << "row " << i; + } + } +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDate32RestoresTheSameCalendarDay) { + std::vector days; + for (const auto& c : boundary_cases()) { + days.push_back(c.epoch_days); + } + auto array = build_date32(days); + + auto column = ColumnDateV2::create(); + ASSERT_TRUE(serde.read_column_from_arrow(*column, array.get(), 0, array->length(), tz).ok()); + + ASSERT_EQ(boundary_cases().size(), column->size()); + const auto& values = column->get_data(); + for (size_t i = 0; i < boundary_cases().size(); ++i) { + const auto& c = boundary_cases()[i]; + EXPECT_EQ(c.year, values[i].year()) << "row " << i; + EXPECT_EQ(c.month, values[i].month()) << "row " << i; + EXPECT_EQ(c.day, values[i].day()) << "row " << i; + } +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, ArrowRoundTripIsLossless) { + auto column = boundary_column(); + arrow::Date32Builder builder; + ASSERT_TRUE(serde.write_column_to_arrow(*column, nullptr, &builder, 0, + static_cast(column->size()), tz) + .ok()); + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + auto restored = ColumnDateV2::create(); + ASSERT_TRUE(serde.read_column_from_arrow(*restored, array.get(), 0, array->length(), tz).ok()); + ASSERT_EQ(column->size(), restored->size()); + for (size_t i = 0; i < column->size(); ++i) { + EXPECT_EQ(column->get_data()[i], restored->get_data()[i]) << "row " << i; + } +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDate32RejectsUnrepresentableDays) { + const auto expect_rejected = [&](int32_t days, const char* why) { + auto array = build_date32({days}); + auto column = ColumnDateV2::create(); + const auto status = serde.read_column_from_arrow(*column, array.get(), 0, 1, tz); + EXPECT_FALSE(status.ok()) << why << " (days=" << days << ")"; + EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris DATE range")) + << status.to_string(); + }; + + expect_rejected(-719529, "one day before 0000-01-01"); + // 0000-02-29 exists in the proleptic Gregorian calendar but Doris has no such date. The old + // `encoded + 719528` mapping decoded it as 0000-02-28; it must be rejected instead. + expect_rejected(-719469, "proleptic-only leap day 0000-02-29"); + expect_rejected(2932897, "one day after 9999-12-31"); +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDate64UsesTheSameCalendar) { + constexpr int64_t millis_per_day = 24LL * 60 * 60 * 1000; + arrow::Date64Builder builder; + for (const auto& c : boundary_cases()) { + ASSERT_TRUE(builder.Append(static_cast(c.epoch_days) * millis_per_day).ok()); + } + std::shared_ptr array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + auto column = ColumnDateV2::create(); + ASSERT_TRUE(serde.read_column_from_arrow(*column, array.get(), 0, array->length(), tz).ok()); + + ASSERT_EQ(boundary_cases().size(), column->size()); + const auto& values = column->get_data(); + for (size_t i = 0; i < boundary_cases().size(); ++i) { + const auto& c = boundary_cases()[i]; + EXPECT_EQ(c.year, values[i].year()) << "row " << i; + EXPECT_EQ(c.month, values[i].month()) << "row " << i; + EXPECT_EQ(c.day, values[i].day()) << "row " << i; + } +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, NestedArrayOfDateUsesTheSameCalendar) { + // ARRAY has no encoding of its own; it delegates every element to the DATE SerDe. The + // reported bug showed up through this path too, so keep it covered. + // DataTypeArray stores its element type as DataTypeNullablePtr, so the nested column has to be + // a ColumnNullable for the element SerDe to match. + auto nested = boundary_column(); + const auto element_count = nested->size(); + auto element_null_map = ColumnUInt8::create(); + element_null_map->get_data().resize_fill(element_count, 0); + auto nullable_nested = ColumnNullable::create(std::move(nested), std::move(element_null_map)); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->get_data().push_back(static_cast(element_count)); + auto array_column = ColumnArray::create(std::move(nullable_nested), std::move(offsets)); + + auto array_type = std::make_shared(std::make_shared()); + auto array_serde = array_type->get_serde(); + + auto value_builder = std::make_shared(); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); + ASSERT_TRUE(array_serde->write_column_to_arrow(*array_column, nullptr, &list_builder, 0, 1, tz) + .ok()); + std::shared_ptr arrow_array; + ASSERT_TRUE(list_builder.Finish(&arrow_array).ok()); + + const auto& list = assert_cast(*arrow_array); + const auto& values = assert_cast(*list.values()); + ASSERT_EQ(boundary_cases().size(), static_cast(values.length())); + for (size_t i = 0; i < boundary_cases().size(); ++i) { + EXPECT_EQ(boundary_cases()[i].epoch_days, values.Value(static_cast(i))) + << "element " << i; + } +} + +// The Parquet and ORC readers do not go through Arrow: rows, dictionary entries and column +// statistics all land in `read_column_from_decoded_values()`. Cover that entry point with the same +// boundary set, in both the strict and the null-on-failure mode the file scanners use. +TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDecodedValuesUsesTheSameCalendar) { + std::vector values; + for (const auto& c : boundary_cases()) { + values.push_back(c.epoch_days); + } + values.push_back(0); // payload of the null row below, never decoded + std::vector null_map(values.size(), 0); + null_map.back() = 1; + + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = static_cast(values.size()); + view.values = reinterpret_cast(values.data()); + view.null_map = null_map.data(); + + auto column = ColumnDateV2::create(); + ASSERT_TRUE(serde.read_column_from_decoded_values(*column, view).ok()); + ASSERT_EQ(values.size(), column->size()); + const auto& data = column->get_data(); + for (size_t i = 0; i < boundary_cases().size(); ++i) { + const auto& c = boundary_cases()[i]; + EXPECT_EQ(c.year, data[i].year()) << "row " << i; + EXPECT_EQ(c.month, data[i].month()) << "row " << i; + EXPECT_EQ(c.day, data[i].day()) << "row " << i; + } +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDecodedValuesRejectsUnrepresentableDaysWhenStrict) { + // -719469 is the proleptic-only 0000-02-29: the only value inside the file-format range that + // Doris cannot represent, and the one the old dictionary fallback decoded as 0000-02-28. + const std::vector values = {-719528, -719470, -719469, -719468, 19723}; + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = static_cast(values.size()); + view.values = reinterpret_cast(values.data()); + view.enable_strict_mode = true; + + auto column = ColumnDateV2::create(); + const auto status = serde.read_column_from_decoded_values(*column, view); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris DATE range")) + << status.to_string(); + EXPECT_NE(std::string::npos, status.to_string().find("-719469")) << status.to_string(); + // A failed batch must not leave a half-written column behind. + EXPECT_EQ(0, column->size()); +} + +TEST_F(DataTypeDateV2SerDeCalendarTest, ReadDecodedValuesNullsOnlyTheUnrepresentableRow) { + const std::vector values = {-719528, -719470, -719469, -719468, 19723}; + NullMap conversion_failures; + conversion_failures.resize_fill(values.size(), 0); + + DecodedColumnView view; + view.value_kind = DecodedValueKind::INT32; + view.row_count = static_cast(values.size()); + view.values = reinterpret_cast(values.data()); + view.conversion_failure_null_map = &conversion_failures; + + auto column = ColumnDateV2::create(); + ASSERT_TRUE(serde.read_column_from_decoded_values(*column, view).ok()); + ASSERT_EQ(values.size(), column->size()); + const std::vector expected_failures = {0, 0, 1, 0, 0}; + for (size_t i = 0; i < expected_failures.size(); ++i) { + EXPECT_EQ(expected_failures[i], conversion_failures[i]) << "row " << i; + } + const auto& data = column->get_data(); + EXPECT_EQ(0, data[0].year()); + EXPECT_EQ(1, data[0].month()); + EXPECT_EQ(1, data[0].day()); + EXPECT_EQ(0, data[1].year()); + EXPECT_EQ(2, data[1].month()); + EXPECT_EQ(28, data[1].day()); + EXPECT_EQ(0, data[3].year()); + EXPECT_EQ(3, data[3].month()); + EXPECT_EQ(1, data[3].day()); + EXPECT_EQ(2024, data[4].year()); +} + +} // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp index b7b664c7c78de2..f892cdf6074a44 100644 --- a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -878,13 +878,16 @@ TEST(DataTypeSerDeParquetTest, Int96DictionaryFailuresFollowDecodedIds) { } TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { - constexpr int64_t MIN_TIMESTAMP_MICROS = -62135596800000000LL; + // TIMESTAMPTZ shares DATETIMEV2's storage, so its floor is 0000-01-01, not 0001-01-01. Year + // zero itself is covered in data_type_datetimev2_serde_calendar_test.cpp. + constexpr int64_t YEAR_ZERO_MICROS = -62167219200000000LL; + constexpr int64_t YEAR_ONE_MICROS = -62135596800000000LL; constexpr int64_t MAX_TIMESTAMP_MICROS = 253402300799999999LL; constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; { TestParquetDecodeSource source; - source.set_fixed_values({MIN_TIMESTAMP_MICROS, MAX_TIMESTAMP_MICROS}); + source.set_fixed_values({YEAR_ZERO_MICROS, YEAR_ONE_MICROS, MAX_TIMESTAMP_MICROS}); ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, .logical_type = ParquetLogicalType::TIMESTAMP, .time_unit = ParquetTimeUnit::MICROS, @@ -894,12 +897,15 @@ TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { auto column = type.create_column(); ASSERT_TRUE(type.get_serde() - ->read_column_from_parquet(*column, source, context, 2, state) + ->read_column_from_parquet(*column, source, context, 3, state) .ok()); const auto& data = assert_cast(*column).get_data(); - EXPECT_EQ(data[0].year(), 1); - EXPECT_EQ(data[1].year(), 9999); - EXPECT_EQ(data[1].microsecond(), 999999); + EXPECT_EQ(data[0].year(), 0); + EXPECT_EQ(data[0].month(), 1); + EXPECT_EQ(data[0].day(), 1); + EXPECT_EQ(data[1].year(), 1); + EXPECT_EQ(data[2].year(), 9999); + EXPECT_EQ(data[2].microsecond(), 999999); } { TestParquetDecodeSource source; diff --git a/be/test/core/value/vdatetime_value_test.cpp b/be/test/core/value/vdatetime_value_test.cpp index 6b3cd7488ae6b2..8522432679e562 100644 --- a/be/test/core/value/vdatetime_value_test.cpp +++ b/be/test/core/value/vdatetime_value_test.cpp @@ -17,10 +17,15 @@ #include "core/value/vdatetime_value.h" +#include +#include +#include #include #include +#include #include +#include #include "common/exception.h" #include "core/data_type_serde/datelike_serde_common.hpp" @@ -1587,4 +1592,105 @@ TEST(VDateTimeValueTest, date_add_days_matches_date_add_interval) { EXPECT_GT(compared, 900000); } +// `daynr_to_epoch_days()` / `epoch_days_to_daynr()` bridge Doris's MySQL calendar and the +// proleptic Gregorian calendar that Arrow date32, Parquet/ORC DATE and Iceberg are defined in. +// cctz is the oracle here: cctz::civil_day is proleptic Gregorian, so its epoch-day offset is +// exactly what those formats expect. +TEST(VDateTimeValueTest, epoch_days_conversion_boundaries) { + struct Case { + int year; + int month; + int day; + int64_t daynr; + int32_t epoch_days; + }; + // 0000-01-01 .. 0000-02-28 are the only dates where the two calendars disagree: Doris has no + // 0000-02-29, so its day numbers run one ahead until 0000-03-01. + const std::vector cases = { + {0, 1, 1, 1, -719528}, {0, 1, 31, 31, -719498}, + {0, 2, 28, 59, -719470}, {0, 3, 1, 60, -719468}, + {1, 1, 1, 366, -719162}, {1899, 12, 31, 693960, -25568}, + {1900, 1, 1, 693961, -25567}, {1969, 12, 31, 719527, -1}, + {1970, 1, 1, 719528, 0}, {2024, 1, 1, 739251, 19723}, + {9999, 12, 31, 3652424, 2932896}}; + + for (const auto& c : cases) { + const int64_t daynr = calc_daynr(c.year, c.month, c.day); + EXPECT_EQ(c.daynr, daynr) << c.year << "-" << c.month << "-" << c.day; + EXPECT_EQ(c.epoch_days, daynr_to_epoch_days(daynr)) + << c.year << "-" << c.month << "-" << c.day; + EXPECT_EQ(daynr, epoch_days_to_daynr(c.epoch_days)) + << c.year << "-" << c.month << "-" << c.day; + } +} + +TEST(VDateTimeValueTest, epoch_days_conversion_rejects_unrepresentable) { + // Before 0000-01-01. + EXPECT_EQ(0, epoch_days_to_daynr(-719529)); + EXPECT_EQ(0, epoch_days_to_daynr(std::numeric_limits::min())); + // 0000-02-29 exists in the proleptic Gregorian calendar but not in Doris. + EXPECT_EQ(0, epoch_days_to_daynr(-719469)); + // After 9999-12-31. + EXPECT_EQ(0, epoch_days_to_daynr(2932897)); + EXPECT_EQ(0, epoch_days_to_daynr(std::numeric_limits::max())); + // The two days on either side of the proleptic-only leap day still convert. + EXPECT_EQ(59, epoch_days_to_daynr(-719470)); + EXPECT_EQ(60, epoch_days_to_daynr(-719468)); +} + +TEST(VDateTimeValueTest, epoch_days_conversion_matches_cctz_over_full_range) { + const cctz::time_zone utc = cctz::utc_time_zone(); + int64_t mismatches = 0; + int64_t round_trip_failures = 0; + int64_t decode_failures = 0; + for (int year = 0; year <= 9999; ++year) { + for (int month = 1; month <= 12; ++month) { + const int days_in_month = + S_DAYS_IN_MONTH[month] + ((month == 2 && is_leap(year)) ? 1 : 0); + for (int day = 1; day <= days_in_month; ++day) { + const int64_t daynr = calc_daynr(year, month, day); + const int32_t epoch_days = daynr_to_epoch_days(daynr); + const int64_t expected = cctz::convert(cctz::civil_day(year, month, day), utc) + .time_since_epoch() + .count() / + (24 * 60 * 60); + if (epoch_days != expected) { + if (++mismatches <= 5) { + ADD_FAILURE() << fmt::format("{:04d}-{:02d}-{:02d}: got {}, cctz says {}", + year, month, day, epoch_days, expected); + } + } + if (epoch_days_to_daynr(epoch_days) != daynr) { + if (++round_trip_failures <= 5) { + ADD_FAILURE() + << fmt::format("{:04d}-{:02d}-{:02d}: round trip lost daynr {}", + year, month, day, daynr); + } + } + // The ordinal is only half the contract: readers hand the recovered daynr to + // `get_date_from_daynr()`, which takes a non-dictionary path for daynr 1..59 and + // for everything outside 1900..2039. Close the loop on the civil date itself. + DateV2Value restored; + if (!restored.get_date_from_daynr( + static_cast(epoch_days_to_daynr(epoch_days)))) { + if (++decode_failures <= 5) { + ADD_FAILURE() << fmt::format("{:04d}-{:02d}-{:02d}: daynr {} not decodable", + year, month, day, daynr); + } + } else if (restored.year() != year || restored.month() != month || + restored.day() != day) { + if (++decode_failures <= 5) { + ADD_FAILURE() << fmt::format( + "{:04d}-{:02d}-{:02d}: decoded back as {:04d}-{:02d}-{:02d}", year, + month, day, restored.year(), restored.month(), restored.day()); + } + } + } + } + } + EXPECT_EQ(0, mismatches); + EXPECT_EQ(0, round_trip_failures); + EXPECT_EQ(0, decode_failures); +} + } // namespace doris diff --git a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp index 974eb817e8872a..1fd947b1189989 100644 --- a/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp +++ b/be/test/exec/sink/writer/iceberg/partition_transformers_test.cpp @@ -22,6 +22,7 @@ #include #include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" namespace doris { @@ -546,4 +547,326 @@ TEST_F(PartitionTransformersTest, test_nullable_column_string_truncate_transform EXPECT_EQ("db", result_strings->get_data_at(2).to_string()); } +// The expected values below were produced with the Apache Iceberg reference implementation +// (iceberg-api 1.10.1, the version fe/pom.xml depends on) via DateTimeUtil / Transforms / +// BucketUtil. They pin two spec requirements that Doris used to violate: +// * the day ordinal is proleptic Gregorian, in which year 0 IS a leap year, so 0000-01-01 is +// -719528 and not the -719527 that Doris's MySQL-calendar `daynr()` implies; +// * `day` and `hour` floor towards negative infinity rather than truncating towards the epoch. +namespace { + +ColumnWithTypeAndName make_date_column(const std::vector>& dates, + ColumnDateV2::MutablePtr& column) { + auto& data = column->get_data(); + for (const auto& [y, m, d] : dates) { + DateV2Value value; + value.unchecked_set_time(y, m, d, 0, 0, 0, 0); + data.push_back(*reinterpret_cast(&value)); + } + return {column->get_ptr(), std::make_shared(), "test_date"}; +} + +// The microsecond field is part of the tuple on purpose: Iceberg's timestamp transforms are +// defined on the full microsecond value, so a test that only ever passes 0 cannot tell flooring +// from truncation, nor a bucket that keeps the sub-second part from one that drops it. +ColumnWithTypeAndName make_timestamp_column( + const std::vector>& timestamps, + ColumnDateTimeV2::MutablePtr& column) { + auto& data = column->get_data(); + for (const auto& [y, mo, d, h, mi, se, us] : timestamps) { + DateV2Value value; + value.unchecked_set_time(y, mo, d, h, mi, se, us); + data.push_back(*reinterpret_cast(&value)); + } + return {column->get_ptr(), std::make_shared(), "test_timestamp"}; +} + +} // namespace + +TEST_F(PartitionTransformersTest, test_date_day_transform_proleptic_gregorian) { + auto column = ColumnDateV2::create(); + auto test_date = make_date_column({{0, 1, 1}, + {0, 2, 28}, + {0, 3, 1}, + {1, 1, 1}, + {1969, 6, 15}, + {1969, 12, 31}, + {1970, 1, 1}, + {2017, 11, 16}, + {9999, 12, 31}}, + column); + + Block block({test_date}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATEV2, false); + DateDayPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {-719528, -719470, -719468, -719162, -200, + -1, 0, 17486, 2932896}; + std::vector expected_human_string = {"0000-01-01", "0000-02-28", "0000-03-01", + "0001-01-01", "1969-06-15", "1969-12-31", + "1970-01-01", "2017-11-16", "9999-12-31"}; + ASSERT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]) << "row " << i; + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])); + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_day_transform_floors_before_epoch) { + auto column = ColumnDateTimeV2::create(); + auto test_timestamp = make_timestamp_column({{0, 1, 1, 0, 0, 0, 0}, + {0, 1, 1, 12, 34, 56, 0}, + {0, 2, 28, 0, 0, 0, 0}, + {0, 2, 28, 23, 59, 59, 999999}, + {1969, 12, 31, 0, 0, 0, 0}, + {1969, 12, 31, 12, 0, 0, 0}, + {1969, 12, 31, 23, 59, 59, 0}, + {1969, 12, 31, 23, 59, 59, 999999}, + {1970, 1, 1, 0, 0, 0, 0}, + {2017, 11, 16, 22, 31, 8, 0}}, + column); + + Block block({test_timestamp}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, false); + TimestampDayPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + // Every 1969-12-31 timestamp must land on -1: Iceberg floors, it does not round towards the + // epoch the way SQL DATEDIFF does. The last microsecond of a day belongs to that same day. + std::vector expected_data = {-719528, -719528, -719470, -719470, -1, + -1, -1, -1, 0, 17486}; + ASSERT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]) << "row " << i; + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_hour_transform_floors_before_epoch) { + auto column = ColumnDateTimeV2::create(); + auto test_timestamp = make_timestamp_column({{0, 1, 1, 0, 0, 0, 0}, + {0, 1, 1, 12, 34, 56, 0}, + {0, 2, 28, 0, 0, 0, 0}, + {0, 2, 28, 23, 59, 59, 999999}, + {1, 1, 1, 0, 0, 0, 0}, + {1969, 6, 15, 10, 0, 0, 0}, + {1969, 12, 31, 0, 0, 0, 0}, + {1969, 12, 31, 12, 0, 0, 0}, + {1969, 12, 31, 23, 30, 0, 0}, + {1969, 12, 31, 23, 59, 59, 999999}, + {1970, 1, 1, 0, 0, 0, 0}, + {1970, 1, 1, 12, 0, 0, 0}, + {2017, 11, 16, 22, 31, 8, 0}, + {9999, 12, 31, 23, 59, 59, 999999}}, + column); + + Block block({test_timestamp}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, false); + TimestampHourPartitionColumnTransform transform(source_type); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + std::vector expected_data = {-17268672, -17268660, -17267280, -17267257, -17259888, + -4790, -24, -12, -1, -1, + 0, 12, 419686, 70389527}; + // The partition path must floor as well: hour ordinal -1 is 1969-12-31-23. + std::vector expected_human_string = { + "0000-01-01-00", "0000-01-01-12", "0000-02-28-00", "0000-02-28-23", "0001-01-01-00", + "1969-06-15-10", "1969-12-31-00", "1969-12-31-12", "1969-12-31-23", "1969-12-31-23", + "1970-01-01-00", "1970-01-01-12", "2017-11-16-22", "9999-12-31-23"}; + ASSERT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]) << "row " << i; + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), result_data[i])) + << "row " << i; + } +} + +TEST_F(PartitionTransformersTest, test_date_bucket_transform_year_zero) { + auto column = ColumnDateV2::create(); + auto test_date = make_date_column({{0, 1, 1}, {0, 2, 28}, {0, 3, 1}, {2017, 11, 16}}, column); + + Block block({test_date}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATEV2, false); + DateBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + // Buckets for the proleptic day ordinals -719528, -719470, -719468 and 17486. Hashing the + // Doris daynr instead would put 0000-01-01 in bucket 6. + std::vector expected_data = {1, 0, 7, 10}; + ASSERT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]) << "row " << i; + } +} + +TEST_F(PartitionTransformersTest, test_date_year_month_transform_floors_before_epoch) { + auto column = ColumnDateV2::create(); + auto test_date = make_date_column({{0, 1, 1}, + {0, 2, 28}, + {0, 3, 1}, + {1, 1, 1}, + {1899, 12, 31}, + {1969, 6, 15}, + {1969, 12, 31}, + {1970, 1, 1}, + {2024, 2, 29}, + {9999, 12, 31}}, + column); + Block block({test_date}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATEV2, false); + + { + DateYearPartitionColumnTransform transform(source_type); + auto result = transform.apply(block, 0); + const auto& data = assert_cast(result.column.get())->get_data(); + // Whole calendar years from 1970, floored. Rounding towards zero would report 0 for + // 1969-06-15 and -1969 for 0000-02-28. + std::vector expected = {-1970, -1970, -1970, -1969, -71, -1, -1, 0, 54, 8029}; + // iceberg-api's TransformUtil.humanYear zero-pads to four digits, so the partition + // directory of a year-zero row is `..._year=0000`, not `..._year=0`. + std::vector expected_human_string = {"0000", "0000", "0000", "0001", "1899", + "1969", "1969", "1970", "2024", "9999"}; + ASSERT_EQ(expected.size(), data.size()); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(expected[i], data[i]) << "row " << i; + EXPECT_EQ(expected_human_string[i], + transform.to_human_string(transform.get_result_type(), data[i])) + << "row " << i; + } + } + { + DateMonthPartitionColumnTransform transform(source_type); + auto result = transform.apply(block, 0); + const auto& data = assert_cast(result.column.get())->get_data(); + std::vector expected = {-23640, -23639, -23638, -23628, -841, + -7, -1, 0, 649, 96359}; + ASSERT_EQ(expected.size(), data.size()); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(expected[i], data[i]) << "row " << i; + } + } +} + +TEST_F(PartitionTransformersTest, test_timestamp_year_month_transform_floors_before_epoch) { + auto column = ColumnDateTimeV2::create(); + auto test_timestamp = make_timestamp_column({{0, 1, 1, 12, 34, 56, 0}, + {0, 2, 28, 0, 0, 0, 0}, + {1, 1, 1, 0, 0, 0, 0}, + {1969, 6, 15, 10, 0, 0, 0}, + {1969, 12, 31, 23, 59, 59, 999999}, + {1970, 1, 1, 0, 0, 0, 0}, + {2024, 1, 1, 12, 0, 0, 0}, + {9999, 12, 31, 23, 59, 59, 999999}}, + column); + Block block({test_timestamp}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, false); + + { + TimestampYearPartitionColumnTransform transform(source_type); + auto result = transform.apply(block, 0); + const auto& data = assert_cast(result.column.get())->get_data(); + std::vector expected = {-1970, -1970, -1969, -1, -1, 0, 54, 8029}; + ASSERT_EQ(expected.size(), data.size()); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(expected[i], data[i]) << "row " << i; + } + } + { + TimestampMonthPartitionColumnTransform transform(source_type); + auto result = transform.apply(block, 0); + const auto& data = assert_cast(result.column.get())->get_data(); + std::vector expected = {-23640, -23639, -23628, -7, -1, 0, 648, 96359}; + ASSERT_EQ(expected.size(), data.size()); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(expected[i], data[i]) << "row " << i; + } + } +} + +// The exact row the iceberg write regression suite stores: before the fix, 1969-12-31 23:59:59 and +// 1970-01-01 00:00:00 collapsed into the same day and hour partition. +TEST_F(PartitionTransformersTest, test_epoch_boundary_rows_land_in_distinct_partitions) { + auto column = ColumnDateTimeV2::create(); + auto test_timestamp = make_timestamp_column({{1969, 12, 31, 23, 59, 59, 999999}, + {1970, 1, 1, 0, 0, 0, 0}, + {2024, 2, 29, 12, 34, 56, 123456}}, + column); + Block block({test_timestamp}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, false); + + TimestampDayPartitionColumnTransform day_transform(source_type); + // Keep the result alive: it owns the ColumnPtr the data reference points into. + auto day_result = day_transform.apply(block, 0); + const auto& days = assert_cast(day_result.column.get())->get_data(); + EXPECT_EQ(-1, days[0]); + EXPECT_EQ(0, days[1]); + EXPECT_EQ(19782, days[2]); + EXPECT_NE(days[0], days[1]) << "before-epoch and epoch rows must not share a day partition"; + EXPECT_EQ("1969-12-31", + day_transform.to_human_string(day_transform.get_result_type(), days[0])); + + TimestampHourPartitionColumnTransform hour_transform(source_type); + auto hour_result = hour_transform.apply(block, 0); + const auto& hours = assert_cast(hour_result.column.get())->get_data(); + EXPECT_EQ(-1, hours[0]); + EXPECT_EQ(0, hours[1]); + EXPECT_EQ(474780, hours[2]); + EXPECT_NE(hours[0], hours[1]) << "before-epoch and epoch rows must not share an hour partition"; + EXPECT_EQ("1969-12-31-23", + hour_transform.to_human_string(hour_transform.get_result_type(), hours[0])); +} + +// Iceberg buckets a timestamp by hashing its full microsecond value (spec: Partition Transforms; +// iceberg-api 1.10.1 `Bucket.BucketLong` over `BucketUtil.hash(long)`). Doris used to hash whole +// seconds times a million, so a DATETIME(6) row landed in a different bucket than the same row +// written by Spark, and bucket pruning on it skipped the Doris-written file. +TEST_F(PartitionTransformersTest, test_timestamp_bucket_transform_keeps_microseconds) { + auto column = ColumnDateTimeV2::create(); + auto test_timestamp = make_timestamp_column({{2024, 2, 29, 12, 34, 56, 123456}, + {2024, 2, 29, 12, 34, 56, 0}, + {1969, 12, 31, 23, 59, 59, 999999}, + {1969, 12, 31, 23, 59, 59, 0}, + {0, 1, 1, 12, 34, 56, 654321}, + {1970, 1, 1, 0, 0, 0, 0}}, + column); + + Block block({test_timestamp}); + auto source_type = + DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_DATETIMEV2, false); + TimestampBucketPartitionColumnTransform transform(source_type, 16); + + auto result = transform.apply(block, 0); + + const auto& result_data = assert_cast(result.column.get())->get_data(); + // Buckets of the micros-since-epoch values 1709210096123456, 1709210096000000, -1, -1000000, + // -62167173903345679 and 0. Truncating to whole seconds would report 12, 12, 15, 15, 8 and 12, + // i.e. rows 0, 2 and 4 would collide with (or move onto) their truncated twins. + std::vector expected_data = {8, 12, 8, 15, 12, 12}; + ASSERT_EQ(expected_data.size(), result_data.size()); + for (size_t i = 0; i < result_data.size(); ++i) { + EXPECT_EQ(expected_data[i], result_data[i]) << "row " << i; + } + EXPECT_NE(result_data[0], result_data[1]) + << "a sub-second timestamp must not share a bucket with its truncated value"; + EXPECT_NE(result_data[2], result_data[3]) + << "a sub-second timestamp must not share a bucket with its truncated value"; +} + } // namespace doris diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 1058d774d44e60..b66a038712bc22 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -149,8 +149,9 @@ DateV2Value make_date_v2(uint16_t year, uint8_t month, uint8_t } int64_t orc_date_offset(uint16_t year, uint8_t month, uint8_t day) { - static constexpr int32_t DATE_THRESHOLD = 719528; - return make_date_v2(year, month, day).daynr() - DATE_THRESHOLD; + // ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, which is not the + // same as Doris's daynr minus the epoch daynr for year-zero dates. + return daynr_to_epoch_days(make_date_v2(year, month, day).daynr()); } DateV2Value make_datetime_v2(uint16_t year, uint8_t month, uint8_t day, @@ -3338,7 +3339,6 @@ void write_complex_orc_file(const std::string& file_path) { void write_map_decimal_date_orc_file(const std::string& file_path) { constexpr size_t ROWS = 4; - constexpr int64_t HIVE_012_1900_DAY_OFFSET = -719530; constexpr int64_t YEAR_0000_12_29_DAY_OFFSET = -719165; constexpr int64_t YEAR_1000_10_16_DAY_OFFSET = -353997; @@ -3370,7 +3370,9 @@ void write_map_decimal_date_orc_file(const std::string& file_path) { key_batch.values[1] = 9999999999L; key_batch.values[2] = 0; key_batch.values[3] = 1; - value_batch.data[0] = HIVE_012_1900_DAY_OFFSET; + // The smallest DATE Doris can represent: -719528 in the proleptic Gregorian calendar the ORC + // spec defines DATE in, one day below what Doris's own MySQL-calendar daynr would suggest. + value_batch.data[0] = orc_date_offset(0, 1, 1); value_batch.data[1] = orc_date_offset(9999, 12, 31); value_batch.data[2] = YEAR_0000_12_29_DAY_OFFSET; value_batch.data[3] = YEAR_1000_10_16_DAY_OFFSET; @@ -3388,6 +3390,42 @@ void write_map_decimal_date_orc_file(const std::string& file_path) { out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } +// A flat `struct` file whose DATE ordinals are written verbatim, so a test can +// place a value that no Doris DATE maps to (the proleptic-only 0000-02-29, or an ordinal outside +// the type's range) next to representable ones. +void write_date_orc_file(const std::string& file_path, + const std::vector>& day_offsets) { + auto type = + std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString("struct")); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(day_offsets.size()); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& id_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + auto& date_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[1]); + + date_batch.hasNulls = true; + for (size_t row = 0; row < day_offsets.size(); ++row) { + id_batch.data[row] = static_cast(row) + 1; + date_batch.notNull[row] = day_offsets[row].has_value() ? 1 : 0; + // Deliberately garbage under a null slot: a decoder must never look at it. + date_batch.data[row] = day_offsets[row].value_or(std::numeric_limits::min()); + } + + struct_batch.numElements = day_offsets.size(); + id_batch.numElements = day_offsets.size(); + date_batch.numElements = day_offsets.size(); + writer->add(*batch); + writer->close(); + + std::ofstream out(file_path, std::ios::binary); + out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + void write_two_stripe_orc_array_map_file(const std::string& file_path) { auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( "struct,map_col:map,payload:string>")); @@ -10864,12 +10902,140 @@ TEST_F(NewOrcReaderTest, ReadMapDecimalDateWithCenturyBoundary) { const auto& map_column = assert_cast(map_nullable.get_nested_column()); ASSERT_EQ(map_column.get_offsets().size(), 4); ASSERT_EQ(map_column.get_values().size(), 4); - EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 0), "1900-01-01"); + EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 0), "0000-01-01"); EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 1), "9999-12-31"); EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 2), "0000-12-29"); EXPECT_EQ(schema[1].children[1].type->to_string(map_column.get_values(), 3), "1000-10-16"); } +// ORC DATE is a proleptic Gregorian day ordinal, so year zero is the window where Doris's own +// MySQL-calendar day number disagrees with the file. Pin the decode there, including a null row +// whose payload must never be looked at. +TEST_F(NewOrcReaderTest, ReadDateYearZeroWindow) { + const auto file_path = (_test_dir / "date_year_zero.orc").string(); + write_date_orc_file(file_path, {orc_date_offset(0, 1, 1), orc_date_offset(0, 2, 28), + orc_date_offset(0, 3, 1), std::nullopt, + orc_date_offset(1970, 1, 1), orc_date_offset(2024, 1, 1)}); + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 6); + + const auto& nullable = assert_cast(*block.get_by_position(1).column); + const std::vector expected = {"0000-01-01", "0000-02-28", "0000-03-01", + "", "1970-01-01", "2024-01-01"}; + for (size_t row = 0; row < expected.size(); ++row) { + if (expected[row].empty()) { + EXPECT_TRUE(nullable.is_null_at(row)) << "row " << row; + continue; + } + ASSERT_FALSE(nullable.is_null_at(row)) << "row " << row; + EXPECT_EQ(expected[row], schema[1].type->to_string(nullable, row)) << "row " << row; + } +} + +TEST_F(NewOrcReaderTest, ReadDateRejectsUnrepresentableOrdinals) { + // -719469 is 0000-02-29, which exists in the proleptic Gregorian calendar but not in Doris; + // -719530 is below 0000-01-01. Both used to decode as 1900-01-01 through the day dictionary. + for (const int64_t bad_offset : {int64_t {-719469}, int64_t {-719530}, int64_t {2932897}}) { + const auto file_path = (_test_dir / fmt::format("date_bad_{}.orc", bad_offset)).string(); + write_date_orc_file(file_path, {orc_date_offset(2024, 1, 1), bad_offset}); + auto reader = create_reader_for_path(file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_FALSE(status.ok()) << "offset " << bad_offset; + // The message must name the column and the offending value, and must not claim a format + // it did not come from: the helper is shared with Parquet. + EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris DATE range")) + << status.to_string(); + EXPECT_NE(std::string::npos, status.to_string().find(std::to_string(bad_offset))) + << status.to_string(); + EXPECT_NE(std::string::npos, status.to_string().find("'d'")) << status.to_string(); + EXPECT_EQ(std::string::npos, status.to_string().find("Parquet")) << status.to_string(); + } +} + +// A pushed-down MIN/MAX is answered from stripe statistics without reading a row, so the +// statistics have to be decoded with exactly the same calendar as the rows. Decoding them through +// the day dictionary used to report 1900-01-01 as the minimum of a file whose smallest row is +// 0000-01-01 -- a value present in no row at all. +TEST_F(NewOrcReaderTest, AggregatePushdownDateMinMaxAgreesWithRowDecode) { + const auto file_path = (_test_dir / "date_year_zero_minmax.orc").string(); + write_date_orc_file(file_path, {orc_date_offset(0, 1, 1), orc_date_offset(0, 2, 28), + std::nullopt, orc_date_offset(2024, 1, 1)}); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto reader = create_reader_for_path(file_path); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(1))}); + format::FileAggregateResult aggregate_result; + const auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(aggregate_result.columns.size(), 1); + EXPECT_EQ(aggregate_result.count, 4); + ASSERT_TRUE(aggregate_result.columns[0].has_min); + ASSERT_TRUE(aggregate_result.columns[0].has_max); + EXPECT_EQ(aggregate_result.columns[0].min_value.get(), make_date_v2(0, 1, 1)); + EXPECT_EQ(aggregate_result.columns[0].max_value.get(), make_date_v2(2024, 1, 1)); +} + +TEST_F(NewOrcReaderTest, AggregatePushdownDateMinMaxFallsBackForUnrepresentableBound) { + const auto file_path = (_test_dir / "date_minmax_fallback.orc").string(); + // The minimum is the proleptic-only 0000-02-29: no Doris DATE bounds this file, so the + // statistics must be refused rather than silently rounded onto a neighbouring day. + write_date_orc_file(file_path, {std::optional {-719469}, orc_date_offset(2024, 1, 1)}); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto reader = create_reader_for_path(file_path); + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + format::FileAggregateRequest aggregate_request; + aggregate_request.agg_type = TPushAggOp::type::MINMAX; + aggregate_request.columns.push_back( + {.projection = format::LocalColumnIndex::top_level(format::LocalColumnId(1))}); + format::FileAggregateResult aggregate_result; + const auto status = reader->get_aggregate_result(aggregate_request, &aggregate_result); + EXPECT_TRUE(status.is()) << status; +} + TEST_F(NewOrcReaderTest, ReadDeepNestedComplexTypes) { const auto complex_file_path = (_test_dir / "deep_complex.orc").string(); write_deep_nested_complex_orc_file(complex_file_path); diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index a5ff4562e915cd..f5d91cec8d504f 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -507,8 +507,11 @@ static List parsePartitionValuesFromJson(String partitionDataJson) { // Master IcebergUtils.UNKNOWN_SNAPSHOT_ID: an empty table / a null last_updated_snapshot_id row. private static final long UNKNOWN_SNAPSHOT_ID = -1; - private static final DateTimeFormatter RANGE_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - private static final DateTimeFormatter RANGE_DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + // Pattern letter `u` (proleptic year), not `y` (year-of-era): Iceberg day/hour ordinals are + // proleptic Gregorian, so a year-zero partition bound is 0000-01-01. `yyyy` would render it as + // 0001-01-01 (year 1 of the BCE era) and collide with the real 0001-01-01 partition. + private static final DateTimeFormatter RANGE_DATE_FORMAT = DateTimeFormatter.ofPattern("uuuu-MM-dd"); + private static final DateTimeFormatter RANGE_DATETIME_FORMAT = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss"); // Sort by partition-range LOW ascending; ties broken by HIGH descending (larger range first), so an // enclosing partition precedes the ones it encloses. Parity with master IcebergUtils.RangeComparator. diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionUtilsTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionUtilsTest.java index 6afffa80d48f1c..a02262f7699b27 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionUtilsTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionUtilsTest.java @@ -587,6 +587,31 @@ public void buildRangeYearTruncatesToYearBoundary() { Assertions.assertEquals(Collections.singletonList("1973-01-01"), rb.getUpperBound()); } + @Test + public void buildRangeYearZeroRendersProlepticYear() { + // day ordinal -719528 is 0000-01-01: Iceberg ordinals are proleptic Gregorian, where year 0 + // exists. MUTATION: a "yyyy" (year-of-era) pattern renders 0001-01-01 here, colliding with + // the range of the real 0001-01-01 partition (ordinal -719162). + IcebergPartitionUtils.RangeBuild rb = IcebergPartitionUtils.buildRange( + "d_day=-719528", "-719528", "day", Types.DateType.get(), 1L, 1L); + Assertions.assertEquals(Collections.singletonList("0000-01-01"), rb.getLowerBound()); + Assertions.assertEquals(Collections.singletonList("0000-01-02"), rb.getUpperBound()); + + IcebergPartitionUtils.RangeBuild year1 = IcebergPartitionUtils.buildRange( + "d_day=-719162", "-719162", "day", Types.DateType.get(), 1L, 1L); + Assertions.assertEquals(Collections.singletonList("0001-01-01"), year1.getLowerBound()); + Assertions.assertNotEquals(rb.getLowerBound(), year1.getLowerBound()); + } + + @Test + public void buildRangeYearZeroHourRendersProlepticYear() { + // hour ordinal -17268672 is 0000-01-01 00:00:00 (-719528 * 24). + IcebergPartitionUtils.RangeBuild rb = IcebergPartitionUtils.buildRange( + "ts_hour=-17268672", "-17268672", "hour", Types.TimestampType.withoutZone(), 1L, 1L); + Assertions.assertEquals(Collections.singletonList("0000-01-01 00:00:00"), rb.getLowerBound()); + Assertions.assertEquals(Collections.singletonList("0000-01-01 01:00:00"), rb.getUpperBound()); + } + @Test public void buildRangeNullValueEmitsSuccessorSignal() { // A NULL partition value -> lower "0000-01-01" + EMPTY upper (the generic model derives lower.successor()). diff --git a/regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out b/regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out new file mode 100644 index 00000000000000..f338898c9cb473 --- /dev/null +++ b/regression-test/data/arrow_flight_sql_p0/test_date_year_zero.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !source -- +1 0000-01-01 0000-01-01 +2 0000-02-28 \N +3 0000-03-01 0000-03-01 +4 0001-01-01 0001-01-01 +5 1969-12-31 1969-12-31 +6 1970-01-01 1970-01-01 +7 2024-01-01 2024-01-01 +8 9999-12-31 9999-12-31 + diff --git a/regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out b/regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out new file mode 100644 index 00000000000000..6c1d9003d9df8c --- /dev/null +++ b/regression-test/data/export_p0/outfile/test_outfile_date_year_zero.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !source -- +1 0000-01-01 +10 \N +2 0000-01-31 +3 0000-02-28 +4 0000-03-01 +5 0001-01-01 +6 1969-12-31 +7 1970-01-01 +8 2024-01-01 +9 9999-12-31 + +-- !readback_parquet -- +1 0000-01-01 +10 \N +2 0000-01-31 +3 0000-02-28 +4 0000-03-01 +5 0001-01-01 +6 1969-12-31 +7 1970-01-01 +8 2024-01-01 +9 9999-12-31 + +-- !readback_orc -- +1 0000-01-01 +10 \N +2 0000-01-31 +3 0000-02-28 +4 0000-03-01 +5 0001-01-01 +6 1969-12-31 +7 1970-01-01 +8 2024-01-01 +9 9999-12-31 + diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out index 552e5319a9d9b7..a615235d54425a 100644 --- a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out @@ -58,7 +58,7 @@ -- !temporal_physical_partitions -- \N \N \N \N \N \N 1 -0 0 0 7 1970-01-01 0 1 +0 -1 -1 7 1969-12-31 -1 1 4 0 0 4 1970-01-01 0 1 6 54 649 4 2024-02-29 474780 1 diff --git a/regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy b/regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy new file mode 100644 index 00000000000000..c480a29e24bdc8 --- /dev/null +++ b/regression-test/suites/arrow_flight_sql_p0/test_date_year_zero.groovy @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Doris numbers DATE values in MySQL's calendar, where year 0 is not a leap year. Arrow `date32` +// is days since 1970-01-01 in the proleptic Gregorian calendar, where year 0 IS a leap year, so +// the two disagree for 0000-01-01 .. 0000-02-28. The MySQL protocol ships the year/month/day +// fields directly and therefore never depends on a calendar; it is the control path here. Any +// difference between the two protocols means the Arrow day ordinal is wrong. +// See https://github.com/apache/doris/issues/67366 +suite("test_date_year_zero", "arrow_flight_sql") { + sql "DROP TABLE IF EXISTS test_date_year_zero" + sql """ + CREATE TABLE test_date_year_zero ( + id INT, + d DATE, + d_null DATE NULL, + arr ARRAY + ) DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + sql """ + INSERT INTO test_date_year_zero VALUES + (1, '0000-01-01', '0000-01-01', ['0000-01-01', '0000-02-28']), + (2, '0000-02-28', NULL, ['0000-03-01']), + (3, '0000-03-01', '0000-03-01', ['0001-01-01']), + (4, '0001-01-01', '0001-01-01', ['1969-12-31', '1970-01-01']), + (5, '1969-12-31', '1969-12-31', ['2024-01-01']), + (6, '1970-01-01', '1970-01-01', ['9999-12-31']), + (7, '2024-01-01', '2024-01-01', ['0000-01-01']), + (8, '9999-12-31', '9999-12-31', NULL); + """ + + // The absolute anchor. The MySQL protocol carries year/month/day verbatim, so these strings + // are what the Arrow path below must reproduce. Cast to string: java.sql.Date runs year-zero + // values through the Julian/Gregorian hybrid calendar. + order_qt_source """ + SELECT CAST(id AS STRING), CAST(d AS STRING), CAST(d_null AS STRING) + FROM test_date_year_zero + """ + + // Read through the driver's own rendering rather than getObject(), for the same reason. + def fetchStrings = { java.sql.Connection conn, String stmtSql -> + def rows = [] + conn.prepareStatement(stmtSql).withCloseable { st -> + st.executeQuery().withCloseable { rs -> + def columnCount = rs.metaData.columnCount + while (rs.next()) { + def row = [] + for (int i = 1; i <= columnCount; ++i) { + row.add(rs.getString(i)) + } + rows.add(row) + } + } + } + return rows + } + + def query = "SELECT id, d, d_null FROM test_date_year_zero ORDER BY id" + def viaMysql = fetchStrings(context.getConnection(), query) + def viaFlight = fetchStrings(context.getArrowFlightSqlConnection(), + "USE ${context.dbName};" + query) + assertEquals(8, viaMysql.size()) + // Before the fix, Arrow reported 0000-01-02 for row 1 and 0000-02-29 for row 2. + assertEquals(viaMysql, viaFlight, + "the Arrow and MySQL protocols disagree on the DATE values") + + // ARRAY has no encoding of its own, it delegates each element to the DATE SerDe. The + // Arrow Flight JDBC driver renders a list as its raw day counts rather than as dates, + // which makes this the strictest check available here: it pins the exact wire values. + // 0000-01-01 is -719528 and 0000-02-28 is -719470 in the proleptic Gregorian calendar; the + // pre-fix encoding produced -719527 and -719469. + def arrayQuery = "SELECT arr FROM test_date_year_zero WHERE id = 1" + def arrayViaFlight = + fetchStrings(context.getArrowFlightSqlConnection(), + "USE ${context.dbName};" + arrayQuery)[0][0].toString() + def arrayElements = arrayViaFlight.replaceAll(/[\[\]\s"]/, "").split(",") as List + assertEquals(["-719528", "-719470"], arrayElements, + "ARRAY elements are not proleptic Gregorian day counts, got: ${arrayViaFlight}") +} diff --git a/regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy b/regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy new file mode 100644 index 00000000000000..7d4cfaa1a13f73 --- /dev/null +++ b/regression-test/suites/export_p0/outfile/test_outfile_date_year_zero.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Parquet and ORC both store DATE as days since 1970-01-01 in the proleptic Gregorian calendar, +// which differs from Doris's MySQL-calendar day number for 0000-01-01 .. 0000-02-28. The writer +// (which shares the Arrow date32 encoder) and the reader must therefore agree, otherwise a table +// exported and re-read by Doris comes back shifted by a day. +// See https://github.com/apache/doris/issues/67366 +suite("test_outfile_date_year_zero", "p0") { + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName") + + def outFilePath = "${bucket}/outfile/date_year_zero/exp_" + + sql """ DROP TABLE IF EXISTS test_outfile_date_year_zero_table """ + sql """ + CREATE TABLE test_outfile_date_year_zero_table ( + `id` INT NOT NULL, + `d` DATE NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + // 0000-01-01 .. 0000-02-28 are the window where the two calendars disagree; 0000-03-01 onwards + // they coincide, so keep dates on both sides of the boundary plus the two range limits. + sql """ + INSERT INTO test_outfile_date_year_zero_table VALUES + (1, '0000-01-01'), (2, '0000-01-31'), (3, '0000-02-28'), (4, '0000-03-01'), + (5, '0001-01-01'), (6, '1969-12-31'), (7, '1970-01-01'), (8, '2024-01-01'), + (9, '9999-12-31'), (10, NULL); + """ + + // Cast to string: the MySQL JDBC driver cannot materialise a year-zero DATE as java.sql.Date. + order_qt_source """ + SELECT CAST(id AS STRING), CAST(d AS STRING) FROM test_outfile_date_year_zero_table + """ + + def outfile_to_S3 = { format -> + def res = sql """ + SELECT * FROM test_outfile_date_year_zero_table t + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3] + def uri = "http://${bucket}.${s3_endpoint}" + + outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1) + "0." + format + return """ + SELECT CAST(id AS STRING), CAST(d AS STRING) FROM S3 ( + "uri" = "${uri}", + "ACCESS_KEY" = "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${format}", + "region" = "${region}" + ) + """ + } + + // Both read-backs must return exactly the source rows: same values, same NULL. + order_qt_readback_parquet(outfile_to_S3("parquet")) + order_qt_readback_orc(outfile_to_S3("orc")) +} diff --git a/regression-test/suites/export_p0/outfile/test_outfile_datetime_year_zero.groovy b/regression-test/suites/export_p0/outfile/test_outfile_datetime_year_zero.groovy new file mode 100644 index 00000000000000..fe2d1cfafe71a5 --- /dev/null +++ b/regression-test/suites/export_p0/outfile/test_outfile_datetime_year_zero.groovy @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// DATETIME starts at 0000-01-01 00:00:00, but the Parquet reader used to reject every timestamp +// below 0001-01-01, so a value Doris accepts, stores and exports could not be read back out of +// Doris's own file. The gate was also applied to the raw instant *before* the timezone offset, +// which lost representable values near both ends of the range whenever the session timezone was +// not UTC. DATETIME is a wall-clock type, so its rendering does not depend on the session +// timezone: the same expected strings must come back under every timezone and both scanners. +// See https://github.com/apache/doris/issues/67447 +suite("test_outfile_datetime_year_zero", "p0") { + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = context.config.otherConfigs.get("s3BucketName") + + def outFilePath = "${bucket}/outfile/datetime_year_zero/exp_" + + sql """ DROP TABLE IF EXISTS test_outfile_datetime_year_zero_table """ + sql """ + CREATE TABLE test_outfile_datetime_year_zero_table ( + `id` INT NOT NULL, + `ts` DATETIME(6) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + // Row 1 and row 10 are the two ends of the DATETIME domain: east of UTC row 1's instant falls + // below the civil minimum and west of UTC row 10's rises above the maximum, so they are the + // rows that pin "apply the offset, then judge the range". Rows 3 and 4 sit inside + // 0000-01-01 .. 0000-02-28, the window where Doris's day numbering and the proleptic + // Gregorian ordinal disagree, and rows 2, 5, 7 and 9 are the ones from the report. + sql """ + INSERT INTO test_outfile_datetime_year_zero_table VALUES + (1, '0000-01-01 00:00:00'), (2, '0000-01-01 12:34:56'), + (3, '0000-01-02 00:00:00'), (4, '0000-02-28 23:59:59.999999'), + (5, '0000-03-01 00:00:00'), (6, '0001-01-01 00:00:00'), + (7, '1969-12-31 23:59:59'), (8, '1970-01-01 00:00:00'), + (9, '2024-01-01 12:00:00'), (10, '9999-12-31 23:59:59.999999'), + (11, NULL); + """ + + def expected = [['1', '0000-01-01 00:00:00.000000'], ['2', '0000-01-01 12:34:56.000000'], + ['3', '0000-01-02 00:00:00.000000'], ['4', '0000-02-28 23:59:59.999999'], + ['5', '0000-03-01 00:00:00.000000'], ['6', '0001-01-01 00:00:00.000000'], + ['7', '1969-12-31 23:59:59.000000'], ['8', '1970-01-01 00:00:00.000000'], + ['9', '2024-01-01 12:00:00.000000'], ['10', '9999-12-31 23:59:59.999999'], + ['11', null]] + + def readSource = { -> + return sql("""SELECT CAST(id AS STRING), CAST(ts AS STRING) + FROM test_outfile_datetime_year_zero_table ORDER BY id""") + } + assertEquals(expected, readSource(), "the table itself does not hold the values under test") + + def outfile_to_S3 = { format -> + def res = sql """ + SELECT * FROM test_outfile_datetime_year_zero_table t + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + return res[0][3] + } + + def uriOf = { outfile_url, format -> + return "http://${bucket}.${s3_endpoint}" + + outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1) + "0." + format + } + + def s3Table = { uri, format, extraProps = "" -> + return """ S3 ( + "uri" = "${uri}", + "ACCESS_KEY" = "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${format}", + "region" = "${region}"${extraProps} + ) """ + } + def timestampTzProp = ',\n "enable_mapping_timestamp_tz" = "true"' + + def originalTimeZone = sql("SHOW VARIABLES LIKE 'time_zone'")[0][1] + def originalScannerV2 = sql("SHOW VARIABLES LIKE 'enable_file_scanner_v2'")[0][1] + + try { + // The Parquet writer encodes DATETIME as an instant in the session timezone, so the + // timezone decides how far each end of the range sits from the raw bound the reader used + // to check. UTC alone would not have caught the offset half of the defect. + for (String timeZone : ["+00:00", "+08:00", "-05:00"]) { + sql """ set time_zone = '${timeZone}' """ + assertEquals(expected, readSource(), + "DATETIME rendering must not depend on the session timezone") + + def uri = uriOf(outfile_to_S3("parquet"), "parquet") + // Only FileScannerV2, which is the default. The legacy scanner is deliberately not + // asserted against: it truncates a negative epoch value towards zero instead of + // flooring it, so any pre-1970 timestamp with a sub-second part moves forward by one + // second there -- 0000-02-28 23:59:59.999999 lands on the proleptic-only 0000-02-29 + // and comes back NULL. That is a separate defect in a path this fix does not touch. + sql """ set enable_file_scanner_v2 = true """ + def readBack = sql """ + SELECT CAST(id AS STRING), CAST(ts AS STRING) FROM ${s3Table(uri, "parquet")} + ORDER BY id; + """ + assertEquals(expected, readBack, + "parquet round trip changed the values (time_zone=${timeZone})") + + // A conversion failure is reported as NULL or as the invalid 0000-00-00 sentinel + // rather than as an error, so a scan that never materializes the column still counts + // every row. Comparing the two counts is what makes silent value loss visible. + def counts = sql """ + SELECT CAST(COUNT(*) AS STRING), CAST(COUNT(ts) AS STRING), + CAST(COUNT(CASE WHEN ts < '0001-01-01' THEN 1 END) AS STRING) + FROM ${s3Table(uri, "parquet")}; + """ + assertEquals([['11', '10', '5']], counts, + "year-zero rows are missing from the file scan (time_zone=${timeZone})") + + // Reading a UTC-adjusted Parquet timestamp as TIMESTAMPTZ takes a different failure + // path from DATETIMEV2, and that is the one the report saw turn into NULL. + def tzTable = s3Table(uri, "parquet", timestampTzProp) + def tzCounts = sql """ + SELECT CAST(COUNT(*) AS STRING), CAST(COUNT(ts) AS STRING) FROM ${tzTable}; + """ + assertEquals([['11', '10']], tzCounts, + "year-zero rows became NULL as TIMESTAMPTZ (time_zone=${timeZone})") + } + + sql """ set time_zone = '${originalTimeZone}' """ + + // ORC materializes the same values through its own reader, which never had the year-one + // floor. It is the control: Parquet has to agree with it. + def orcUri = uriOf(outfile_to_S3("orc"), "orc") + def orcReadBack = sql """ + SELECT CAST(id AS STRING), CAST(ts AS STRING) FROM ${s3Table(orcUri, "orc")} ORDER BY id; + """ + assertEquals(expected, orcReadBack, "orc round trip changed the DATETIME values") + + // A conversion failure is materialized, not raised, so it persists into whatever table the + // scan feeds. This is the shape that turns a read bug into stored bad data. + def parquetUri = uriOf(outfile_to_S3("parquet"), "parquet") + sql """ DROP TABLE IF EXISTS test_outfile_datetime_year_zero_restore """ + sql """ + CREATE TABLE test_outfile_datetime_year_zero_restore ( + `id` INT NOT NULL, + `ts` DATETIME(6) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1"); + """ + sql """ + INSERT INTO test_outfile_datetime_year_zero_restore + SELECT id, ts FROM ${s3Table(parquetUri, "parquet")} + """ + def restored = sql """ + SELECT CAST(id AS STRING), CAST(ts AS STRING) + FROM test_outfile_datetime_year_zero_restore ORDER BY id + """ + assertEquals(expected, restored, "loading the exported file back stored different values") + } finally { + sql """ set time_zone = '${originalTimeZone}' """ + sql """ set enable_file_scanner_v2 = ${originalScannerV2} """ + } + +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_epoch_boundary.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_epoch_boundary.groovy new file mode 100644 index 00000000000000..839fa155ed2778 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_epoch_boundary.groovy @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Iceberg's time transforms floor towards negative infinity (the reference implementation does +// this in DateTimeUtil.convertDays/convertMicros: for a negative input it evaluates one unit later +// and then subtracts one). Doris used to derive them from datetime_diff(), which rounds towards +// zero, so every value before 1970-01-01 that was not exactly on a unit boundary got the wrong +// partition -- 1969-12-31 23:59:59 landed in the same day and hour partition as 1970-01-01 +// 00:00:00. The day ordinal additionally has to be proleptic Gregorian, which Doris's +// MySQL-calendar day number is not for 0000-01-01 .. 0000-02-28. +// +// Spark writes through the Iceberg reference implementation, so writing identical rows from both +// engines and comparing the resulting partition metadata is a direct conformance check. +// See https://github.com/apache/doris/issues/67366 +suite("test_iceberg_write_partition_epoch_boundary", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_partition_epoch_boundary" + String dbName = "iceberg_write_partition_epoch_boundary_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + // Iceberg rejects redundant partition fields (year(x) and month(x) on the same column), so + // every transform gets its own column carrying the same value. + def createTable = { String name -> + sql """drop table if exists ${name}""" + sql """ + create table ${name} ( + id int not null, + p_date_year date, + p_date_month date, + p_date_day date, + p_date_bucket date, + p_ts_year datetime, + p_ts_month datetime, + p_ts_day datetime, + p_ts_hour datetime + ) + partition by list ( + year(p_date_year), month(p_date_month), day(p_date_day), bucket(8, p_date_bucket), + year(p_ts_year), month(p_ts_month), day(p_ts_day), hour(p_ts_hour) + ) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + } + + // Values chosen to straddle every boundary the transforms care about: + // * the proleptic-vs-MySQL calendar window 0000-01-01 .. 0000-02-28, + // * the epoch itself, where flooring and truncation diverge, + // * a partial pre-epoch year/month/day/hour, and a modern leap day as a control. + def row = { int id, String d, String ts -> + return "(${id}, date '${d}', date '${d}', date '${d}', date '${d}', " + + "timestamp '${ts}', timestamp '${ts}', timestamp '${ts}', timestamp '${ts}')" + } + def rows = [ + row(1, '0000-01-01', '0000-01-01 12:34:56'), + row(2, '0000-02-28', '0000-02-28 00:00:00'), + row(3, '0000-03-01', '0000-03-01 00:00:00'), + row(4, '1969-06-15', '1969-06-15 10:00:00'), + row(5, '1969-12-31', '1969-12-31 23:59:59'), + row(6, '1970-01-01', '1970-01-01 00:00:00'), + row(7, '2024-02-29', '2024-02-29 12:34:56') + ].join(",\n ") + + // Everything is cast to string: the year transform surfaces as a JDBC YEAR type that + // getObject() refuses, and comparing text keeps the Doris and Spark shapes identical. + def fields = ['p_date_year_year', 'p_date_month_month', 'p_date_day_day', + 'p_date_bucket_bucket', 'p_ts_year_year', 'p_ts_month_month', + 'p_ts_day_day', 'p_ts_hour_hour'] + + def dorisPartitionTuples = { String name -> + sql """refresh table ${name}""" + def projection = fields.collect { "cast(struct_element(`partition`, '${it}') as string)" } + .join(", ") + return sql("select ${projection} from ${name}\$partitions order by 1, 2, 3, 4, 5, 6, 7, 8") + } + + def sparkPartitionTuples = { String name -> + spark_iceberg """refresh table demo.${dbName}.${name}""" + def projection = fields.collect { "cast(partition.${it} as string)" }.join(", ") + return spark_iceberg( + "select ${projection} from demo.${dbName}.${name}.partitions order by 1, 2, 3, 4, 5, 6, 7, 8") + } + + createTable("epoch_boundary_doris") + createTable("epoch_boundary_spark") + + sql """insert into epoch_boundary_doris values ${rows}""" + spark_iceberg """insert into demo.${dbName}.epoch_boundary_spark values ${rows}""" + + // The core conformance assertion: identical source rows must produce identical Iceberg + // partition tuples no matter which engine wrote them. + def dorisPartitions = dorisPartitionTuples("epoch_boundary_doris") + def sparkPartitions = sparkPartitionTuples("epoch_boundary_spark") + logger.info("doris partitions: ${dorisPartitions}") + logger.info("spark partitions: ${sparkPartitions}") + assertEquals(7, dorisPartitions.size()) + assertEquals(sparkPartitions.toString(), dorisPartitions.toString(), + "Doris and Spark disagree on Iceberg partition values") + + // Backstop, so a regression cannot hide behind both engines being compared only against each + // other: 1969-12-31 23:59:59 must floor to hour -1, and seven distinct source rows must not + // share a partition. + def hours = dorisPartitions.collect { it[7].toString() } + assertTrue(hours.contains("-1"), + "1969-12-31 23:59:59 must floor to hour -1, got: ${hours}") + assertEquals(7, dorisPartitions.collect { it.toString() }.unique().size(), + "distinct timestamps must not share a partition: ${dorisPartitions}") + + // The values themselves must survive the round trip through the Doris writer, not just their + // partition tuples. Cast to string: the MySQL JDBC driver cannot materialise a year-zero DATE + // as java.sql.Date, and Spark's rendering is the reference for the proleptic Gregorian + // calendar the file is written in. + // + // Iceberg `timestamp` is not adjusted to UTC, so the timestamp column lands on the civil + // Parquet timestamp path. Doris used to reject every value below 0001-01-01 there and to add + // its MySQL day number straight to a proleptic Gregorian ordinal, which shifted the whole + // 0000-01-01 .. 0000-02-28 window by a day; rows 1 and 2 read back as NULL and row 3 was the + // first one that survived. See https://github.com/apache/doris/issues/67447 + sql """refresh table epoch_boundary_doris""" + spark_iceberg """refresh table demo.${dbName}.epoch_boundary_doris""" + assertSparkDorisResultEquals( + spark_iceberg("""select cast(id as string), cast(p_date_day as string), + cast(p_ts_day as string) + from demo.${dbName}.epoch_boundary_doris order by 1"""), + sql("""select cast(id as string), cast(p_date_day as string), + cast(p_ts_day as string) + from epoch_boundary_doris order by 1""")) + + // The same rows written by Spark, read by Doris: the reader has to accept a year-zero + // timestamp produced by the reference implementation, not only one it wrote itself. + sql """refresh table epoch_boundary_spark""" + spark_iceberg """refresh table demo.${dbName}.epoch_boundary_spark""" + assertSparkDorisResultEquals( + spark_iceberg("""select cast(id as string), cast(p_date_day as string), + cast(p_ts_day as string) + from demo.${dbName}.epoch_boundary_spark order by 1"""), + sql("""select cast(id as string), cast(p_date_day as string), + cast(p_ts_day as string) + from epoch_boundary_spark order by 1""")) + + // Everything above writes parquet. ORC has its own DATE encoder and, more importantly, its own + // statistics decoder: a pushed-down MIN/MAX is answered from ORC stripe statistics without + // reading a row, so a statistics decoder that disagrees with the row decoder returns a value + // that appears in no row at all. Keep the format-specific coverage on DATE, which is where the + // two calendars differ. + sql """drop table if exists epoch_boundary_orc""" + sql """ + create table epoch_boundary_orc ( + id int not null, + d date + ) + properties ( + "format-version" = "2", + "write.format.default" = "orc" + ) + """ + sql """ + insert into epoch_boundary_orc values + (1, date '0000-01-01'), (2, date '0000-02-28'), (3, date '0000-03-01'), + (4, date '1969-12-31'), (5, date '1970-01-01'), (6, date '2024-02-29') + """ + sql """refresh table epoch_boundary_orc""" + spark_iceberg """refresh table demo.${dbName}.epoch_boundary_orc""" + assertSparkDorisResultEquals( + spark_iceberg("""select cast(id as string), cast(d as string) + from demo.${dbName}.epoch_boundary_orc order by 1"""), + sql("""select cast(id as string), cast(d as string) + from epoch_boundary_orc order by 1""")) + + // MIN/MAX without GROUP BY is pushed into the file scan and answered from stripe statistics + // alone. It must agree with what the rows say: before the fix the ORC statistics went through + // Doris's day dictionary and reported 1900-01-01 as the minimum of a file whose smallest row + // is 0000-01-01. + def orcMinMax = sql """select cast(min(d) as string), cast(max(d) as string) + from epoch_boundary_orc""" + def orcSmallest = sql """select cast(d as string) from epoch_boundary_orc order by d limit 1""" + def orcLargest = sql """select cast(d as string) from epoch_boundary_orc order by d desc limit 1""" + assertEquals([[orcSmallest[0][0], orcLargest[0][0]]].toString(), orcMinMax.toString(), + "pushed-down MIN/MAX disagrees with the rows of the ORC file") +}