[fix](arrow) encode DATE in the proleptic Gregorian calendar at format boundaries

be/src/core/data_type_serde/data_type_datetimev2_serde.cpp

@@ -104,7 +104,6 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data,
  static constexpr int64_t MICROS_PER_MINUTE = MICROS_PER_SECOND * 60;
  static constexpr int64_t MICROS_PER_HOUR = MICROS_PER_MINUTE * 60;
  static constexpr int64_t MICROS_PER_DAY = MICROS_PER_HOUR * 24;
- static const int64_t EPOCH_DAYNR = calc_daynr(1970, 1, 1);
 
  int64_t days_since_epoch = timestamp_micros / MICROS_PER_DAY;
  int64_t micros_of_day = timestamp_micros % MICROS_PER_DAY;
@@ -113,11 +112,16 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data,
  --days_since_epoch;
  }
 
- const int64_t daynr = EPOCH_DAYNR + days_since_epoch;
- if (daynr <= 0) {
+ // A local Parquet timestamp carries a civil value whose day ordinal is defined in the
+ // proleptic Gregorian calendar, while Doris numbers days in MySQL's calendar, where year 0 is
+ // not a leap year. Adding `calc_daynr(1970, 1, 1)` directly conflates the two and shifts
+ // 0000-01-01 .. 0000-02-28 one day early. `epoch_days_to_daynr()` bridges them and returns 0
+ // for everything Doris cannot represent, including the proleptic-only 0000-02-29.
+ const int64_t daynr = epoch_days_to_daynr(days_since_epoch);
+ if (daynr == 0) {
  return Status::DataQualityError(
- "Decoded DATETIMEV2 timestamp is out of range: micros={}, daynr={}",
- timestamp_micros, daynr);
+ "Decoded DATETIMEV2 timestamp is out of range: micros={}, epoch_days={}",
+ timestamp_micros, days_since_epoch);
  }
 
  DateV2Value<DateTimeV2ValueType> datetime_value;
@@ -154,6 +158,11 @@ Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data
  DateV2Value<DateTimeV2ValueType> datetime_value;
  datetime_value.from_unixtime(epoch_seconds, timezone);
  datetime_value.set_microsecond(static_cast<uint32_t>(micros_of_second));
+ // The civil range has to be checked here, after the offset is applied, not on the raw instant:
+ // a local 0001-01-01 00:00:00 east of UTC is an instant below the civil minimum, and a local
+ // 9999-12-31 23:59:59 west of it is an instant above the maximum. cctz resolves year 0 (it
+ // uses the proleptic Gregorian calendar), and a year outside 0..9999 fails the check below
+ // because `from_unixtime()` narrows it into a `uint16_t`.
  if (!datetime_value.is_valid_date()) {
  return Status::DataQualityError(
  "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}",

be/src/core/data_type_serde/data_type_datev2_serde.cpp

@@ -22,6 +22,7 @@
 #include <fmt/core.h>
 
 #include <cstdint>
+#include <limits>
 #include <vector>
 
 #include "common/config.h"
@@ -41,13 +42,8 @@
 
 namespace doris {
 
-// This number represents the number of days from 0000-01-01 to 1970-01-01.
-static constexpr int32_t date_threshold = 719528;
-
 namespace {
 
-constexpr int32_t DORIS_DATE_EPOCH_DAYNR = 719528;
-
 Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
  const OrcDecodedColumnView& orc_view) {
  const auto* orc_batch = dynamic_cast<const ::orc::LongVectorBatch*>(orc_view.batch);
@@ -64,22 +60,44 @@ Status decode_date_orc_values(const DataTypeSerDe& serde, IColumn& column,
  orc_serde_utils::orc_decode_row_count(orc_view.rows, orc_view.selected_rows);
  std::vector<int32_t> date_values;
  date_values.resize(output_rows);
- auto& date_dict = date_day_offset_dict::get();
  for (size_t row = 0; row < output_rows; ++row) {
  const auto source_row = orc_serde_utils::orc_source_row_at(row, orc_view.selected_rows);
- const auto date = date_dict[cast_set<int>(orc_batch->data[source_row])];
- date_values[row] = cast_set<int32_t>(date.daynr() - DORIS_DATE_EPOCH_DAYNR);
+ // The payload of a null slot is undefined, so it is zeroed rather than range-checked;
+ // `read_column_from_decoded_values` never decodes it.
+ if (view.null_map != nullptr && view.null_map[row] != 0) {
+ date_values[row] = 0;
+ continue;
+ }
+ // ORC DATE is days since 1970-01-01 in the proleptic Gregorian calendar, the same encoding
+ // `decode_epoch_days()` expects, but ORC hands it over in an int64 batch. Reject a value
+ // that does not fit the int32 view here, while the real file value is still available for
+ // the message, instead of narrowing it blindly.
+ const int64_t file_days = orc_batch->data[source_row];
+ if (file_days < std::numeric_limits<int32_t>::min() ||
+ file_days > std::numeric_limits<int32_t>::max()) {
+ return Status::DataQualityError(
+ "DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
+ "0000-02-29 excluded)",
+ file_days);
+ }
+ date_values[row] = static_cast<int32_t>(file_days);
  }
  view.values = reinterpret_cast<const uint8_t*>(date_values.data());
  RETURN_IF_ERROR(orc_serde_utils::read_decoded_values(serde, column, &view));
  return Status::OK();
 }
 
-Status decode_parquet_date(int32_t encoded_date, DateV2Value<DateV2ValueType>* value) {
+// Shared by every "days since 1970-01-01" source: Parquet, ORC and Arrow date32/date64. The
+// message must therefore not name one format, and it carries the value so a broken file can be
+// identified without re-reading it.
+Status decode_epoch_days(int64_t encoded_date, DateV2Value<DateV2ValueType>* value) {
  DORIS_CHECK(value != nullptr);
- const int64_t day_number = static_cast<int64_t>(encoded_date) + date_threshold;
- if (day_number < 0 || !value->get_date_from_daynr(static_cast<uint64_t>(day_number))) {
- return Status::DataQualityError("Parquet DATE value is out of range");
+ const int64_t day_number = epoch_days_to_daynr(encoded_date);
+ if (day_number == 0 || !value->get_date_from_daynr(static_cast<uint64_t>(day_number))) {
+ return Status::DataQualityError(
+ "DATE value {} is outside the Doris DATE range (0000-01-01..9999-12-31, "
+ "0000-02-29 excluded)",
+ encoded_date);
  }
  return Status::OK();
 }
@@ -99,7 +117,7 @@ class DateV2ParquetConsumer final : public ParquetFixedValueConsumer {
  _data.resize(old_size + num_values);
  for (size_t row = 0; row < num_values; ++row) {
  DateV2Value<DateV2ValueType> value;
- const auto status = decode_parquet_date(
+ const auto status = decode_epoch_days(
  unaligned_load<int32_t>(values + row * sizeof(int32_t)), &value);
  if (!status.ok()) {
  if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) {
@@ -216,13 +234,12 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const IColumn& column, const N
  const auto& col_data = static_cast<const ColumnDateV2&>(column).get_data();
  auto& date32_builder = assert_cast<arrow::Date32Builder&>(*array_builder);
  for (size_t i = start; i < end; ++i) {
- auto daynr = col_data[i].daynr() - date_threshold;
  if (null_map && (*null_map)[i]) {
  RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(), column, *array_builder));
  } else {
- RETURN_IF_ERROR(
-checkArrowStatus(date32_builder.Append(cast_set<int, int64_t, false>(daynr)),
- column, *array_builder));
+ RETURN_IF_ERROR(checkArrowStatus(
+ date32_builder.Append(daynr_to_epoch_days(col_data[i].daynr())), column,
+ *array_builder));
  }
  }
  return Status::OK();
@@ -257,10 +274,9 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
  "Arrow Date64 value must contain whole days: row={}, milliseconds={}",
  value_i, milliseconds);
  }
- const int64_t daynr = milliseconds / MILLISECONDS_PER_DAY + date_threshold;
  DateV2Value<DateV2ValueType> value;
- if (daynr <= 0 || daynr > DATE_MAX_DAYNR ||
- !value.get_date_from_daynr(static_cast<uint64_t>(daynr))) {
+ if (const auto status = decode_epoch_days(milliseconds / MILLISECONDS_PER_DAY, &value);
+ !status.ok()) {
  return Status::InvalidArgument(
  "Arrow Date64 value is outside the Doris DATE range: "
  "row={}, milliseconds={}",
@@ -281,7 +297,11 @@ Status DataTypeDateV2SerDe::read_column_from_arrow(IColumn& column, const arrow:
  auto date_value = unaligned_load<int32_t>(raw_byte_ptr);
 
  DateV2Value<DateV2ValueType> v;
- v.get_date_from_daynr(date_value + date_threshold);
+ if (const auto status = decode_epoch_days(date_value, &v); !status.ok()) {
+ return Status::InvalidArgument(
+ "Arrow Date32 value is outside the Doris DATE range: row={}, days={}",
+ value_i, date_value);
+ }
  col_data.emplace_back(v);
  }
  } else {
@@ -309,7 +329,7 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column,
  continue;
  }
  DateV2Value<DateV2ValueType> date_v2;
- const auto status = decode_parquet_date(values[row], &date_v2);
+ const auto status = decode_epoch_days(values[row], &date_v2);
  if (!status.ok()) {
  // Decoded values back both metadata conversion and native materialization. Preserve
  // strict errors while allowing nullable non-strict scans to mark only the bad row.
@@ -401,7 +421,7 @@ Status DataTypeDateV2SerDe::write_column_to_orc(const std::string& timezone, con
  if (cur_batch->notNull[row_id] == 0) {
  continue;
  }
- cur_batch->data[row_id] = col_data[row_id].daynr() - date_threshold;
+ cur_batch->data[row_id] = daynr_to_epoch_days(col_data[row_id].daynr());
  }
  cur_batch->numElements = end - start;
  return Status::OK();

be/src/core/data_type_serde/data_type_timestamptz_serde.cpp

@@ -89,9 +89,13 @@ Status append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& da
  TimestampTzValue timestamp_tz;
  timestamp_tz.from_unixtime(epoch_seconds, UTC);
  timestamp_tz.set_microsecond(static_cast<uint32_t>(micros_of_second));
+ // `from_unixtime()` splits the instant with cctz (proleptic Gregorian, so year 0 exists) and
+ // narrows the civil year into a `uint16_t`. This is the exact range check for the target
+ // type: a year below 0 wraps above 9999 and a year of 10000 exceeds it, so both are rejected
+ // here, as is the proleptic-only 0000-02-29, which Doris's calendar does not have.
  if (!timestamp_tz.is_valid_date()) {
  return Status::DataQualityError(
- "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}",
+ "Decoded TIMESTAMPTZ is outside the Doris 0000-9999 range: micros={}",
  timestamp_micros);
  }
  data.push_back(timestamp_tz);

be/src/core/data_type_serde/parquet_timestamp.h

@@ -33,14 +33,35 @@ struct ParquetInt96Timestamp {
 #pragma pack()
 static_assert(sizeof(ParquetInt96Timestamp) == 12);
 
-inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL;
-inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL;
+// Doris DATETIMEV2 and TIMESTAMPTZ both start at 0000-01-01 00:00:00 (`MIN_DATE_V2`) and end at
+// 9999-12-31 23:59:59.999999. Parquet timestamps count from 1970-01-01 in the proleptic Gregorian
+// calendar, where year 0 IS a leap year, so 0000-01-01 is 366 days below 0001-01-01 rather than
+// 365. Pinning the lower bound at 0001-01-01 made the reader stricter than the type it
+// materializes into: every year-zero value Doris itself writes came back as a conversion failure.
+inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62167219200000000LL; // 0000-01-01 00:00:00
+inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS =
+ 253402300799999999LL; // 9999-12-31 23:59:59.999999
+
+// A UTC-adjusted Parquet timestamp is an instant, not a civil value: which Doris DATETIME it
+// becomes depends on the reader's timezone, so the raw instant must not be measured against the
+// civil range. A local 0001-01-01 00:00:00 in Asia/Shanghai is an instant *below* the civil
+// minimum, and a local 9999-12-31 23:59:59 in America/New_York is one *above* the maximum; both
+// are representable and must survive. What stays here is only a coarse guard keeping the value in
+// the domain where the cctz conversion and the narrowing to a `uint16_t` year behave. No timezone
+// offset has ever exceeded 16 hours, so one day of slack admits every instant that can still land
+// inside the civil range. The exact range is enforced per target type after the conversion:
+// `epoch_days_to_daynr()` for a civil timestamp, `is_valid_date()` for an instant.
+inline constexpr int64_t DORIS_TIMESTAMP_OFFSET_SLACK_MICROS = 86400000000LL;
+inline constexpr int64_t MIN_DORIS_INSTANT_MICROS =
+ MIN_DORIS_TIMESTAMP_MICROS - DORIS_TIMESTAMP_OFFSET_SLACK_MICROS;
+inline constexpr int64_t MAX_DORIS_INSTANT_MICROS =
+ MAX_DORIS_TIMESTAMP_MICROS + DORIS_TIMESTAMP_OFFSET_SLACK_MICROS;
 
 inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) {
- if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS ||
- timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) {
+ if (timestamp_micros < MIN_DORIS_INSTANT_MICROS ||
+ timestamp_micros > MAX_DORIS_INSTANT_MICROS) {
  return Status::DataQualityError(
- "Parquet timestamp is outside the Doris 0001-9999 range: micros={}",
+ "Parquet timestamp is outside the Doris 0000-9999 range: micros={}",
  timestamp_micros);
  }
  return Status::OK();
@@ -83,10 +104,10 @@ inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value,
  const __int128 days = static_cast<int64_t>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS;
  const __int128 timestamp_micros =
  days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND;
- if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS ||
- timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) {
+ if (timestamp_micros < MIN_DORIS_INSTANT_MICROS ||
+ timestamp_micros > MAX_DORIS_INSTANT_MICROS) {
  return Status::DataQualityError(
- "Parquet INT96 timestamp is outside the Doris 0001-9999 range");
+ "Parquet INT96 timestamp is outside the Doris 0000-9999 range");
  }
  *result = static_cast<int64_t>(timestamp_micros);
  return Status::OK();

be/src/core/value/timestamptz_value.h

@@ -35,8 +35,9 @@ struct CastParameters;
 
 // TIMESTAMPTZ can be understood as a DATETIME type with timezone conversion functionality.
 // Doris automatically handles timezone conversions internally.
-// The storage format of TIMESTAMPTZ is the same as DATETIMEV2, both are 8-byte integers
-// representing microseconds from 0001-01-01 00:00:00.000000 to 9999-12-31 23:59:59.999999.
+// The storage format of TIMESTAMPTZ is the same as DATETIMEV2: both are 8-byte packed civil
+// values spanning 0000-01-01 00:00:00.000000 (`MIN_DATETIME_V2`, also the default) to
+// 9999-12-31 23:59:59.999999. Year 0 is inside the domain, so readers must not floor at year 1.
 // TIMESTAMPTZ does not store timezone information; conversions are performed during read and write
 // operations according to the specified timezone.
 // This requires that both reading and writing operations need a timezone parameter.

be/src/core/value/vdatetime_value.h

@@ -1694,8 +1694,6 @@ class date_day_offset_dict {
 
  static constexpr int START_YEAR = 1900; // 1900-01-01
  static constexpr int END_YEAR = 2039; // 2039-10-24
- static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR =
- 719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1)
 
  static std::array<DateV2Value<DateV2ValueType>, DICT_DAYS> DATE_DAY_OFFSET_ITEMS;
  static std::array<std::array<std::array<int, 31>, 12>, 140> DATE_DAY_OFFSET_DICT;
@@ -1710,6 +1708,9 @@ class date_day_offset_dict {
  date_day_offset_dict& operator=(const date_day_offset_dict&) = default;
 
 public:
+ static constexpr int DAY_OFFSET_CAL_START_POINT_DAYNR =
+ 719528; // 1970-01-01 (start from 0000-01-01, 0000-01-01 is day 1, returns 1)
+
  static bool can_speed_up_calc_daynr(int year) { return year >= START_YEAR && year <= END_YEAR; }
 
  static int get_offset_by_daynr(int daynr) { return daynr - DAY_OFFSET_CAL_START_POINT_DAYNR; }
@@ -1770,6 +1771,47 @@ inline uint32_t calc_daynr(uint16_t year, uint8_t month, uint8_t day) {
  return delsum + y / 4 - y / 100 + y / 400;
 }
 
+// Doris follows MySQL's calendar, in which year 0 is NOT a leap year (see `is_leap()`): 0000-02-29
+// does not exist and `calc_daynr()` therefore numbers 0000-01-01 .. 0000-02-28 one day ahead of the
+// proleptic Gregorian calendar. Arrow `date32`, Parquet/ORC `DATE` and the Iceberg spec all define
+// their day ordinal in the proleptic Gregorian calendar, where year 0 IS a leap year. The two
+// numberings coincide from 0000-03-01 (daynr 60) onwards, so the whole difference is the missing
+// 0000-02-29. Every conversion between a Doris DATE and an external "days since 1970-01-01" value
+// must go through the two helpers below instead of adding/subtracting the epoch daynr directly.
+inline constexpr int64_t DAYNR_OF_UNIX_EPOCH =
+ date_day_offset_dict::DAY_OFFSET_CAL_START_POINT_DAYNR;
+inline constexpr int64_t DAYNR_OF_0000_03_01 = 60; // first daynr shared by both calendars
+
+// Doris daynr -> days since 1970-01-01 in the proleptic Gregorian calendar.
+inline constexpr int32_t daynr_to_epoch_days(int64_t daynr) {
+ return static_cast<int32_t>(daynr - DAYNR_OF_UNIX_EPOCH -
+ (daynr < DAYNR_OF_0000_03_01 ? 1 : 0));
+}
+
+// The three boundaries of the external ordinal domain, derived from the daynr domain rather than
+// typed out, so a change to either calendar constant cannot leave them behind.
+inline constexpr int64_t EPOCH_DAYS_MIN = daynr_to_epoch_days(1); // 0000-01-01, smallest Doris DATE
+inline constexpr int64_t EPOCH_DAYS_0000_02_29 =
+ daynr_to_epoch_days(DAYNR_OF_0000_03_01) - 1; // exists in proleptic Gregorian only
+inline constexpr int64_t EPOCH_DAYS_MAX =
+ daynr_to_epoch_days(DATE_MAX_DAYNR); // 9999-12-31, the largest Doris DATE
+
+static_assert(DAYNR_OF_UNIX_EPOCH == 719528, "calc_daynr(1970, 1, 1)");
+static_assert(EPOCH_DAYS_MIN == -719528);
+static_assert(EPOCH_DAYS_0000_02_29 == -719469);
+static_assert(EPOCH_DAYS_MAX == 2932896);
+
+// Inverse of `daynr_to_epoch_days()`. Returns 0, which is never a valid daynr, when the value has
+// no Doris DATE representation: before 0000-01-01, after 9999-12-31, or the proleptic-only
+// 0000-02-29. Callers must reject 0 instead of feeding it to `get_date_from_daynr()`.
+inline constexpr int64_t epoch_days_to_daynr(int64_t epoch_days) {
+ if (epoch_days < EPOCH_DAYS_MIN || epoch_days > EPOCH_DAYS_MAX ||
+ epoch_days == EPOCH_DAYS_0000_02_29) {
+ return 0;
+ }
+ return epoch_days + DAYNR_OF_UNIX_EPOCH + (epoch_days < EPOCH_DAYS_0000_02_29 ? 1 : 0);
+}
+
 // DAY / WEEK fast path. Real workloads hold dates that cluster in a narrow range, so the two
 // dictionary lookups below (`daynr(y, m, d)` and `daynr -> date`) are L1-resident; measured about
 // 3x faster than the generic `date_add_interval<DAY>` (no TimeInterval, no second-level