From e4c2dc261a7a8f1dca9286f2509a4e4fca062ce7 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 3 Nov 2026 10:21:39 +0000 Subject: [PATCH] [fix](be) Reject Arrow timestamps outside the DATETIMEV2 and TIMESTAMPTZ ranges (#69697) ### What problem does this PR solve? Issue Number: None Related PR: #69581 #68516 Problem Summary: **In short.** An Arrow timestamp whose civil year is outside 0000..9999 is read into DATETIMEV2 or TIMESTAMPTZ as a different, valid-looking value: timestamp(us) 2005980768000000000, which is 65537-01-01, reads as a non-null 0001-01-01. This PR checks the year before the value is narrowed into Doris' 16-bit year field, as the ORC path already does, and fails the read otherwise. The review of #69581 found it on the Lance path; the code is shared by every Arrow reader. **Background.** `DataTypeDateTimeV2SerDe::read_column_from_arrow` and `DataTypeTimeStampTzSerDe::read_column_from_arrow` decode Arrow timestamps for ADBC, Arrow stream load, paimon-rust, remote Doris and, once #69581 lands, Lance, nested leaves included. Both convert epoch seconds with `DateV2Value::from_unixtime()`, which passes cctz's 64-bit year to `unchecked_set_time(uint16_t year, ...)` ("there's no overflow check since it's hot path"). #68516 added a check before this narrowing to the ORC path (`orc_timestamp_to_datetime()`). **The problem, and what it cost.** | Arrow timestamp(us) | DATETIMEV2 before | TIMESTAMPTZ before | |---|---|---| | 253402300800000000 (10000-01-01) | stored as year 10000, outside the type's range | error | | 2005980768000000000 (65537-01-01) | 0001-01-01 00:00:00 | 0001-01-01 00:00:00+00:00 | | a year before 0000 | wraps to 655xx or back into 0000..9999 | error, or a wrapped valid year | In the wrapped cases the read succeeds with a wrong value, and nothing in the result tells the user. **How this PR fixes it.** - DATETIMEV2: convert to civil time in the value's time zone (UTC for a timezone-naive Arrow timestamp, the session time zone for a zoned one) and reject a year outside 0000..9999 before packing, like `orc_timestamp_to_datetime()`. After packing, `is_valid_date()` is checked too: cctz counts year 0 as a leap year and Doris does not (`is_leap(0)` is false), so the civil date 0000-02-29 passes the year check, and before this PR it was stored as an invalid date. - TIMESTAMPTZ is stored in UTC, so `append_timestamptz_from_utc_epoch_micros` checks the instant against 0000-01-01 00:00:00 UTC .. 9999-12-31 23:59:59.999999 UTC before converting. Its runtime `is_valid_date()` check after packing stays, for the same leap day: timestamp(us) -62162121600000000 (0000-02-29 UTC) is inside the range but packs into an invalid value. The range message now says 0000-9999, the range `is_valid_date()` accepts. - Both fail the read with DataQualityError, as the other range errors of these readers do. NULL slots are still skipped before conversion. - Not changed: the Parquet INT64 / INT96 timestamp paths validate the UTC range before packing (`parquet_timestamp_micros()`, `parquet_int96_timestamp_micros()`), so they could not wrap. Arrow reads into the deprecated DATE / DATETIME (v1) types go through `VecDateTimeValue::from_unixtime()` with the same unchecked narrowing; they are left as they are. `orc_timestamp_to_datetime()` checks only the year, so an ORC timestamp on 0000-02-29 is still stored as an invalid date; it is not changed here. **Merge order: after #69581.** A NULL STRUCT, LIST or MAP row may hide any child value, an out-of-range timestamp included, because Arrow leaves the payload under a NULL parent unspecified. #69581 masks those child slots in the shared nested SerDes (`mask_arrow_children_hidden_by_null_parents`, df95b41a331) before any leaf SerDe reads them. Without it, the checks here fail a batch whose visible values are valid: a STRUCT with rows `[NULL, {ts: 0}]` whose NULL row carries 2005980768000000000 reads with DataQualityError on this PR alone, and reads both rows on a tree with both PRs. **Results.** | Arrow timestamp(us) | DATETIMEV2 after | TIMESTAMPTZ after | |---|---|---| | 253402300799999999 (9999-12-31 23:59:59.999999) | read | read | | -62167219200000000 (0000-01-01) | read | read | | -62162121600000000 (0000-02-29, not a Doris date) | DataQualityError | DataQualityError | | 10000-01-01, 65537-01-01, -62167219200000001 (year -1) | DataQualityError | DataQualityError | | zoned, at +08:00: 253402271999999999 / 253402272000000000 | 9999-12-31 23:59:59.999999 / DataQualityError | | --- .../data_type_datetimev2_serde.cpp | 24 +++++- .../data_type_timestamptz_serde.cpp | 21 ++++- .../data_type_serde_datetime_v2_test.cpp | 77 +++++++++++++++++++ .../data_type_serde_timestamptz_test.cpp | 39 ++++++++++ 4 files changed, 157 insertions(+), 4 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 36fc722e60d..9df0a8e060a 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -723,7 +723,29 @@ Status DataTypeDateTimeV2SerDe::read_column_from_arrow(IColumn& column, --seconds; remainder += divisor; } - v.from_unixtime(seconds, real_ctz); + // from_unixtime() narrows the cctz year to uint16_t without a check, so an instant + // whose civil year in real_ctz is outside 0000..9999 would read as another value: + // timestamp(us) 2005980768000000000 (65537-01-01) as 0001-01-01, and the years + // 10000..65535 as dates DATETIMEV2 cannot hold. Check the civil year before packing, + // as orc_timestamp_to_datetime() does for ORC. + const auto civil = cctz::convert( + cctz::time_point(cctz::seconds(seconds)), real_ctz); + if (civil.year() < 0 || civil.year() > 9999) { + return Status::DataQualityError( + "Arrow timestamp {} of type {} is outside the DATETIMEV2 range", date_value, + type->ToString()); + } + v.unchecked_set_time( + static_cast(civil.year()), static_cast(civil.month()), + static_cast(civil.day()), static_cast(civil.hour()), + static_cast(civil.minute()), static_cast(civil.second()), 0); + // cctz counts year 0 as a leap year and Doris does not (is_leap(0) is false), so the + // civil date 0000-02-29 passes the year check but is not a DATETIMEV2 value. + if (!v.is_valid_date()) { + return Status::DataQualityError( + "Arrow timestamp {} of type {} is not a valid DATETIMEV2 value", date_value, + type->ToString()); + } // Get the fractional part. // add 0 on the right to make it 6 digits. DateTimeV2Value microsecond is 6 digits, // the scale decides to keep the first few digits, so the valid digits should be kept at the front. diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index 52254438d0b..a1cd4ec292b 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -78,8 +78,21 @@ Status decode_timestamp_tz_orc_values(IColumn& nested_column, Status append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; + // 0000-01-01 00:00:00 UTC and 9999-12-31 23:59:59.999999 UTC, the TIMESTAMPTZ range. + static constexpr int64_t MIN_TIMESTAMPTZ_MICROS = -62167219200000000LL; + static constexpr int64_t MAX_TIMESTAMPTZ_MICROS = 253402300799999999LL; static const auto UTC = cctz::utc_time_zone(); + // from_unixtime() narrows the cctz year to uint16_t without a check, so an instant past year + // 65535 wraps back into 0000..9999 and would still pass is_valid_date() after packing: + // timestamp(us) 2005980768000000000 (65537-01-01) read as 0001-01-01. The Arrow reader passes + // any int64 here, so check the range before converting; Parquet values arrive inside it. + if (timestamp_micros < MIN_TIMESTAMPTZ_MICROS || timestamp_micros > MAX_TIMESTAMPTZ_MICROS) { + return Status::DataQualityError( + "Decoded TIMESTAMPTZ is outside the Doris 0000-9999 range: micros={}", + timestamp_micros); + } + int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; int64_t micros_of_second = timestamp_micros % MICROS_PER_SECOND; if (micros_of_second < 0) { @@ -90,10 +103,12 @@ Status append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& da TimestampTzValue timestamp_tz; timestamp_tz.from_unixtime(epoch_seconds, UTC); timestamp_tz.set_microsecond(static_cast(micros_of_second)); + // The range check does not make every instant inside it a Doris date: cctz counts year 0 as a + // leap year and Doris does not (is_leap(0) is false), so 0000-02-29 UTC, timestamp(us) + // -62162121600000000, passes the range and packs into an invalid value. if (!timestamp_tz.is_valid_date()) { - return Status::DataQualityError( - "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", - timestamp_micros); + return Status::DataQualityError("Decoded TIMESTAMPTZ is not a valid Doris date: micros={}", + timestamp_micros); } data.push_back(timestamp_tz); return Status::OK(); diff --git a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp index 51dec6fc83b..6f9d0c58151 100644 --- a/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_datetime_v2_test.cpp @@ -31,10 +31,12 @@ #include "core/assert_cast.h" #include "core/column/column.h" +#include "core/column/column_array.h" #include "core/column/column_nullable.h" #include "core/data_type/common_data_type_serder_test.h" #include "core/data_type/common_data_type_test.h" #include "core/data_type/data_type.h" +#include "core/data_type_serde/data_type_array_serde.h" #include "core/data_type_serde/data_type_datetimev2_serde.h" #include "core/data_type_serde/data_type_datev2_serde.h" #include "core/data_type_serde/data_type_nullable_serde.h" @@ -589,4 +591,79 @@ TEST_F(DataTypeDateTimeV2SerDeTest, ArrowTimezoneNaiveTimestampIgnoresSessionTim EXPECT_EQ("1969-12-31 23:59:59.123456", column->get_data()[2].to_string(6)); } +// The civil year of an Arrow timestamp must be checked before it is narrowed into DATETIMEV2: +// unchecked, 65537-01-01 read as 0001-01-01 and 10000-01-01 as a date DATETIMEV2 cannot hold. A +// zoned timestamp is checked in the session time zone, after the shift. +TEST_F(DataTypeDateTimeV2SerDeTest, RejectsArrowTimestampsOutsideDateTimeV2Range) { + const auto read = [](const std::shared_ptr& type, int64_t value, + const cctz::time_zone& timezone, std::string* result) { + arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); + EXPECT_TRUE(builder.Append(value).ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + auto column = ColumnDateTimeV2::create(); + auto status = + serde_datetime_v2_6->read_column_from_arrow(*column, array.get(), 0, 1, timezone); + if (status.ok()) { + *result = column->get_data()[0].to_string(6); + } + return status; + }; + const auto naive = arrow::timestamp(arrow::TimeUnit::MICRO); + const auto utc = cctz::utc_time_zone(); + std::string result; + ASSERT_TRUE(read(naive, 253402300799999999, utc, &result).ok()); + EXPECT_EQ("9999-12-31 23:59:59.999999", result); + ASSERT_TRUE(read(naive, -62167219200000000, utc, &result).ok()); + EXPECT_EQ("0000-01-01 00:00:00.000000", result); + // 10000-01-01, 65537-01-01 (read as 0001-01-01 before the check) and the last microsecond of + // year -1. + for (const int64_t value : {int64_t {253402300800000000}, int64_t {2005980768000000000}, + int64_t {-62167219200000001}}) { + SCOPED_TRACE(value); + const auto status = read(naive, value, utc, &result); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("is outside the DATETIMEV2 range")) + << status.to_string(); + } + // 0000-02-29 is inside the year range, but year 0 is not a leap year in Doris. + const auto leap_day = read(naive, -62162121600000000, utc, &result); + EXPECT_FALSE(leap_day.ok()); + EXPECT_NE(std::string::npos, leap_day.to_string().find("is not a valid DATETIMEV2 value")) + << leap_day.to_string(); + + // 9999-12-31 15:59:59.999999 UTC is 9999-12-31 23:59:59.999999 at +08:00, and one microsecond + // later is 10000-01-01 there. + cctz::time_zone plus_eight; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("+08:00", plus_eight)); + const auto zoned = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + ASSERT_TRUE(read(zoned, 253402271999999999, plus_eight, &result).ok()); + EXPECT_EQ("9999-12-31 23:59:59.999999", result); + EXPECT_FALSE(read(zoned, 253402272000000000, plus_eight, &result).ok()); +} + +// A timestamp leaf of ARRAY reaches the same check through the element SerDe. +TEST_F(DataTypeDateTimeV2SerDeTest, RejectsNestedArrowTimestampOutsideDateTimeV2Range) { + arrow::TimestampBuilder values_builder(arrow::timestamp(arrow::TimeUnit::MICRO), + arrow::default_memory_pool()); + ASSERT_TRUE(values_builder.Append(2005980768000000000).ok()); + std::shared_ptr values; + ASSERT_TRUE(values_builder.Finish(&values).ok()); + arrow::Int32Builder offsets_builder; + ASSERT_TRUE(offsets_builder.AppendValues({0, 1}).ok()); + std::shared_ptr offsets; + ASSERT_TRUE(offsets_builder.Finish(&offsets).ok()); + auto list = arrow::ListArray::FromArrays(*offsets, *values); + ASSERT_TRUE(list.ok()) << list.status().ToString(); + + DataTypeArraySerDe serde(std::make_shared(serde_datetime_v2_6)); + auto column = ColumnArray::create( + ColumnNullable::create(ColumnDateTimeV2::create(), ColumnUInt8::create())); + const auto status = serde.read_column_from_arrow(*column, list.ValueUnsafe().get(), 0, 1, + cctz::utc_time_zone()); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("is outside the DATETIMEV2 range")) + << status.to_string(); +} + } // namespace doris diff --git a/be/test/core/data_type_serde/data_type_serde_timestamptz_test.cpp b/be/test/core/data_type_serde/data_type_serde_timestamptz_test.cpp index 742715ef62e..9de6e14d898 100644 --- a/be/test/core/data_type_serde/data_type_serde_timestamptz_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_timestamptz_test.cpp @@ -395,4 +395,43 @@ TEST_F(DataTypeTimeStampTzSerDeTest, ArrowTimestampToTimestampTz) { read_timestamp(arrow::TimeUnit::NANO, -876543211, "Asia/Shanghai", "1969-12-31 23:59:59.123456+00:00", "1970-01-01 07:59:59.123456+08:00"); } + +// An Arrow instant must be range-checked before it is narrowed into TIMESTAMPTZ: unchecked, +// 65537-01-01 UTC wrapped back to 0001-01-01 and passed the validity check after packing. +TEST_F(DataTypeTimeStampTzSerDeTest, RejectsArrowTimestampsOutsideTimestampTzRange) { + const auto read = [](int64_t micros, std::string* result) { + arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + arrow::default_memory_pool()); + EXPECT_TRUE(builder.Append(micros).ok()); + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + auto column = ColumnTimeStampTz::create(); + auto status = serde_tz_6->read_column_from_arrow(*column, array.get(), 0, 1, + cctz::utc_time_zone()); + if (status.ok()) { + *result = column->get_data()[0].to_string(cctz::utc_time_zone(), 6); + } + return status; + }; + std::string result; + ASSERT_TRUE(read(253402300799999999, &result).ok()); + EXPECT_EQ("9999-12-31 23:59:59.999999+00:00", result); + ASSERT_TRUE(read(-62167219200000000, &result).ok()); + EXPECT_EQ("0000-01-01 00:00:00.000000+00:00", result); + // 10000-01-01, 65537-01-01 (read as 0001-01-01 before the check) and the last microsecond of + // year -1. + for (const int64_t micros : {int64_t {253402300800000000}, int64_t {2005980768000000000}, + int64_t {-62167219200000001}}) { + SCOPED_TRACE(micros); + const auto status = read(micros, &result); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("outside the Doris 0000-9999 range")) + << status.to_string(); + } + // 0000-02-29 UTC is inside the range, but year 0 is not a leap year in Doris. + const auto leap_day = read(-62162121600000000, &result); + EXPECT_FALSE(leap_day.ok()); + EXPECT_NE(std::string::npos, leap_day.to_string().find("not a valid Doris date")) + << leap_day.to_string(); +} } // namespace doris