Uh oh!
There was an error while loading. Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork 4.3k
ARROW-10924: [C++] Validate temporal data in ValidateArrayFull#12014
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Uh oh!
There was an error while loading. Please reload this page.
Changes from all commits
359b29efd2e08db13946bbd1209eaeedda59400fbbec369c468f62a5f68c2a8827b6a44ace496b7b9d1636c3dfc6d7189bd8be683997c7a1e409c7327411f275febd1e1ba73acc7192fFile filter
Filter by extension
Conversations
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -168,6 +168,81 @@ struct ValidateArrayImpl { | ||
| return Status::OK(); | ||
| } | ||
| Status Visit(const Date64Type& type) { | ||
| RETURN_NOT_OK(ValidateFixedWidthBuffers()); | ||
| if (full_validation) { | ||
| using c_type = typename Date64Type::c_type; | ||
| return VisitArrayDataInline<Date64Type>( | ||
| data, | ||
| [&](c_type date) { | ||
| constexpr c_type kFullDayMillis = 1000 * 60 * 60 * 24; | ||
| if (date % kFullDayMillis != 0) { | ||
| return Status::Invalid(type, " ", date, | ||
| " does not represent a whole number of days"); | ||
| } | ||
| return Status::OK(); | ||
| }, | ||
| []() { return Status::OK(); }); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
| Status Visit(const Time32Type& type) { | ||
| RETURN_NOT_OK(ValidateFixedWidthBuffers()); | ||
| if (full_validation) { | ||
| using c_type = typename Time32Type::c_type; | ||
| return VisitArrayDataInline<Time32Type>( | ||
JabariBooker marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| data, | ||
| [&](c_type time) { | ||
| constexpr c_type kFullDaySeconds = 60 * 60 * 24; | ||
| constexpr c_type kFullDayMillis = kFullDaySeconds * 1000; | ||
| if (type.unit() == TimeUnit::SECOND && | ||
| (time < 0 || time >= kFullDaySeconds)) { | ||
| return Status::Invalid(type, " ", time, | ||
| " is not within the acceptable range of ", "[0, ", | ||
| kFullDaySeconds, ") s"); | ||
| } | ||
| if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= kFullDayMillis)) { | ||
| return Status::Invalid(type, " ", time, | ||
| " is not within the acceptable range of ", "[0, ", | ||
| kFullDayMillis, ") ms"); | ||
| } | ||
| return Status::OK(); | ||
| }, | ||
| []() { return Status::OK(); }); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
| Status Visit(const Time64Type& type) { | ||
| RETURN_NOT_OK(ValidateFixedWidthBuffers()); | ||
| if (full_validation) { | ||
| using c_type = typename Time64Type::c_type; | ||
| return VisitArrayDataInline<Time64Type>( | ||
| data, | ||
| [&](c_type time) { | ||
| constexpr c_type kFullDayMicro = 1000000LL * 60 * 60 * 24; | ||
| constexpr c_type kFullDayNano = kFullDayMicro * 1000; | ||
| if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= kFullDayMicro)) { | ||
| return Status::Invalid(type, " ", time, | ||
| " is not within the acceptable range of ", "[0, ", | ||
| kFullDayMicro, ") us"); | ||
| } | ||
| if (type.unit() == TimeUnit::NANO && (time < 0 || time >= kFullDayNano)) { | ||
| return Status::Invalid(type, " ", time, | ||
| " is not within the acceptable range of ", "[0, ", | ||
| kFullDayNano, ") ns"); | ||
| } | ||
| return Status::OK(); | ||
| }, | ||
| []() { return Status::OK(); }); | ||
| } | ||
| return Status::OK(); | ||
| } | ||
| Status Visit(const BinaryType& type) { return ValidateBinaryLike(type); } | ||
| Status Visit(const LargeBinaryType& type) { return ValidateBinaryLike(type); } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1616,14 +1616,12 @@ TEST(GroupBy, MinMaxTypes) { | ||
| types.insert(types.end(), NumericTypes().begin(), NumericTypes().end()); | ||
| types.insert(types.end(), TemporalTypes().begin(), TemporalTypes().end()); | ||
| types.push_back(month_interval()); | ||
| for (const auto& ty : types) { | ||
| SCOPED_TRACE(ty->ToString()); | ||
| auto in_schema = schema({field("argument0", ty), field("key", int64())}); | ||
| auto table = TableFromJSON(in_schema, {R"([ | ||
| const std::vector<std::string> default_table = {R"([ | ||
| [1, 1], | ||
| [null, 1] | ||
| ])", | ||
| R"([ | ||
| R"([ | ||
| [0, 2], | ||
| [null, 3], | ||
| [3, 4], | ||
| @@ -1632,11 +1630,54 @@ TEST(GroupBy, MinMaxTypes) { | ||
| [3, 1], | ||
| [0, 2] | ||
| ])", | ||
| R"([ | ||
| R"([ | ||
| [0, 2], | ||
| [1, null], | ||
| [null, 3] | ||
| ])"}); | ||
| ])"}; | ||
| const std::vector<std::string> date64_table = {R"([ | ||
| [86400000, 1], | ||
| [null, 1] | ||
| ])", | ||
| R"([ | ||
| [0, 2], | ||
| [null, 3], | ||
| [259200000, 4], | ||
| [432000000, 4], | ||
| [345600000, null], | ||
| [259200000, 1], | ||
| [0, 2] | ||
| ])", | ||
| R"([ | ||
| [0, 2], | ||
| [86400000, null], | ||
| [null, 3] | ||
| ])"}; | ||
| const std::string default_expected = | ||
| R"([ | ||
| [{"min": 1, "max": 3}, 1], | ||
| [{"min": 0, "max": 0}, 2], | ||
| [{"min": null, "max": null}, 3], | ||
| [{"min": 3, "max": 5}, 4], | ||
| [{"min": 1, "max": 4}, null] | ||
| ])"; | ||
| const std::string date64_expected = | ||
| R"([ | ||
| [{"min": 86400000, "max": 259200000}, 1], | ||
| [{"min": 0, "max": 0}, 2], | ||
| [{"min": null, "max": null}, 3], | ||
| [{"min": 259200000, "max": 432000000}, 4], | ||
| [{"min": 86400000, "max": 345600000}, null] | ||
| ])"; | ||
| for (const auto& ty : types) { | ||
| SCOPED_TRACE(ty->ToString()); | ||
| auto in_schema = schema({field("argument0", ty), field("key", int64())}); | ||
| auto table = | ||
| TableFromJSON(in_schema, (ty->name() == "date64") ? date64_table : default_table); | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: can check | ||
| ASSERT_OK_AND_ASSIGN( | ||
| Datum aggregated_and_grouped, | ||
| @@ -1652,13 +1693,7 @@ TEST(GroupBy, MinMaxTypes) { | ||
| field("hash_min_max", struct_({field("min", ty), field("max", ty)})), | ||
| field("key_0", int64()), | ||
| }), | ||
| R"([ | ||
| [{"min": 1, "max": 3}, 1], | ||
| [{"min": 0, "max": 0}, 2], | ||
| [{"min": null, "max": null}, 3], | ||
| [{"min": 3, "max": 5}, 4], | ||
| [{"min": 1, "max": 4}, null] | ||
| ])"), | ||
| (ty->name() == "date64") ? date64_expected : default_expected), | ||
Member There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: ditto here | ||
| aggregated_and_grouped, | ||
| /*verbose=*/true); | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.
Uh oh!
There was an error while loading. Please reload this page.