ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rok wants to merge 10 commits into from
Closed

ARROW-13174: [C++][Compute] Add strftime kernel #10647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0e8f2fa
Add strftime kernel.
rok Jun 27, 2021
1679aa6
Review feedback.
rok Jul 13, 2021
3ed11c6
Apply suggestions from code review
rok Jul 16, 2021
d44625b
Adding locale option.
rok Jul 20, 2021
00d685d
Removing R logic.
rok Jul 26, 2021
2c9970a
Changing locale instantiation.
rok Jul 27, 2021
5ca3f22
Review feedback.
rok Jul 28, 2021
91600d9
Refactor MakeTemporal.
rok Aug 6, 2021
989d2dd
Various changes and improvements
pitrou Aug 18, 2021
2ab5b5c
Fix std::locale construction
pitrou Aug 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions cpp/src/arrow/compute/api_scalar.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
DataMember("format", &StrptimeOptions::format),
DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
: FunctionOptions(internal::kStrftimeOptionsType),
format(std::move(format)),
locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
: FunctionOptions(internal::kPadOptionsType),
width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow
30 changes: 30 additions & 0 deletions cpp/src/arrow/compute/api_scalar.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
public:
explicit StrftimeOptions(std::string format, std::string locale = "C");
StrftimeOptions();

constexpr static char const kTypeName[] = "StrftimeOptions";

constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

/// The desired format string.
std::string format;
/// The desired output locale string.
std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
public:
explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow
1 change: 1 addition & 0 deletions cpp/src/arrow/compute/function_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
options.emplace_back(new PadOptions(5, " "));
options.emplace_back(new PadOptions(10, "A"));
options.emplace_back(new TrimOptions(" "));
Expand Down
182 changes: 157 additions & 25 deletions cpp/src/arrow/compute/kernels/scalar_temporal.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
field("iso_day_of_week", int64())});
return type;
}

const std::string& GetInputTimezone(const DataType& type) {
return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
try {
return locate_zone(timezone);
} catch (const std::runtime_error& ex) {
return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
}
}

Result<std::locale> GetLocale(const std::string& locale) {
try {
return std::locale(locale.c_str());
} catch (const std::runtime_error& ex) {
return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
}
}

struct NonZonedLocalizer {
// No-op conversions: UTC -> UTC
template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
op};
return kernel.Exec(ctx, batch, out);
} else {
const time_zone* tz;
try {
tz = locate_zone(timezone);
} catch (const std::runtime_error& ex) {
return Status::Invalid(ex.what());
}
ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
using ExecTemplate = Op<Duration, ZonedLocalizer>;
auto op = ExecTemplate(options, ZonedLocalizer{tz});
applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
}
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
const StrftimeOptions& options;
const time_zone* tz;
const std::locale locale;

static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
const StrftimeOptions& options = StrftimeState::Get(ctx);

const auto& timezone = GetInputTimezone(type);
if (timezone.empty()) {
return Status::Invalid(
"Timestamps without a time zone cannot be reliably formatted.");
}
ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

return Strftime{options, tz, std::move(locale)};
}

static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
TimestampFormatter formatter{self.options.format, self.tz, self.locale};

if (in.is_valid) {
const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
} else {
out->is_valid = false;
}
return Status::OK();
}

static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
TimestampFormatter formatter{self.options.format, self.tz, self.locale};

StringBuilder string_builder;
// Presize string data using a heuristic
{
ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
RETURN_NOT_OK(string_builder.Reserve(in.length));
RETURN_NOT_OK(
string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
}

auto visit_null = [&]() { return string_builder.AppendNull(); };
auto visit_value = [&](int64_t arg) {
ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
return string_builder.Append(std::move(formatted));
};
RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

std::shared_ptr<Array> out_array;
RETURN_NOT_OK(string_builder.Finish(&out_array));
*out = *std::move(out_array->data());

return Status::OK();
}

struct TimestampFormatter {
const char* format;
const time_zone* tz;
std::ostringstream bufstream;

explicit TimestampFormatter(const std::string& format, const time_zone* tz,
const std::locale& locale)
: format(format.c_str()), tz(tz) {
bufstream.imbue(locale);
// Propagate errors as C++ exceptions (to get an actual error message)
bufstream.exceptions(std::ios::failbit | std::ios::badbit);
}

Result<std::string> operator()(int64_t arg) {
bufstream.str("");
const auto zt = arrow_vendored::date::zoned_time<Duration>{
tz, sys_time<Duration>(Duration{arg})};
try {
arrow_vendored::date::to_stream(bufstream, format, zt);
} catch (const std::runtime_error& ex) {
bufstream.clear();
return Status::Invalid("Failed formatting timestamp: ", ex.what());
}
// XXX could return a view with std::ostringstream::view() (C++20)
return std::move(bufstream).str();
}
};
};

#else
template <typename Duration>
struct Strftime {
static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
return Status::NotImplemented("Strftime not yet implemented on windows.");
}
static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
return Status::NotImplemented("Strftime not yet implemented on windows.");
}
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
if (timezone.empty()) {
iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
} else {
try {
const time_zone* tz = locate_zone(timezone);
iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
} catch (const std::runtime_error& ex) {
return Status::Invalid(ex.what());
}
ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
}
ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
};
RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
} else {
try {
const time_zone* tz = locate_zone(timezone);
auto visit_value = [&](int64_t arg) {
const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
field_builders[0]->UnsafeAppend(iso_calendar[0]);
field_builders[1]->UnsafeAppend(iso_calendar[1]);
field_builders[2]->UnsafeAppend(iso_calendar[2]);
return struct_builder->Append();
};
RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
} catch (const std::runtime_error& ex) {
return Status::Invalid(ex.what());
}
ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
auto visit_value = [&](int64_t arg) {
const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
field_builders[0]->UnsafeAppend(iso_calendar[0]);
field_builders[1]->UnsafeAppend(iso_calendar[1]);
field_builders[2]->UnsafeAppend(iso_calendar[2]);
return struct_builder->Append();
};
RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
}

std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
"Returns an error if timestamp has a defined timezone. Null values return null."),
{"values"}};

const FunctionDoc strftime_doc{
"Format timestamps according to a format string",
("For each input timestamp, emit a formatted string.\n"
"The time format string and locale can be set using StrftimeOptions.\n"
"An error is returned if the timestamps don't have a defined timezone,\n"
"or if the timezone cannot be found in the timezone database."),
{"timestamps"},
"StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
"subsecond", float64(), &subsecond_doc);
DCHECK_OK(registry->AddFunction(std::move(subsecond)));

static auto default_strftime_options = StrftimeOptions();
auto strftime = MakeSimpleUnaryTemporal<Strftime>(
"strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-13174: [C++][Compute] Add strftime kernel#10647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

rok
wants to merge
10
commits into

from

 Closed


ARROW-13174: [C++][Compute] Add strftime kernel
#10647
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
0e8f2fa

Add strftime kernel.


rok Jun 27, 2021
1679aa6

Review feedback.


rok Jul 13, 2021
3ed11c6

Apply suggestions from code review


rok Jul 16, 2021
d44625b

Adding locale option.


rok Jul 20, 2021
00d685d

Removing R logic.


rok Jul 26, 2021
2c9970a

Changing locale instantiation.


rok Jul 27, 2021
5ca3f22

Review feedback.


rok Jul 28, 2021
91600d9

Refactor MakeTemporal.


rok Aug 6, 2021
989d2dd

Various changes and improvements


pitrou Aug 18, 2021
2ab5b5c

Fix std::locale construction


pitrou Aug 18, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


15 changes: 15 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -145,6 +145,8 @@ static auto kSetLookupOptionsType = GetFunctionOptionsType<SetLookupOptions>(
static auto kStrptimeOptionsType = GetFunctionOptionsType<StrptimeOptions>(
 DataMember("format", &StrptimeOptions::format),
 DataMember("unit", &StrptimeOptions::unit));
static auto kStrftimeOptionsType = GetFunctionOptionsType<StrftimeOptions>(
 DataMember("format", &StrftimeOptions::format));
static auto kPadOptionsType = GetFunctionOptionsType<PadOptions>(
 DataMember("width", &PadOptions::width), DataMember("padding", &PadOptions::padding));
static auto kTrimOptionsType = GetFunctionOptionsType<TrimOptions>(
Expand DownExpand Up@@ -238,6 +240,14 @@ StrptimeOptions::StrptimeOptions(std::string format, TimeUnit::type unit)
StrptimeOptions::StrptimeOptions() : StrptimeOptions("", TimeUnit::SECOND) {}
constexpr char StrptimeOptions::kTypeName[];

StrftimeOptions::StrftimeOptions(std::string format, std::string locale)
 : FunctionOptions(internal::kStrftimeOptionsType),
 format(std::move(format)),
 locale(std::move(locale)) {}
StrftimeOptions::StrftimeOptions() : StrftimeOptions(kDefaultFormat) {}
constexpr char StrftimeOptions::kTypeName[];
constexpr const char* StrftimeOptions::kDefaultFormat;

PadOptions::PadOptions(int64_t width, std::string padding)
 : FunctionOptions(internal::kPadOptionsType),
 width(width),
Expand DownExpand Up@@ -294,6 +304,7 @@ void RegisterScalarOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kExtractRegexOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSetLookupOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrptimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kStrftimeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPadOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kTrimOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSliceOptionsType));
Expand DownExpand Up@@ -495,5 +506,9 @@ Result<Datum> DayOfWeek(const Datum& arg, DayOfWeekOptions options, ExecContext*
 return CallFunction("day_of_week", {arg}, &options, ctx);
}

Result<Datum> Strftime(const Datum& arg, StrftimeOptions options, ExecContext* ctx) {
 return CallFunction("strftime", {arg}, &options, ctx);
}

} // namespace compute
} // namespace arrow

30 changes: 30 additions & 0 deletions
cpp/src/arrow/compute/api_scalar.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,21 @@ class ARROW_EXPORT StrptimeOptions : public FunctionOptions {
 TimeUnit::type unit;
};

class ARROW_EXPORT StrftimeOptions : public FunctionOptions {
 public:
 explicit StrftimeOptions(std::string format, std::string locale = "C");
 StrftimeOptions();

 constexpr static char const kTypeName[] = "StrftimeOptions";

 constexpr static const char* kDefaultFormat = "%Y-%m-%dT%H:%M:%SZ";

 /// The desired format string.
 std::string format;
 /// The desired output locale string.
 std::string locale;
};

class ARROW_EXPORT PadOptions : public FunctionOptions {
 public:
 explicit PadOptions(int64_t width, std::string padding = " ");
Expand DownExpand Up@@ -999,5 +1014,20 @@ Result<Datum> Nanosecond(const Datum& values, ExecContext* ctx = NULLPTR);
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Subsecond(const Datum& values, ExecContext* ctx = NULLPTR);

/// \brief Format timestamps according to a format string
///
/// Return formatted time strings according to the format string
/// `StrftimeOptions::format` and to the locale specifier `Strftime::locale`.
///
/// \param[in] values input timestamps
/// \param[in] options for setting format string and locale
/// \param[in] ctx the function execution context, optional
/// \return the resulting datum
///
/// \since 6.0.0
/// \note API not yet finalized
ARROW_EXPORT Result<Datum> Strftime(const Datum& values, StrftimeOptions options,
 ExecContext* ctx = NULLPTR);

} // namespace compute
} // namespace arrow

1 change: 1 addition & 0 deletions
cpp/src/arrow/compute/function_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -79,6 +79,7 @@ TEST(FunctionOptions, Equality) {
 options.emplace_back(new SetLookupOptions(ArrayFromJSON(boolean(), "[true, false]")));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::MILLI));
 options.emplace_back(new StrptimeOptions("%Y", TimeUnit::type::NANO));
 options.emplace_back(new StrftimeOptions("%Y-%m-%dT%H:%M:%SZ", "C"));
 options.emplace_back(new PadOptions(5, " "));
 options.emplace_back(new PadOptions(10, "A"));
 options.emplace_back(new TrimOptions(" "));
Expand Down

182 changes: 157 additions & 25 deletions
cpp/src/arrow/compute/kernels/scalar_temporal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.

#include <sstream>

#include "arrow/builder.h"
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common.h"
Expand DownExpand Up@@ -55,13 +57,18 @@ using internal::applicator::ScalarUnaryNotNull;
using internal::applicator::SimpleUnary;

using DayOfWeekState = OptionsWrapper<DayOfWeekOptions>;
using StrftimeState = OptionsWrapper<StrftimeOptions>;

const std::shared_ptr<DataType>& IsoCalendarType() {
 static auto type = struct_({field("iso_year", int64()), field("iso_week", int64()),
 field("iso_day_of_week", int64())});
 return type;
}

const std::string& GetInputTimezone(const DataType& type) {
 return checked_cast<const TimestampType&>(type).timezone();
}

const std::string& GetInputTimezone(const Datum& datum) {
 return checked_cast<const TimestampType&>(*datum.type()).timezone();
}
Expand All@@ -74,6 +81,22 @@ const std::string& GetInputTimezone(const ArrayData& array) {
 return checked_cast<const TimestampType&>(*array.type).timezone();
}

Result<const time_zone*> LocateZone(const std::string& timezone) {
 try {
 return locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot locate timezone '", timezone, "': ", ex.what());
 }
}

Result<std::locale> GetLocale(const std::string& locale) {
 try {
 return std::locale(locale.c_str());
 } catch (const std::runtime_error& ex) {
 return Status::Invalid("Cannot find locale '", locale, "': ", ex.what());
 }
}

struct NonZonedLocalizer {
 // No-op conversions: UTC -> UTC
 template <typename Duration>
Expand DownExpand Up@@ -117,12 +140,7 @@ struct TemporalComponentExtractBase {
 op};
 return kernel.Exec(ctx, batch, out);
 } else {
 const time_zone* tz;
 try {
 tz = locate_zone(timezone);
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 using ExecTemplate = Op<Duration, ZonedLocalizer>;
 auto op = ExecTemplate(options, ZonedLocalizer{tz});
 applicator::ScalarUnaryNotNullStateful<OutType, TimestampType, ExecTemplate> kernel{
Expand DownExpand Up@@ -444,6 +462,114 @@ struct Nanosecond {
 }
};

// ----------------------------------------------------------------------
// Convert timestamps to a string representation with an arbitrary format

#ifndef _WIN32
template <typename Duration>
struct Strftime {
 const StrftimeOptions& options;
 const time_zone* tz;
 const std::locale locale;

 static Result<Strftime> Make(KernelContext* ctx, const DataType& type) {
 const StrftimeOptions& options = StrftimeState::Get(ctx);

 const auto& timezone = GetInputTimezone(type);
 if (timezone.empty()) {
 return Status::Invalid(
 "Timestamps without a time zone cannot be reliably formatted.");
 }
 ARROW_ASSIGN_OR_RAISE(const time_zone* tz, LocateZone(timezone));

 ARROW_ASSIGN_OR_RAISE(std::locale locale, GetLocale(options.locale));

 return Strftime{options, tz, std::move(locale)};
 }

 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 if (in.is_valid) {
 const int64_t in_val = internal::UnboxScalar<const TimestampType>::Unbox(in);
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(in_val));
 checked_cast<StringScalar*>(out)->value = Buffer::FromString(std::move(formatted));
 } else {
 out->is_valid = false;
 }
 return Status::OK();
 }

 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 ARROW_ASSIGN_OR_RAISE(auto self, Make(ctx, *in.type));
 TimestampFormatter formatter{self.options.format, self.tz, self.locale};

 StringBuilder string_builder;
 // Presize string data using a heuristic
 {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(42));
 const auto string_size = static_cast<int64_t>(ceil(formatted.size() * 1.1));
 RETURN_NOT_OK(string_builder.Reserve(in.length));
 RETURN_NOT_OK(
 string_builder.ReserveData((in.length - in.GetNullCount()) * string_size));
 }

 auto visit_null = [&]() { return string_builder.AppendNull(); };
 auto visit_value = [&](int64_t arg) {
 ARROW_ASSIGN_OR_RAISE(auto formatted, formatter(arg));
 return string_builder.Append(std::move(formatted));
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));

 std::shared_ptr<Array> out_array;
 RETURN_NOT_OK(string_builder.Finish(&out_array));
 *out = *std::move(out_array->data());

 return Status::OK();
 }

 struct TimestampFormatter {
 const char* format;
 const time_zone* tz;
 std::ostringstream bufstream;

 explicit TimestampFormatter(const std::string& format, const time_zone* tz,
 const std::locale& locale)
 : format(format.c_str()), tz(tz) {
 bufstream.imbue(locale);
 // Propagate errors as C++ exceptions (to get an actual error message)
 bufstream.exceptions(std::ios::failbit | std::ios::badbit);
 }

 Result<std::string> operator()(int64_t arg) {
 bufstream.str("");
 const auto zt = arrow_vendored::date::zoned_time<Duration>{
 tz, sys_time<Duration>(Duration{arg})};
 try {
 arrow_vendored::date::to_stream(bufstream, format, zt);
 } catch (const std::runtime_error& ex) {
 bufstream.clear();
 return Status::Invalid("Failed formatting timestamp: ", ex.what());
 }
 // XXX could return a view with std::ostringstream::view() (C++20)
 return std::move(bufstream).str();
 }
 };
};

#else
template <typename Duration>
struct Strftime {
 static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
 static Status Call(KernelContext* ctx, const ArrayData& in, ArrayData* out) {
 return Status::NotImplemented("Strftime not yet implemented on windows.");
 }
};
#endif

// ----------------------------------------------------------------------
// Extract ISO calendar values from timestamp

Expand DownExpand Up@@ -473,12 +599,8 @@ struct ISOCalendar {
 if (timezone.empty()) {
 iso_calendar = GetIsoCalendar<Duration>(in_val, NonZonedLocalizer{});
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 iso_calendar = GetIsoCalendar<Duration>(in_val, ZonedLocalizer{tz});
 }
 ScalarVector values = {std::make_shared<Int64Scalar>(iso_calendar[0]),
 std::make_shared<Int64Scalar>(iso_calendar[1]),
Expand DownExpand Up@@ -518,19 +640,15 @@ struct ISOCalendar {
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } else {
 try {
 const time_zone* tz = locate_zone(timezone);
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 } catch (const std::runtime_error& ex) {
 return Status::Invalid(ex.what());
 }
 ARROW_ASSIGN_OR_RAISE(auto tz, LocateZone(timezone));
 auto visit_value = [&](int64_t arg) {
 const auto iso_calendar = GetIsoCalendar<Duration>(arg, ZonedLocalizer{tz});
 field_builders[0]->UnsafeAppend(iso_calendar[0]);
 field_builders[1]->UnsafeAppend(iso_calendar[1]);
 field_builders[2]->UnsafeAppend(iso_calendar[2]);
 return struct_builder->Append();
 };
 RETURN_NOT_OK(VisitArrayDataInline<Int64Type>(in, visit_value, visit_null));
 }

 std::shared_ptr<Array> out_array;
Expand DownExpand Up@@ -713,6 +831,15 @@ const FunctionDoc subsecond_doc{
 "Returns an error if timestamp has a defined timezone. Null values return null."),
 {"values"}};

const FunctionDoc strftime_doc{
 "Format timestamps according to a format string",
 ("For each input timestamp, emit a formatted string.\n"
 "The time format string and locale can be set using StrftimeOptions.\n"
 "An error is returned if the timestamps don't have a defined timezone,\n"
 "or if the timezone cannot be found in the timezone database."),
 {"timestamps"},
 "StrftimeOptions"};

} // namespace

void RegisterScalarTemporal(FunctionRegistry* registry) {
Expand DownExpand Up@@ -782,6 +909,11 @@ void RegisterScalarTemporal(FunctionRegistry* registry) {
 auto subsecond = MakeTemporal<Subsecond, TemporalComponentExtract, DoubleType>(
 "subsecond", float64(), &subsecond_doc);
 DCHECK_OK(registry->AddFunction(std::move(subsecond)));

 static auto default_strftime_options = StrftimeOptions();
 auto strftime = MakeSimpleUnaryTemporal<Strftime>(
 "strftime", utf8(), &strftime_doc, &default_strftime_options, StrftimeState::Init);
 DCHECK_OK(registry->AddFunction(std::move(strftime)));
}

} // namespace internal
Expand Down
Loading