GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bkietz merged 9 commits into from
Jun 22, 2023
Merged

GH-32190: [C++][Compute] Implement cumulative prod, max and min functions #36020

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
91e474f
GH-32190: [C++] Implement cumulative product, max, and min compute fu…
js8544 Jun 7, 2023
4bd7ca2
fix python lint
js8544 Jun 7, 2023
439b5fc
fix
js8544 Jun 20, 2023
58c0952
remove identity for decimal; fix doc
js8544 Jun 20, 2023
ce630a5
Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc
js8544 Jun 21, 2023
c61d6cf
Apply suggestions from code review
js8544 Jun 21, 2023
bfc5396
change identity to a trait
js8544 Jun 20, 2023
8611992
Update python/pyarrow/compute.py
js8544 Jun 22, 2023
cee20ec
revert unnecessary change
js8544 Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 29 additions & 10 deletions cpp/src/arrow/compute/api_vector.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
DataMember("k", &SelectKOptions::k),
DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
DataMember("start", &CumulativeSumOptions::start),
DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
DataMember("start", &CumulativeOptions::start),
DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
DataMember("sort_keys", &RankOptions::sort_keys),
DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
: CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
: FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
: FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
: CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
: FunctionOptions(internal::kCumulativeOptionsType),
start(std::move(start)),
skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
bool check_overflow, ExecContext* ctx) {
auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
bool check_overflow, ExecContext* ctx) {
auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
ExecContext* ctx) {
return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
ExecContext* ctx) {
return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down
61 changes: 50 additions & 11 deletions cpp/src/arrow/compute/api_vector.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
public:
explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
static constexpr char const kTypeName[] = "CumulativeSumOptions";
static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

/// Optional starting value for cumulative operation computation
std::shared_ptr<Scalar> start;
explicit CumulativeOptions(bool skip_nulls = false);
explicit CumulativeOptions(double start, bool skip_nulls = false);
explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
static constexpr char const kTypeName[] = "CumulativeOptions";
static CumulativeOptions Defaults() { return CumulativeOptions(); }

/// Optional starting value for cumulative operation computation, default depends on the
/// operation and input type.
/// - sum: 0
/// - prod: 1
/// - min: maximum of the input type
/// - max: minimum of the input type
std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.
Outdated

/// If true, nulls in the input are ignored and produce a corresponding null output.
/// When false, the first null encountered is propagated through the remaining output.
bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
const Datum& values,
const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down
54 changes: 28 additions & 26 deletions cpp/src/arrow/compute/function_internal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
const std::optional<T>& value) {
return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
}
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
-> Result<decltype(MakeScalar(value.value()))> {
return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
const std::shared_ptr<Scalar>& value) {
using value_type = typename T::value_type;
return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
const std::shared_ptr<Scalar>& value) {
using value_type = typename T::value_type;
if (value->type->id() == Type::NA) {
return std::nullopt;
}
return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down
81 changes: 81 additions & 0 deletions cpp/src/arrow/compute/kernels/base_arithmetic_internal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
}
};

struct Max {
template <typename T, typename Arg0, typename Arg1>
static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
Arg1 arg1, Status*) {
static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
return std::max(arg0, arg1);
}

template <typename T, typename Arg0, typename Arg1>
static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
Status*) {
static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.
Outdated
return right;
} else if (std::isnan(right)) {
return left;
} else {
return std::max(left, right);
}
}
};

struct Min {
template <typename T, typename Arg0, typename Arg1>
static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
Arg1 arg1, Status*) {
static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
return std::min(arg0, arg1);
}

template <typename T, typename Arg0, typename Arg1>
static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
Status*) {
static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
if (std::isnan(left)) {
return right;
} else if (std::isnan(right)) {
return left;
} else {
return std::min(left, right);
}
}
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
template <typename Value>
static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
template <typename Value>
static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
template <typename Value>
static constexpr Value value{std::numeric_limits<Value>::min()};
Comment on lines +679 to +680

@bkietzbkietzJun 21, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100

... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();

Multiplication will require more effort since we can't construct 1 without knowing the scale.

@js8544js8544Jun 22, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.

};

template <>
struct Identity<Min> {
template <typename Value>
static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow
3 changes: 3 additions & 0 deletions cpp/src/arrow/compute/kernels/codegen_internal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
GH-32190: [C++][Compute] Implement cumulative prod, max and min functions#36020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

bkietz
merged 9 commits into

from

Jun 22, 2023
 Merged


GH-32190: [C++][Compute] Implement cumulative prod, max and min functions
#36020
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
91e474f

GH-32190: [C++] Implement cumulative product, max, and min compute fu…


js8544 Jun 7, 2023
4bd7ca2

fix python lint


js8544 Jun 7, 2023
439b5fc

fix


js8544 Jun 20, 2023
58c0952

remove identity for decimal; fix doc


js8544 Jun 20, 2023
ce630a5

Update cpp/src/arrow/compute/kernels/vector_cumulative_ops.cc


js8544 Jun 21, 2023
c61d6cf

Apply suggestions from code review


js8544 Jun 21, 2023
bfc5396

change identity to a trait


js8544 Jun 20, 2023
8611992

Update python/pyarrow/compute.py


js8544 Jun 22, 2023
cee20ec

revert unnecessary change


js8544 Jun 21, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


39 changes: 29 additions & 10 deletions
cpp/src/arrow/compute/api_vector.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compute/exec.h"
#include "arrow/compute/function.h"
#include "arrow/compute/function_internal.h"
#include "arrow/compute/kernels/vector_sort_internal.h"
#include "arrow/compute/registry.h"
Expand DownExpand Up@@ -142,9 +143,9 @@ static auto kPartitionNthOptionsType = GetFunctionOptionsType<PartitionNthOption
static auto kSelectKOptionsType = GetFunctionOptionsType<SelectKOptions>(
 DataMember("k", &SelectKOptions::k),
 DataMember("sort_keys", &SelectKOptions::sort_keys));
static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOptions>(
 DataMember("start", &CumulativeSumOptions::start),
 DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls));
static auto kCumulativeOptionsType = GetFunctionOptionsType<CumulativeOptions>(
 DataMember("start", &CumulativeOptions::start),
 DataMember("skip_nulls", &CumulativeOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
 DataMember("sort_keys", &RankOptions::sort_keys),
 DataMember("null_placement", &RankOptions::null_placement),
Expand DownExpand Up@@ -198,13 +199,15 @@ SelectKOptions::SelectKOptions(int64_t k, std::vector<SortKey> sort_keys)
 sort_keys(std::move(sort_keys)) {}
constexpr char SelectKOptions::kTypeName[];

CumulativeSumOptions::CumulativeSumOptions(double start, bool skip_nulls)
 : CumulativeSumOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeSumOptionsType),
CumulativeOptions::CumulativeOptions(bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType), skip_nulls(skip_nulls) {}
CumulativeOptions::CumulativeOptions(double start, bool skip_nulls)
 : CumulativeOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeOptions::CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
 : FunctionOptions(internal::kCumulativeOptionsType),
 start(std::move(start)),
 skip_nulls(skip_nulls) {}
constexpr char CumulativeSumOptions::kTypeName[];
constexpr char CumulativeOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
 RankOptions::Tiebreaker tiebreaker)
Expand All@@ -224,7 +227,7 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
 DCHECK_OK(registry->AddFunctionOptionsType(kSortOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeOptionsType));
 DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand DownExpand Up@@ -375,12 +378,28 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {
// ----------------------------------------------------------------------
// Cumulative functions

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
Result<Datum> CumulativeSum(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_sum_checked" : "cumulative_sum";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProd(const Datum& values, const CumulativeOptions& options,
 bool check_overflow, ExecContext* ctx) {
 auto func_name = check_overflow ? "cumulative_prod_checked" : "cumulative_prod";
 return CallFunction(func_name, {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeOptions& options,
 ExecContext* ctx) {
 return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

61 changes: 50 additions & 11 deletions
cpp/src/arrow/compute/api_vector.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,21 +210,29 @@ class ARROW_EXPORT PartitionNthOptions : public FunctionOptions {
 NullPlacement null_placement;
};

/// \brief Options for cumulative sum function
class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
/// \brief Options for cumulative functions
/// \note Also aliased as CumulativeSumOptions for backward compatibility
class ARROW_EXPORT CumulativeOptions : public FunctionOptions {
 public:
 explicit CumulativeSumOptions(double start = 0, bool skip_nulls = false);
 explicit CumulativeSumOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeSumOptions";
 static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

 /// Optional starting value for cumulative operation computation
 std::shared_ptr<Scalar> start;
 explicit CumulativeOptions(bool skip_nulls = false);
 explicit CumulativeOptions(double start, bool skip_nulls = false);
 explicit CumulativeOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
 static constexpr char const kTypeName[] = "CumulativeOptions";
 static CumulativeOptions Defaults() { return CumulativeOptions(); }

 /// Optional starting value for cumulative operation computation, default depends on the
 /// operation and input type.
 /// - sum: 0
 /// - prod: 1
 /// - min: maximum of the input type
 /// - max: minimum of the input type
 std::optional<std::shared_ptr<Scalar>> start;
Comment thread
bkietz marked this conversation as resolved.


Outdated


 /// If true, nulls in the input are ignored and produce a corresponding null output.
 /// When false, the first null encountered is propagated through the remaining output.
 bool skip_nulls = false;
};
using CumulativeSumOptions = CumulativeOptions; // For backward compatibility

/// @}

Expand DownExpand Up@@ -607,10 +615,41 @@ Result<Datum> RunEndDecode(const Datum& value, ExecContext* ctx = NULLPTR);
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeSum(
 const Datum& values,
 const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative product of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative prod behavior
/// \param[in] check_overflow whether to check for overflow, if true, return Invalid
/// status on overflow, otherwise wrap around on overflow
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeProd(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 bool check_overflow = false, ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative max of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative max behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMax(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

/// \brief Compute the cumulative min of an array-like object
///
/// \param[in] values array-like input
/// \param[in] options configures cumulative min behavior
/// \param[in] ctx the function execution context, optional
ARROW_EXPORT
Result<Datum> CumulativeMin(
 const Datum& values, const CumulativeOptions& options = CumulativeOptions::Defaults(),
 ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down

54 changes: 28 additions & 26 deletions
cpp/src/arrow/compute/function_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
#include "arrow/compute/function.h"
#include "arrow/compute/type_fwd.h"
#include "arrow/result.h"
#include "arrow/scalar.h"
#include "arrow/status.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -283,12 +284,6 @@ static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 return MakeScalar(value);
}

template <typename T>
static inline Result<decltype(MakeScalar(std::declval<T>()))> GenericToScalar(
 const std::optional<T>& value) {
 return value.has_value() ? MakeScalar(value.value()) : MakeScalar("");
}

// For Clang/libc++: when iterating through vector<bool>, we can't
// pass it by reference so the overload above doesn't apply
static inline Result<std::shared_ptr<Scalar>> GenericToScalar(bool value) {
Expand DownExpand Up@@ -382,6 +377,16 @@ static inline Result<std::shared_ptr<Scalar>> GenericToScalar(const Datum& value
 }
}

static inline Result<std::shared_ptr<Scalar>> GenericToScalar(std::nullopt_t) {
 return std::make_shared<NullScalar>();
}

template <typename T>
static inline auto GenericToScalar(const std::optional<T>& value)
 -> Result<decltype(MakeScalar(value.value()))> {
 return value.has_value() ? MakeScalar(value.value()) : std::make_shared<NullScalar>();
}

template <typename T>
static inline enable_if_primitive_ctype<typename CTypeTraits<T>::ArrowType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand All@@ -404,26 +409,6 @@ GenericFromScalar(const std::shared_ptr<Scalar>& value) {
 return ValidateEnumValue<T>(raw_val);
}

template <typename>
constexpr bool is_optional_impl = false;
template <typename T>
constexpr bool is_optional_impl<std::optional<T>> = true;

template <typename T>
using is_optional =
 std::integral_constant<bool, is_optional_impl<std::decay_t<T>> ||
 std::is_same<T, std::nullopt_t>::value>;

template <typename T, typename R = void>
using enable_if_optional = enable_if_t<is_optional<T>::value, Result<T>>;

template <typename T>
static inline enable_if_optional<T> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 return GenericFromScalar<value_type>(value);
}

template <typename T, typename U>
using enable_if_same_result = enable_if_same<T, U, Result<T>>;

Expand DownExpand Up@@ -510,6 +495,23 @@ static inline enable_if_same_result<T, Datum> GenericFromScalar(
 return Status::Invalid("Cannot deserialize Datum from ", value->ToString());
}

template <typename>
constexpr inline bool is_optional_v = false;
template <typename T>
constexpr inline bool is_optional_v<std::optional<T>> = true;
template <>
constexpr inline bool is_optional_v<std::nullopt_t> = true;

template <typename T>
static inline std::enable_if_t<is_optional_v<T>, Result<T>> GenericFromScalar(
 const std::shared_ptr<Scalar>& value) {
 using value_type = typename T::value_type;
 if (value->type->id() == Type::NA) {
 return std::nullopt;
 }
 return GenericFromScalar<value_type>(value);
}

template <typename T>
static enable_if_same<typename CTypeTraits<T>::ArrowType, ListType, Result<T>>
GenericFromScalar(const std::shared_ptr<Scalar>& value) {
Expand Down

81 changes: 81 additions & 0 deletions
cpp/src/arrow/compute/kernels/base_arithmetic_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -17,6 +17,7 @@

#pragma once

#include <limits>
#include "arrow/compute/api_scalar.h"
#include "arrow/compute/kernels/common_internal.h"
#include "arrow/compute/kernels/util_internal.h"
Expand DownExpand Up@@ -605,6 +606,86 @@ struct Sign {
 }
};

struct Max {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::max(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
Comment thread
bkietz marked this conversation as resolved.


Outdated

 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::max(left, right);
 }
 }
};

struct Min {
 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_not_floating_value<T> Call(KernelContext*, Arg0 arg0,
 Arg1 arg1, Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 return std::min(arg0, arg1);
 }

 template <typename T, typename Arg0, typename Arg1>
 static constexpr enable_if_floating_value<T> Call(KernelContext*, Arg0 left, Arg1 right,
 Status*) {
 static_assert(std::is_same<T, Arg0>::value && std::is_same<Arg0, Arg1>::value);
 if (std::isnan(left)) {
 return right;
 } else if (std::isnan(right)) {
 return left;
 } else {
 return std::min(left, right);
 }
 }
};

/// The term identity is from the mathematical notation monoid.
/// For any associative binary operation, identity is defined as:
/// Op(identity, x) = x for all x.
template <typename Op>
struct Identity;

template <>
struct Identity<Add> {
 template <typename Value>
 static constexpr Value value{0};
};

template <>
struct Identity<AddChecked> : Identity<Add> {};

template <>
struct Identity<Multiply> {
 template <typename Value>
 static constexpr Value value{1};
};

template <>
struct Identity<MultiplyChecked> : Identity<Multiply> {};

template <>
struct Identity<Max> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::min()};

Comment on lines

+679
 to 
+680

@bkietzbkietzJun 21, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These cumulative ops aren't currently instantiated for decimal types anyway: https://github.com/apache/arrow/blob/44edc27/cpp/src/arrow/compute/kernels/codegen_internal.h#L1070-L1100
... if you feel like filing a follow up issue for them, it should be fairly straightforward to specify these as specializations:

Suggested change

template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<typenameValue>
static constexpr Valuevalue{std::numeric_limits<Value>::min()};
template<>
static constexpr Decimal128value<Decimal128>=Decimal128::GetMinSentinel();
Multiplication will require more effort since we can't construct 1 without knowing the scale.
@js8544js8544Jun 22, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove them for this pr. Will send another pr for decimals.
};

template <>
struct Identity<Min> {
 template <typename Value>
 static constexpr Value value{std::numeric_limits<Value>::max()};
};

} // namespace internal
} // namespace compute
} // namespace arrow

3 changes: 3 additions & 0 deletions
cpp/src/arrow/compute/kernels/codegen_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -213,6 +213,9 @@ using enable_if_integer_value =
template <typename T, typename R = T>
using enable_if_floating_value = enable_if_t<std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_not_floating_value = enable_if_t<!std::is_floating_point<T>::value, R>;

template <typename T, typename R = T>
using enable_if_decimal_value =
 enable_if_t<std::is_same<Decimal128, T>::value || std::is_same<Decimal256, T>::value,
Expand Down
Loading