Skip to content

ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
westonpace merged 7 commits into from
Oct 3, 2022
Merged

ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator #13782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3ac0674
ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…
westonpace May 10, 2022
f626342
ARROW-17287: Update after rebase
westonpace Sep 26, 2022
0a83cb0
ARROW-17287: Addressing PR review comments
westonpace Sep 26, 2022
7c325b6
ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…
westonpace Sep 26, 2022
641be91
ARROW-17287: Lint. Fix python build.
westonpace Sep 27, 2022
dc6fbf7
ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy
westonpace Sep 27, 2022
a75fbdc
ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…
westonpace Sep 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions cpp/src/arrow/compute/exec/exec_plan.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
ExecContext* exec_context) {
std::shared_ptr<std::shared_ptr<Table>> output_table =
std::make_shared<std::shared_ptr<Table>>();
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
ExecPlan::Make(exec_context));
Declaration with_sink = Declaration::Sequence(
{declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
ExecContext* exec_context) {
return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
Declaration declaration, ExecContext* exec_context) {
return DeclarationToTableAsync(std::move(declaration), exec_context)
.Then([](const std::shared_ptr<Table>& table) {
return TableBatchReader(table).ToRecordBatches();
});
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
Declaration declaration, ExecContext* exec_context) {
return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
ExecContext* exec_context) {
AsyncGenerator<std::optional<ExecBatch>> sink_gen;
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
ExecPlan::Make(exec_context));
Declaration with_sink =
Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
auto collected_fut = CollectAsyncGenerator(sink_gen);
return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
.Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
return ::arrow::internal::MapVector(
[](std::optional<ExecBatch> batch) { return std::move(*batch); },
std::move(collected));
});
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
ExecContext* exec_context) {
return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down
33 changes: 33 additions & 0 deletions cpp/src/arrow/compute/exec/exec_plan.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down
21 changes: 15 additions & 6 deletions cpp/src/arrow/compute/exec/expression.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
if (impl_ == nullptr) return nullptr;

return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
if (impl_ == nullptr) return nullptr;

return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
if (impl_ == nullptr) return nullptr;

return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
return Modify(
return ModifyExpression(
std::move(expr), [](Expression expr) { return expr; },
[](Expression expr, ...) -> Result<Expression> {
auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
"ReplaceFieldsWithKnownValues called on an unbound Expression");
}

return Modify(
return ModifyExpression(
std::move(expr),
[&known_values](Expression expr) -> Result<Expression> {
if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
}
} AlreadyCanonicalized;

return Modify(
return ModifyExpression(
std::move(expr),
[&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
const Expression::Call& guarantee) {
if (guarantee.function_name != "is_valid") return expr;

return Modify(
return ModifyExpression(
std::move(expr), [](Expression expr) { return expr; },
[&](Expression expr, ...) -> Result<Expression> {
auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

if (auto inequality = Inequality::ExtractOne(guarantee)) {
ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
std::move(expr), [](Expression expr) { return expr; },
[&](Expression expr, ...) -> Result<Expression> {
return inequality->Simplify(std::move(expr));
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/compute/exec/expression.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
// XXX someday
// Result<PipelineGraph> GetPipelines();

bool is_valid() const { return impl_ != NULLPTR; }

/// Access a Call or return nullptr if this expression is not a call
const Call* call() const;
/// Access a Datum or return nullptr if this expression is not a literal
Expand Down
47 changes: 0 additions & 47 deletions cpp/src/arrow/compute/exec/expression_internal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
const PostVisitCall& post_call) {
ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

auto call = expr.call();
if (!call) return expr;

bool at_least_one_modified = false;
std::vector<Expression> modified_arguments;

for (size_t i = 0; i < call->arguments.size(); ++i) {
ARROW_ASSIGN_OR_RAISE(auto modified_argument,
Modify(call->arguments[i], pre, post_call));

if (Identical(modified_argument, call->arguments[i])) {
continue;
}

if (!at_least_one_modified) {
modified_arguments = call->arguments;
at_least_one_modified = true;
}

modified_arguments[i] = std::move(modified_argument);
}

if (at_least_one_modified) {
// reconstruct the call expression with the modified arguments
auto modified_call = *call;
modified_call.arguments = std::move(modified_arguments);
return post_call(Expression(std::move(modified_call)), &expr);
}

return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow
7 changes: 3 additions & 4 deletions cpp/src/arrow/compute/exec/filter_node.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
public:
FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
: MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
std::shared_ptr<Schema> output_schema, Expression filter)
: MapNode(plan, std::move(inputs), std::move(output_schema)),
filter_(std::move(filter)) {}

static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
filter_expression.type()->ToString());
}
return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
std::move(filter_expression),
filter_options.async_mode);
std::move(filter_expression));
}

const char* kind_name() const override { return "FilterNode"; }
Expand Down
13 changes: 2 additions & 11 deletions cpp/src/arrow/compute/exec/map_node.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
std::shared_ptr<Schema> output_schema, bool async_mode)
std::shared_ptr<Schema> output_schema)
: ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
std::move(output_schema),
/*num_outputs=*/1) {
if (async_mode) {
executor_ = plan_->exec_context()->executor();
} else {
executor_ = nullptr;
}
}
/*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
EVENT(span_, "StopProducing");
if (executor_) {
this->stop_source_.RequestStop();
}
if (input_counter_.Cancel()) {
this->Finish();
}
Expand Down
7 changes: 1 addition & 6 deletions cpp/src/arrow/compute/exec/map_node.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
public:
MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
std::shared_ptr<Schema> output_schema, bool async_mode);
std::shared_ptr<Schema> output_schema);

void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
protected:
// Counter for the number of batches received
AtomicCounter input_counter_;

::arrow::internal::Executor* executor_;

// Variable used to cancel remaining tasks in the executor
StopSource stop_source_;
};

} // namespace compute
Expand Down
12 changes: 4 additions & 8 deletions cpp/src/arrow/compute/exec/options.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
public:
explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
: filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
explicit FilterNodeOptions(Expression filter_expression)
: filter_expression(std::move(filter_expression)) {}

Expression filter_expression;
bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
public:
explicit ProjectNodeOptions(std::vector<Expression> expressions,
std::vector<std::string> names = {}, bool async_mode = true)
: expressions(std::move(expressions)),
names(std::move(names)),
async_mode(async_mode) {}
std::vector<std::string> names = {})
: expressions(std::move(expressions)), names(std::move(names)) {}

std::vector<Expression> expressions;
std::vector<std::string> names;
bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down
8 changes: 3 additions & 5 deletions cpp/src/arrow/compute/exec/project_node.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
public:
ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
bool async_mode)
: MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
: MapNode(plan, std::move(inputs), std::move(output_schema)),
exprs_(std::move(exprs)) {}

static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
++i;
}
return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
schema(std::move(fields)), std::move(exprs),
project_options.async_mode);
schema(std::move(fields)), std::move(exprs));
}

const char* kind_name() const override { return "ProjectNode"; }
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/arrow/compute/exec/util.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
return Status::OK();
}

Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator by westonpace · Pull Request #13782 · apache/arrow · GitHub
Skip to content
ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator#13782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 7 commits into

from

Oct 3, 2022
 Merged


ARROW-17287: [C++] Create scan node that doesn't rely on the merged generator
#13782
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
3ac0674

ARROW-17287: Initial creation of a "scan node" which doesn't use asyn…


westonpace May 10, 2022
f626342

ARROW-17287: Update after rebase


westonpace Sep 26, 2022
0a83cb0

ARROW-17287: Addressing PR review comments


westonpace Sep 26, 2022
7c325b6

ARROW-17287: Moved Modify to compute/exec/util and renamed to ModifyE…


westonpace Sep 26, 2022
641be91

ARROW-17287: Lint. Fix python build.


westonpace Sep 27, 2022
dc6fbf7

ARROW-17287: Add ARROW_DS_EXPORT to MakeBasicDatasetEvolutionStrategy


westonpace Sep 27, 2022
a75fbdc

ARROW-17287: Fixed end-to-end benchmark to also test scanv2 node. Mo…


westonpace Sep 28, 2022

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


57 changes: 57 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -32,11 +32,13 @@
#include "arrow/datum.h"
#include "arrow/record_batch.h"
#include "arrow/result.h"
#include "arrow/table.h"
#include "arrow/util/async_generator.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/key_value_metadata.h"
#include "arrow/util/logging.h"
#include "arrow/util/tracing_internal.h"
#include "arrow/util/vector.h"

namespace arrow {

Expand DownExpand Up@@ -555,6 +557,61 @@ bool Declaration::IsValid(ExecFactoryRegistry* registry) const {
 return !this->factory_name.empty() && this->options != nullptr;
}

Future<std::shared_ptr<Table>> DeclarationToTableAsync(Declaration declaration,
 ExecContext* exec_context) {
 std::shared_ptr<std::shared_ptr<Table>> output_table =
 std::make_shared<std::shared_ptr<Table>>();
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink = Declaration::Sequence(
 {declaration, {"table_sink", TableSinkNodeOptions(output_table.get())}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 return exec_plan->finished().Then([exec_plan, output_table] { return *output_table; });
}

Result<std::shared_ptr<Table>> DeclarationToTable(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToTableAsync(std::move(declaration), exec_context)
 .Then([](const std::shared_ptr<Table>& table) {
 return TableBatchReader(table).ToRecordBatches();
 });
}

Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context) {
 return DeclarationToBatchesAsync(std::move(declaration), exec_context).result();
}

Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(Declaration declaration,
 ExecContext* exec_context) {
 AsyncGenerator<std::optional<ExecBatch>> sink_gen;
 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ExecPlan> exec_plan,
 ExecPlan::Make(exec_context));
 Declaration with_sink =
 Declaration::Sequence({declaration, {"sink", SinkNodeOptions(&sink_gen)}});
 ARROW_RETURN_NOT_OK(with_sink.AddToPlan(exec_plan.get()));
 ARROW_RETURN_NOT_OK(exec_plan->StartProducing());
 auto collected_fut = CollectAsyncGenerator(sink_gen);
 return AllComplete({exec_plan->finished(), Future<>(collected_fut)})
 .Then([collected_fut, exec_plan]() -> Result<std::vector<ExecBatch>> {
 ARROW_ASSIGN_OR_RAISE(auto collected, collected_fut.result());
 return ::arrow::internal::MapVector(
 [](std::optional<ExecBatch> batch) { return std::move(*batch); },
 std::move(collected));
 });
}

Result<std::vector<ExecBatch>> DeclarationToExecBatches(Declaration declaration,
 ExecContext* exec_context) {
 return DeclarationToExecBatchesAsync(std::move(declaration), exec_context).result();
}

namespace internal {

void RegisterSourceNode(ExecFactoryRegistry*);
Expand Down

33 changes: 33 additions & 0 deletions
cpp/src/arrow/compute/exec/exec_plan.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -483,6 +483,39 @@ struct ARROW_EXPORT Declaration {
 std::string label;
};

/// \brief Utility method to run a declaration and collect the results into a table
///
/// This method will add a sink node to the declaration to collect results into a
/// table. It will then create an ExecPlan from the declaration, start the exec plan,
/// block until the plan has finished, and return the created table.
ARROW_EXPORT Result<std::shared_ptr<Table>> DeclarationToTable(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToTable
ARROW_EXPORT Future<std::shared_ptr<Table>> DeclarationToTableAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into ExecBatch
/// vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<ExecBatch>> DeclarationToExecBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToExecBatches
ARROW_EXPORT Future<std::vector<ExecBatch>> DeclarationToExecBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Utility method to run a declaration and collect the results into a vector
///
/// \see DeclarationToTable for details
ARROW_EXPORT Result<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatches(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Asynchronous version of \see DeclarationToBatches
ARROW_EXPORT Future<std::vector<std::shared_ptr<RecordBatch>>> DeclarationToBatchesAsync(
 Declaration declaration, ExecContext* exec_context = default_exec_context());

/// \brief Wrap an ExecBatch generator in a RecordBatchReader.
///
/// The RecordBatchReader does not impose any ordering on emitted batches.
Expand Down

21 changes: 15 additions & 6 deletions
cpp/src/arrow/compute/exec/expression.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include "arrow/chunked_array.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/exec/expression_internal.h"
#include "arrow/compute/exec/util.h"
#include "arrow/compute/exec_internal.h"
#include "arrow/compute/function_internal.h"
#include "arrow/io/memory.h"
Expand DownExpand Up@@ -77,9 +78,15 @@ Expression call(std::string function, std::vector<Expression> arguments,
 return Expression(std::move(call));
}

const Datum* Expression::literal() const { return std::get_if<Datum>(impl_.get()); }
const Datum* Expression::literal() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Datum>(impl_.get());
}

const Expression::Parameter* Expression::parameter() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Parameter>(impl_.get());
}

Expand All@@ -91,6 +98,8 @@ const FieldRef* Expression::field_ref() const {
}

const Expression::Call* Expression::call() const {
 if (impl_ == nullptr) return nullptr;

 return std::get_if<Call>(impl_.get());
}

Expand DownExpand Up@@ -654,7 +663,7 @@ bool ExpressionHasFieldRefs(const Expression& expr) {
}

Result<Expression> FoldConstants(Expression expr) {
 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [](Expression expr, ...) -> Result<Expression> {
 auto call = CallNotNull(expr);
Expand DownExpand Up@@ -799,7 +808,7 @@ Result<Expression> ReplaceFieldsWithKnownValues(const KnownFieldValues& known_va
 "ReplaceFieldsWithKnownValues called on an unbound Expression");
 }

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&known_values](Expression expr) -> Result<Expression> {
 if (auto ref = expr.field_ref()) {
Expand DownExpand Up@@ -870,7 +879,7 @@ Result<Expression> Canonicalize(Expression expr, compute::ExecContext* exec_cont
 }
 } AlreadyCanonicalized;

 return Modify(
 return ModifyExpression(
 std::move(expr),
 [&AlreadyCanonicalized, exec_context](Expression expr) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1112,7 +1121,7 @@ Result<Expression> SimplifyIsValidGuarantee(Expression expr,
 const Expression::Call& guarantee) {
 if (guarantee.function_name != "is_valid") return expr;

 return Modify(
 return ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 auto call = expr.call();
Expand DownExpand Up@@ -1154,7 +1163,7 @@ Result<Expression> SimplifyWithGuarantee(Expression expr,

 if (auto inequality = Inequality::ExtractOne(guarantee)) {
 ARROW_ASSIGN_OR_RAISE(auto simplified,
Modify(
ModifyExpression(
 std::move(expr), [](Expression expr) { return expr; },
 [&](Expression expr, ...) -> Result<Expression> {
 return inequality->Simplify(std::move(expr));
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/compute/exec/expression.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -100,6 +100,8 @@ class ARROW_EXPORT Expression {
 // XXX someday
 // Result<PipelineGraph> GetPipelines();

 bool is_valid() const { return impl_ != NULLPTR; }

 /// Access a Call or return nullptr if this expression is not a call
 const Call* call() const;
 /// Access a Datum or return nullptr if this expression is not a literal
Expand Down

47 changes: 0 additions & 47 deletions
cpp/src/arrow/compute/exec/expression_internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -287,52 +287,5 @@ inline Result<std::shared_ptr<compute::Function>> GetFunction(
 return GetCastFunction(*to_type);
}

/// Modify an Expression with pre-order and post-order visitation.
/// `pre` will be invoked on each Expression. `pre` will visit Calls before their
/// arguments, `post_call` will visit Calls (and no other Expressions) after their
/// arguments. Visitors should return the Identical expression to indicate no change; this
/// will prevent unnecessary construction in the common case where a modification is not
/// possible/necessary/...
///
/// If an argument was modified, `post_call` visits a reconstructed Call with the modified
/// arguments but also receives a pointer to the unmodified Expression as a second
/// argument. If no arguments were modified the unmodified Expression* will be nullptr.
template <typename PreVisit, typename PostVisitCall>
Result<Expression> Modify(Expression expr, const PreVisit& pre,
 const PostVisitCall& post_call) {
 ARROW_ASSIGN_OR_RAISE(expr, Result<Expression>(pre(std::move(expr))));

 auto call = expr.call();
 if (!call) return expr;

 bool at_least_one_modified = false;
 std::vector<Expression> modified_arguments;

 for (size_t i = 0; i < call->arguments.size(); ++i) {
 ARROW_ASSIGN_OR_RAISE(auto modified_argument,
 Modify(call->arguments[i], pre, post_call));

 if (Identical(modified_argument, call->arguments[i])) {
 continue;
 }

 if (!at_least_one_modified) {
 modified_arguments = call->arguments;
 at_least_one_modified = true;
 }

 modified_arguments[i] = std::move(modified_argument);
 }

 if (at_least_one_modified) {
 // reconstruct the call expression with the modified arguments
 auto modified_call = *call;
 modified_call.arguments = std::move(modified_arguments);
 return post_call(Expression(std::move(modified_call)), &expr);
 }

 return post_call(std::move(expr), nullptr);
}

} // namespace compute
} // namespace arrow

7 changes: 3 additions & 4 deletions
cpp/src/arrow/compute/exec/filter_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -38,8 +38,8 @@ namespace {
class FilterNode : public MapNode {
 public:
 FilterNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, Expression filter, bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, Expression filter)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 filter_(std::move(filter)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand All@@ -61,8 +61,7 @@ class FilterNode : public MapNode {
 filter_expression.type()->ToString());
 }
 return plan->EmplaceNode<FilterNode>(plan, std::move(inputs), std::move(schema),
 std::move(filter_expression),
 filter_options.async_mode);
 std::move(filter_expression));
 }

 const char* kind_name() const override { return "FilterNode"; }
Expand Down

13 changes: 2 additions & 11 deletions
cpp/src/arrow/compute/exec/map_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,16 +34,10 @@ namespace arrow {
namespace compute {

MapNode::MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode)
 std::shared_ptr<Schema> output_schema)
 : ExecNode(plan, std::move(inputs), /*input_labels=*/{"target"},
 std::move(output_schema),
 /*num_outputs=*/1) {
 if (async_mode) {
 executor_ = plan_->exec_context()->executor();
 } else {
 executor_ = nullptr;
 }
}
 /*num_outputs=*/1) {}

void MapNode::ErrorReceived(ExecNode* input, Status error) {
 DCHECK_EQ(input, inputs_[0]);
Expand DownExpand Up@@ -82,9 +76,6 @@ void MapNode::StopProducing(ExecNode* output) {

void MapNode::StopProducing() {
 EVENT(span_, "StopProducing");
 if (executor_) {
 this->stop_source_.RequestStop();
 }
 if (input_counter_.Cancel()) {
 this->Finish();
 }
Expand Down

7 changes: 1 addition & 6 deletions
cpp/src/arrow/compute/exec/map_node.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -45,7 +45,7 @@ namespace compute {
class ARROW_EXPORT MapNode : public ExecNode {
 public:
 MapNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, bool async_mode);
 std::shared_ptr<Schema> output_schema);

 void ErrorReceived(ExecNode* input, Status error) override;

Expand All@@ -69,11 +69,6 @@ class ARROW_EXPORT MapNode : public ExecNode {
 protected:
 // Counter for the number of batches received
 AtomicCounter input_counter_;

 ::arrow::internal::Executor* executor_;

 // Variable used to cancel remaining tasks in the executor
 StopSource stop_source_;
};

} // namespace compute
Expand Down

12 changes: 4 additions & 8 deletions
cpp/src/arrow/compute/exec/options.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -84,11 +84,10 @@ class ARROW_EXPORT TableSourceNodeOptions : public ExecNodeOptions {
/// excluded in the batch emitted by this node.
class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
 public:
 explicit FilterNodeOptions(Expression filter_expression, bool async_mode = true)
 : filter_expression(std::move(filter_expression)), async_mode(async_mode) {}
 explicit FilterNodeOptions(Expression filter_expression)
 : filter_expression(std::move(filter_expression)) {}

 Expression filter_expression;
 bool async_mode;
};

/// \brief Make a node which executes expressions on input batches, producing new batches.
Expand All@@ -100,14 +99,11 @@ class ARROW_EXPORT FilterNodeOptions : public ExecNodeOptions {
class ARROW_EXPORT ProjectNodeOptions : public ExecNodeOptions {
 public:
 explicit ProjectNodeOptions(std::vector<Expression> expressions,
 std::vector<std::string> names = {}, bool async_mode = true)
 : expressions(std::move(expressions)),
 names(std::move(names)),
 async_mode(async_mode) {}
 std::vector<std::string> names = {})
 : expressions(std::move(expressions)), names(std::move(names)) {}

 std::vector<Expression> expressions;
 std::vector<std::string> names;
 bool async_mode;
};

/// \brief Make a node which aggregates input batches, optionally grouped by keys.
Expand Down

8 changes: 3 additions & 5 deletions
cpp/src/arrow/compute/exec/project_node.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -41,9 +41,8 @@ namespace {
class ProjectNode : public MapNode {
 public:
 ProjectNode(ExecPlan* plan, std::vector<ExecNode*> inputs,
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs,
 bool async_mode)
 : MapNode(plan, std::move(inputs), std::move(output_schema), async_mode),
 std::shared_ptr<Schema> output_schema, std::vector<Expression> exprs)
 : MapNode(plan, std::move(inputs), std::move(output_schema)),
 exprs_(std::move(exprs)) {}

 static Result<ExecNode*> Make(ExecPlan* plan, std::vector<ExecNode*> inputs,
Expand DownExpand Up@@ -72,8 +71,7 @@ class ProjectNode : public MapNode {
 ++i;
 }
 return plan->EmplaceNode<ProjectNode>(plan, std::move(inputs),
 schema(std::move(fields)), std::move(exprs),
 project_options.async_mode);
 schema(std::move(fields)), std::move(exprs));
 }

 const char* kind_name() const override { return "ProjectNode"; }
Expand Down

2 changes: 1 addition & 1 deletion
cpp/src/arrow/compute/exec/util.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -399,7 +399,7 @@ Status TableSinkNodeConsumer::Consume(ExecBatch batch) {
}

Future<> TableSinkNodeConsumer::Finish() {
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(batches_));
 ARROW_ASSIGN_OR_RAISE(*out_, Table::FromRecordBatches(schema_, batches_));
 return Status::OK();
}

Expand Down
Loading