ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lidavidm wants to merge 10 commits into from
Closed

ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning #10264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4280ef5
ARROW-12644: [C++][Dataset] URL-decode partition segments
lidavidm May 6, 2021
3ed3c57
ARROW-12644: [Python][Dataset] URL-decode partition segments
lidavidm May 6, 2021
b9f9fbf
ARROW-12644: [R][Dataset] URL-decode partition segments
lidavidm May 7, 2021
b6675a9
ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests
lidavidm May 7, 2021
b9583e8
ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding
lidavidm May 7, 2021
a7c6f60
ARROW-12751: [C++][Python][R] Rename URL-decode option
lidavidm Jun 1, 2021
719913e
ARROW-12644: [C++][Dataset] Rename URL to URI
lidavidm Jun 2, 2021
bc3a2cb
ARROW-12644: [C++][Dataset] Always validate UTF-8
lidavidm Jun 2, 2021
6dff21f
ARROW-12644: [C++][Dataset] Fix MSVC build error
lidavidm Jun 2, 2021
6147fb5
ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error
lidavidm Jun 3, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 128 additions & 18 deletions cpp/src/arrow/dataset/partition.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
auto decoded = internal::UriUnescape(encoded);
if (!util::ValidateUTF8(decoded)) {
return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
encoded);
}
return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
class DefaultPartitioning : public Partitioning {
public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
switch (segment_encoding) {
case SegmentEncoding::None:
os << "SegmentEncoding::None";
break;
case SegmentEncoding::Uri:
os << "SegmentEncoding::Uri";
break;
default:
os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
break;
}
return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
std::vector<compute::Expression> expressions;

for (const Key& key : ParseKeys(path)) {
ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
for (const Key& key : parsed) {
ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
if (expr == compute::literal(true)) continue;
expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
ArrayVector dictionaries,
KeyValuePartitioningOptions options)
: KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
const std::string& path) const {
std::vector<Key> keys;

int i = 0;
for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
if (i >= schema_->num_fields()) break;

keys.push_back({schema_->field(i++)->name(), std::move(segment)});
switch (options_.segment_encoding) {
case SegmentEncoding::None: {
if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
}
keys.push_back({schema_->field(i++)->name(), std::move(segment)});
break;
}
case SegmentEncoding::Uri: {
ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
break;
}
default:
return Status::NotImplemented("Unknown segment encoding: ",
options_.segment_encoding);
}
}

return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
KeyValuePartitioningOptions options;
options.segment_encoding = segment_encoding;
return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
const {
HivePartitioningOptions options;
options.segment_encoding = segment_encoding;
options.null_fallback = null_fallback;
return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
PartitioningFactoryOptions options)
: KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
Reset();
util::InitializeUTF8();
}

std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
if (field_index == field_names_.size()) break;

RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
switch (options_.segment_encoding) {
case SegmentEncoding::None: {
if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
}
RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
break;
}
case SegmentEncoding::Uri: {
ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
break;
}
default:
return Status::NotImplemented("Unknown segment encoding: ",
options_.segment_encoding);
}
}
}

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
// drop fields which aren't in field_names_
auto out_schema = SchemaFromColumnNames(schema, field_names_);

return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
options_.AsPartitioningOptions());
}

private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
const std::string& segment, const HivePartitioningOptions& options) {
auto name_end = string_view(segment).find_first_of('=');
// Not round-trippable
if (name_end == string_view::npos) {
return util::nullopt;
}

// Static method, so we have no better place for it
util::InitializeUTF8();

auto name = segment.substr(0, name_end);
auto value = segment.substr(name_end + 1);
if (value == null_fallback) {
return Key{name, util::nullopt};
std::string value;
switch (options.segment_encoding) {
case SegmentEncoding::None: {
value = segment.substr(name_end + 1);
if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
return Status::Invalid("Partition segment was not valid UTF-8: ", value);
}
break;
}
case SegmentEncoding::Uri: {
auto raw_value = util::string_view(segment).substr(name_end + 1);
ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
break;
}
default:
return Status::NotImplemented("Unknown segment encoding: ",
options.segment_encoding);
}

if (value == options.null_fallback) {
return Key{std::move(name), util::nullopt};
}
return Key{name, value};
return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
const std::string& path) const {
std::vector<Key> keys;

for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
if (auto key = ParseKey(segment, null_fallback_)) {
ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
if (auto key = maybe_key) {
keys.push_back(std::move(*key));
}
}
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
} else if (!values[i]->is_valid) {
// If no key is available just provide a placeholder segment to maintain the
// field_index <-> path nesting relation
segments[i] = name + "=" + null_fallback_;
segments[i] = name + "=" + hive_options_.null_fallback;
} else {
segments[i] = name + "=" + values[i]->ToString();
}
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
public:
explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
: KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
: KeyValuePartitioningFactory(options), options_(std::move(options)) {}

std::string type_name() const override { return "hive"; }

Result<std::shared_ptr<Schema>> Inspect(
const std::vector<std::string>& paths) override {
auto options = options_.AsHivePartitioningOptions();
for (auto path : paths) {
for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
ARROW_ASSIGN_OR_RAISE(auto maybe_key,
HivePartitioning::ParseKey(segment, options));
if (auto key = maybe_key) {
RETURN_NOT_OK(InsertRepr(key->name, key->value));
}
}
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
auto out_schema = SchemaFromColumnNames(schema, field_names_);

return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
}
}

private:
const std::string null_fallback_;
const HivePartitioningFactoryOptions options_;
std::vector<std::string> field_names_;
};

Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning#10264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

lidavidm
wants to merge
10
commits into

from

 Closed


ARROW-12644: [C++][Python][R][Dataset] URL-decode path segments in partitioning
#10264
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
4280ef5

ARROW-12644: [C++][Dataset] URL-decode partition segments


lidavidm May 6, 2021
3ed3c57

ARROW-12644: [Python][Dataset] URL-decode partition segments


lidavidm May 6, 2021
b9f9fbf

ARROW-12644: [R][Dataset] URL-decode partition segments


lidavidm May 7, 2021
b6675a9

ARROW-12644: [Python][R][Dataset] Don't use invalid filenames in tests


lidavidm May 7, 2021
b9583e8

ARROW-12644: [C++][Dataset] Validate UTF8 after URL-decoding


lidavidm May 7, 2021
a7c6f60

ARROW-12751: [C++][Python][R] Rename URL-decode option


lidavidm Jun 1, 2021
719913e

ARROW-12644: [C++][Dataset] Rename URL to URI


lidavidm Jun 2, 2021
bc3a2cb

ARROW-12644: [C++][Dataset] Always validate UTF-8


lidavidm Jun 2, 2021
6dff21f

ARROW-12644: [C++][Dataset] Fix MSVC build error


lidavidm Jun 2, 2021
6147fb5

ARROW-12644: [Python][Dataset] Fix MSVC/Cython build error


lidavidm Jun 3, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


146 changes: 128 additions & 18 deletions
cpp/src/arrow/dataset/partition.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,8 @@
#include "arrow/util/logging.h"
#include "arrow/util/make_unique.h"
#include "arrow/util/string_view.h"
#include "arrow/util/uri.h"
#include "arrow/util/utf8.h"

namespace arrow {

Expand All@@ -46,6 +48,18 @@ using util::string_view;

namespace dataset {

namespace {
/// Apply UriUnescape, then ensure the results are valid UTF-8.
Result<std::string> SafeUriUnescape(util::string_view encoded) {
 auto decoded = internal::UriUnescape(encoded);
 if (!util::ValidateUTF8(decoded)) {
 return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
 encoded);
 }
 return decoded;
}
} // namespace

std::shared_ptr<Partitioning> Partitioning::Default() {
 class DefaultPartitioning : public Partitioning {
 public:
Expand DownExpand Up@@ -158,6 +172,21 @@ Result<Partitioning::PartitionedBatches> KeyValuePartitioning::Partition(
 return out;
}

std::ostream& operator<<(std::ostream& os, SegmentEncoding segment_encoding) {
 switch (segment_encoding) {
 case SegmentEncoding::None:
 os << "SegmentEncoding::None";
 break;
 case SegmentEncoding::Uri:
 os << "SegmentEncoding::Uri";
 break;
 default:
 os << "(invalid SegmentEncoding " << static_cast<int8_t>(segment_encoding) << ")";
 break;
 }
 return os;
}

Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) const {
 ARROW_ASSIGN_OR_RAISE(auto match, FieldRef(key.name).FindOneOrNone(*schema_));
 if (match.empty()) {
Expand DownExpand Up@@ -209,7 +238,8 @@ Result<compute::Expression> KeyValuePartitioning::ConvertKey(const Key& key) con
Result<compute::Expression> KeyValuePartitioning::Parse(const std::string& path) const {
 std::vector<compute::Expression> expressions;

 for (const Key& key : ParseKeys(path)) {
 ARROW_ASSIGN_OR_RAISE(auto parsed, ParseKeys(path));
 for (const Key& key : parsed) {
 ARROW_ASSIGN_OR_RAISE(auto expr, ConvertKey(key));
 if (expr == compute::literal(true)) continue;
 expressions.push_back(std::move(expr));
Expand DownExpand Up@@ -259,15 +289,38 @@ Result<std::string> KeyValuePartitioning::Format(const compute::Expression& expr
 return FormatValues(values);
}

std::vector<KeyValuePartitioning::Key> DirectoryPartitioning::ParseKeys(
DirectoryPartitioning::DirectoryPartitioning(std::shared_ptr<Schema> schema,
 ArrayVector dictionaries,
 KeyValuePartitioningOptions options)
 : KeyValuePartitioning(std::move(schema), std::move(dictionaries), options) {
 util::InitializeUTF8();
}

Result<std::vector<KeyValuePartitioning::Key>> DirectoryPartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 int i = 0;
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (i >= schema_->num_fields()) break;

 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 keys.push_back({schema_->field(i++)->name(), std::move(segment)});
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 keys.push_back({schema_->field(i++)->name(), std::move(decoded)});
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }

 return keys;
Expand DownExpand Up@@ -308,6 +361,20 @@ Result<std::string> DirectoryPartitioning::FormatValues(
 return fs::internal::JoinAbstractPath(std::move(segments));
}

KeyValuePartitioningOptions PartitioningFactoryOptions::AsPartitioningOptions() const {
 KeyValuePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 return options;
}

HivePartitioningOptions HivePartitioningFactoryOptions::AsHivePartitioningOptions()
 const {
 HivePartitioningOptions options;
 options.segment_encoding = segment_encoding;
 options.null_fallback = null_fallback;
 return options;
}

namespace {
class KeyValuePartitioningFactory : public PartitioningFactory {
 protected:
Expand DownExpand Up@@ -430,6 +497,7 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 PartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), field_names_(std::move(field_names)) {
 Reset();
 util::InitializeUTF8();
 }

 std::string type_name() const override { return "schema"; }
Expand All@@ -441,7 +509,23 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (field_index == field_names_.size()) break;

 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 switch (options_.segment_encoding) {
 case SegmentEncoding::None: {
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(segment))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", segment);
 }
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), segment));
 break;
 }
 case SegmentEncoding::Uri: {
 ARROW_ASSIGN_OR_RAISE(auto decoded, SafeUriUnescape(segment));
 RETURN_NOT_OK(InsertRepr(static_cast<int>(field_index++), decoded));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options_.segment_encoding);
 }
 }
 }

Expand All@@ -458,7 +542,8 @@ class DirectoryPartitioningFactory : public KeyValuePartitioningFactory {
 // drop fields which aren't in field_names_
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_);
 return std::make_shared<DirectoryPartitioning>(std::move(out_schema), dictionaries_,
 options_.AsPartitioningOptions());
 }

 private:
Expand All@@ -481,28 +566,50 @@ std::shared_ptr<PartitioningFactory> DirectoryPartitioning::MakeFactory(
 new DirectoryPartitioningFactory(std::move(field_names), options));
}

util::optional<KeyValuePartitioning::Key> HivePartitioning::ParseKey(
 const std::string& segment, const std::string& null_fallback) {
Result<util::optional<KeyValuePartitioning::Key>> HivePartitioning::ParseKey(
 const std::string& segment, const HivePartitioningOptions& options) {
 auto name_end = string_view(segment).find_first_of('=');
 // Not round-trippable
 if (name_end == string_view::npos) {
 return util::nullopt;
 }

 // Static method, so we have no better place for it
 util::InitializeUTF8();

 auto name = segment.substr(0, name_end);
 auto value = segment.substr(name_end + 1);
 if (value == null_fallback) {
 return Key{name, util::nullopt};
 std::string value;
 switch (options.segment_encoding) {
 case SegmentEncoding::None: {
 value = segment.substr(name_end + 1);
 if (ARROW_PREDICT_FALSE(!util::ValidateUTF8(value))) {
 return Status::Invalid("Partition segment was not valid UTF-8: ", value);
 }
 break;
 }
 case SegmentEncoding::Uri: {
 auto raw_value = util::string_view(segment).substr(name_end + 1);
 ARROW_ASSIGN_OR_RAISE(value, SafeUriUnescape(raw_value));
 break;
 }
 default:
 return Status::NotImplemented("Unknown segment encoding: ",
 options.segment_encoding);
 }

 if (value == options.null_fallback) {
 return Key{std::move(name), util::nullopt};
 }
 return Key{name, value};
 return Key{std::move(name), std::move(value)};
}

std::vector<KeyValuePartitioning::Key> HivePartitioning::ParseKeys(
Result<std::vector<KeyValuePartitioning::Key>> HivePartitioning::ParseKeys(
 const std::string& path) const {
 std::vector<Key> keys;

 for (const auto& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key, ParseKey(segment, hive_options_));
 if (auto key = maybe_key) {
 keys.push_back(std::move(*key));
 }
 }
Expand All@@ -521,7 +628,7 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
 } else if (!values[i]->is_valid) {
 // If no key is available just provide a placeholder segment to maintain the
 // field_index <-> path nesting relation
 segments[i] = name + "=" + null_fallback_;
 segments[i] = name + "=" + hive_options_.null_fallback;
 } else {
 segments[i] = name + "=" + values[i]->ToString();
 }
Expand All@@ -533,15 +640,18 @@ Result<std::string> HivePartitioning::FormatValues(const ScalarVector& values) c
class HivePartitioningFactory : public KeyValuePartitioningFactory {
 public:
 explicit HivePartitioningFactory(HivePartitioningFactoryOptions options)
 : KeyValuePartitioningFactory(options), null_fallback_(options.null_fallback) {}
 : KeyValuePartitioningFactory(options), options_(std::move(options)) {}

 std::string type_name() const override { return "hive"; }

 Result<std::shared_ptr<Schema>> Inspect(
 const std::vector<std::string>& paths) override {
 auto options = options_.AsHivePartitioningOptions();
 for (auto path : paths) {
 for (auto&& segment : fs::internal::SplitAbstractPath(path)) {
 if (auto key = HivePartitioning::ParseKey(segment, null_fallback_)) {
 ARROW_ASSIGN_OR_RAISE(auto maybe_key,
 HivePartitioning::ParseKey(segment, options));
 if (auto key = maybe_key) {
 RETURN_NOT_OK(InsertRepr(key->name, key->value));
 }
 }
Expand All@@ -565,12 +675,12 @@ class HivePartitioningFactory : public KeyValuePartitioningFactory {
 auto out_schema = SchemaFromColumnNames(schema, field_names_);

 return std::make_shared<HivePartitioning>(std::move(out_schema), dictionaries_,
null_fallback_);
options_.AsHivePartitioningOptions());
 }
 }

 private:
 const std::string null_fallback_;
 const HivePartitioningFactoryOptions options_;
 std::vector<std::string> field_names_;
};

Expand Down
Loading