GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
westonpace merged 8 commits into from
May 4, 2023
Merged

GH-34386: [C++] Add a PathFromUriOrPath method #34420

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f33aba1
Add a PathFromUriOrPath method to help convert URIs to paths after yo…
westonpace Mar 3, 2023
d8d7fc8
Add ARROW_EXPORT for Windows
westonpace Mar 3, 2023
7758b09
Moved PathFromUriOrPath into an abstract filesystem method
westonpace May 2, 2023
2f2024a
Simplified the URI parsing logic by combining into one helper method
westonpace May 2, 2023
2f5673d
Make sure to normalize slashes if the user passes in a path and not a…
westonpace May 2, 2023
7fe9d2b
Change behavior to always remove trailing slash
westonpace May 3, 2023
275549a
Add an option to RemoveTrailingSlash to keep / as / and then set that…
westonpace May 3, 2023
c89e9ca
Add a check on Windows for remove trailing slash to handle C:/ style …
westonpace May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 15 additions & 18 deletions cpp/src/arrow/filesystem/filesystem.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
latencies_->Sleep();
return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
Uri uri;
auto status = uri.Parse(uri_string);
if (!status.ok()) {
#ifdef _WIN32
// Could be a "file:..." URI with backslashes instead of regular slashes.
RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
if (uri.scheme() != "file") {
return status;
}
#else
return status;
#endif
}
return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
const std::string& uri_string,
const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
if (internal::DetectAbsolutePath(uri_string)) {
// Normalize path separators
if (out_path != nullptr) {
*out_path = ToSlashes(uri_string);
*out_path =
std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
}
return std::make_shared<LocalFileSystem>();
}
Expand Down
22 changes: 22 additions & 0 deletions cpp/src/arrow/filesystem/filesystem.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
/// may allow normalizing irregular path forms (such as Windows local paths).
virtual Result<std::string> NormalizePath(std::string path);

/// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
/// path
///
/// \param uri_string A URI representing a resource in the given filesystem.
///
/// This method will check to ensure the given filesystem is compatible with the
/// URI. This can be useful when the user provides both a URI and a filesystem or
/// when a user provides multiple URIs that should be compatible with the same
/// filesystem.
///
/// uri_string can be an absolute path instead of a URI. In that case it will ensure
/// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
/// is capable of reading local paths) and will normalize the path's file separators.
///
/// Note, this method only checks to ensure the URI scheme is valid. It will not detect
/// inconsistencies like a mismatching region or endpoint override.
///
/// \return The path inside the filesystem that is indicated by the URI.
virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

virtual bool Equals(const FileSystem& other) const = 0;

virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

Result<std::string> NormalizePath(std::string path) override;
Result<std::string> PathFromUri(const std::string& uri_string) const override;

bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

std::string type_name() const override { return "slow"; }
bool Equals(const FileSystem& other) const override;
Result<std::string> PathFromUri(const std::string& uri_string) const override;

using FileSystem::GetFileInfo;
Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down
6 changes: 6 additions & 0 deletions cpp/src/arrow/filesystem/gcsfs.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
/*accept_local_paths=*/false,
internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
return impl_->GetFileInfo(p);
Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/filesystem/gcsfs.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
const GcsOptions& options() const;

bool Equals(const FileSystem& other) const override;
Result<std::string> PathFromUri(const std::string& uri_string) const override;

Result<FileInfo> GetFileInfo(const std::string& path) override;
Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down
17 changes: 15 additions & 2 deletions cpp/src/arrow/filesystem/gcsfs_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
// Smoke test for FileSystemFromUri
ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
PreexistingBucketPath()));
std::string path;
ASSERT_OK_AND_ASSIGN(
auto fs,
FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
EXPECT_EQ(fs->type_name(), "gcs");
EXPECT_EQ(path, PreexistingBucketName());
ASSERT_OK_AND_ASSIGN(
path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
EXPECT_EQ(path, PreexistingBucketName());
ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
PreexistingBucketPath()));
EXPECT_EQ(fs2->type_name(), "gcs");
ASSERT_THAT(fs->PathFromUri("/foo/bar"),
Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
ASSERT_THAT(
fs->PathFromUri("s3:///foo/bar"),
Raises(StatusCode::Invalid,
testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down
6 changes: 6 additions & 0 deletions cpp/src/arrow/filesystem/hdfs.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
/*accept_local_paths=*/false,
internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
return impl_->GetFileInfo(select);
}
Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/filesystem/hdfs.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
std::string type_name() const override { return "hdfs"; }
HdfsOptions options() const;
bool Equals(const FileSystem& other) const override;
Result<std::string> PathFromUri(const std::string& uri_string) const override;

/// \cond FALSE
using FileSystem::GetFileInfo;
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/filesystem/hdfs_test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
ARROW_LOG(INFO) << "!!! uri = " << ss.str();
ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
ASSERT_EQ(path, "/");
ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
ASSERT_EQ(path, "/");

// Sanity check
ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down
57 changes: 21 additions & 36 deletions cpp/src/arrow/filesystem/localfs.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
// Can't use locale-dependent functions from the C/C++ stdlib
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
// Is it a /-prefixed local path?
if (s.length() >= 1 && s[0] == '/') {
return true;
}
#ifdef _WIN32
// Is it a \-prefixed local path?
if (s.length() >= 1 && s[0] == '\\') {
return true;
}
// Does it start with a drive letter in addition to being /- or \-prefixed,
// e.g. "C:\..."?
if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
IsDriveLetter(s[0])) {
return true;
}
#endif
return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
RETURN_NOT_OK(ValidatePath(path));
ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
std::stringstream ss;
ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
*out_path = ss.str();
*out_path =
std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
uri.ToString(), "'");
#endif
} else {
*out_path = uri.path();
*out_path =
std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
}

// TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
RETURN_NOT_OK(ValidatePath(path));
ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
return fn.ToString();
return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down
9 changes: 1 addition & 8 deletions cpp/src/arrow/filesystem/localfs.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
std::string type_name() const override { return "local"; }

Result<std::string> NormalizePath(std::string path) override;
Result<std::string> PathFromUri(const std::string& uri_string) const override;

bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
GH-34386: [C++] Add a PathFromUriOrPath method#34420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

westonpace
merged 8 commits into

from

May 4, 2023
 Merged


GH-34386: [C++] Add a PathFromUriOrPath method
#34420
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


8 commits


Select commit
Hold shift + click to select a range
f33aba1

Add a PathFromUriOrPath method to help convert URIs to paths after yo…


westonpace Mar 3, 2023
d8d7fc8

Add ARROW_EXPORT for Windows


westonpace Mar 3, 2023
7758b09

Moved PathFromUriOrPath into an abstract filesystem method


westonpace May 2, 2023
2f2024a

Simplified the URI parsing logic by combining into one helper method


westonpace May 2, 2023
2f5673d

Make sure to normalize slashes if the user passes in a path and not a…


westonpace May 2, 2023
7fe9d2b

Change behavior to always remove trailing slash


westonpace May 3, 2023
275549a

Add an option to RemoveTrailingSlash to keep / as / and then set that…


westonpace May 3, 2023
c89e9ca

Add a check on Windows for remove trailing slash to handle C:/ style …


westonpace May 3, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


33 changes: 15 additions & 18 deletions
cpp/src/arrow/filesystem/filesystem.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -60,6 +60,7 @@ using internal::ConcatAbstractPath;
using internal::EnsureTrailingSlash;
using internal::GetAbstractPathParent;
using internal::kSep;
using internal::ParseFileSystemUri;
using internal::RemoveLeadingSlash;
using internal::RemoveTrailingSlash;
using internal::ToSlashes;
Expand DownExpand Up@@ -254,6 +255,10 @@ Result<std::shared_ptr<io::OutputStream>> FileSystem::OpenAppendStream(
 return OpenAppendStream(path, std::shared_ptr<const KeyValueMetadata>{});
}

Result<std::string> FileSystem::PathFromUri(const std::string& uri_string) const {
 return Status::NotImplemented("PathFromUri is not yet supported on this filesystem");
}

//////////////////////////////////////////////////////////////////////////
// SubTreeFileSystem implementation

Expand DownExpand Up@@ -484,6 +489,10 @@ Result<std::shared_ptr<io::OutputStream>> SubTreeFileSystem::OpenAppendStream(
 return base_fs_->OpenAppendStream(real_path, metadata);
}

Result<std::string> SubTreeFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

//////////////////////////////////////////////////////////////////////////
// SlowFileSystem implementation

Expand All@@ -505,6 +514,10 @@ SlowFileSystem::SlowFileSystem(std::shared_ptr<FileSystem> base_fs,

bool SlowFileSystem::Equals(const FileSystem& other) const { return this == &other; }

Result<std::string> SlowFileSystem::PathFromUri(const std::string& uri_string) const {
 return base_fs_->PathFromUri(uri_string);
}

Result<FileInfo> SlowFileSystem::GetFileInfo(const std::string& path) {
 latencies_->Sleep();
 return base_fs_->GetFileInfo(path);
Expand DownExpand Up@@ -662,23 +675,6 @@ Status CopyFiles(const std::shared_ptr<FileSystem>& source_fs,

namespace {

Result<Uri> ParseFileSystemUri(const std::string& uri_string) {
 Uri uri;
 auto status = uri.Parse(uri_string);
 if (!status.ok()) {
#ifdef _WIN32
 // Could be a "file:..." URI with backslashes instead of regular slashes.
 RETURN_NOT_OK(uri.Parse(ToSlashes(uri_string)));
 if (uri.scheme() != "file") {
 return status;
 }
#else
 return status;
#endif
 }
 return std::move(uri);
}

Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
 const std::string& uri_string,
 const io::IOContext& io_context,
Expand DownExpand Up@@ -763,7 +759,8 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(
 if (internal::DetectAbsolutePath(uri_string)) {
 // Normalize path separators
 if (out_path != nullptr) {
 *out_path = ToSlashes(uri_string);
 *out_path =
 std::string(RemoveTrailingSlash(ToSlashes(uri_string), /*preserve_root=*/true));
 }
 return std::make_shared<LocalFileSystem>();
 }
Expand Down

22 changes: 22 additions & 0 deletions
cpp/src/arrow/filesystem/filesystem.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -171,6 +171,26 @@ class ARROW_EXPORT FileSystem : public std::enable_shared_from_this<FileSystem>
 /// may allow normalizing irregular path forms (such as Windows local paths).
 virtual Result<std::string> NormalizePath(std::string path);

 /// \brief Ensure a URI (or path) is compatible with the given filesystem and return the
 /// path
 ///
 /// \param uri_string A URI representing a resource in the given filesystem.
 ///
 /// This method will check to ensure the given filesystem is compatible with the
 /// URI. This can be useful when the user provides both a URI and a filesystem or
 /// when a user provides multiple URIs that should be compatible with the same
 /// filesystem.
 ///
 /// uri_string can be an absolute path instead of a URI. In that case it will ensure
 /// the filesystem (if supplied) is the local filesystem (or some custom filesystem that
 /// is capable of reading local paths) and will normalize the path's file separators.
 ///
 /// Note, this method only checks to ensure the URI scheme is valid. It will not detect
 /// inconsistencies like a mismatching region or endpoint override.
 ///
 /// \return The path inside the filesystem that is indicated by the URI.
 virtual Result<std::string> PathFromUri(const std::string& uri_string) const;

 virtual bool Equals(const FileSystem& other) const = 0;

 virtual bool Equals(const std::shared_ptr<FileSystem>& other) const {
Expand DownExpand Up@@ -336,6 +356,7 @@ class ARROW_EXPORT SubTreeFileSystem : public FileSystem {
 std::shared_ptr<FileSystem> base_fs() const { return base_fs_; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -410,6 +431,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {

 std::string type_name() const override { return "slow"; }
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 using FileSystem::GetFileInfo;
 Result<FileInfo> GetFileInfo(const std::string& path) override;
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -873,6 +873,12 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
 return impl_->options().Equals(fs.impl_->options());
}

Result<std::string> GcsFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"gs", "gcs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kPrepend);
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/gcsfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -178,6 +178,7 @@ class ARROW_EXPORT GcsFileSystem : public FileSystem {
 const GcsOptions& options() const;

 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 Result<FileInfo> GetFileInfo(const std::string& path) override;
 Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
Expand Down

17 changes: 15 additions & 2 deletions
cpp/src/arrow/filesystem/gcsfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -54,6 +54,7 @@
#include "arrow/filesystem/path_util.h"
#include "arrow/filesystem/test_util.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/testing/matchers.h"
#include "arrow/testing/util.h"
#include "arrow/util/future.h"
#include "arrow/util/key_value_metadata.h"
Expand DownExpand Up@@ -1383,12 +1384,24 @@ TEST_F(GcsIntegrationTest, OpenInputFileClosed) {

TEST_F(GcsIntegrationTest, TestFileSystemFromUri) {
 // Smoke test for FileSystemFromUri
 ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri(std::string("gs://anonymous@") +
 PreexistingBucketPath()));
 std::string path;
 ASSERT_OK_AND_ASSIGN(
 auto fs,
 FileSystemFromUri(std::string("gs://anonymous@") + PreexistingBucketPath(), &path));
 EXPECT_EQ(fs->type_name(), "gcs");
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(
 path, fs->PathFromUri(std::string("gs://anonymous@") + PreexistingBucketPath()));
 EXPECT_EQ(path, PreexistingBucketName());
 ASSERT_OK_AND_ASSIGN(auto fs2, FileSystemFromUri(std::string("gcs://anonymous@") +
 PreexistingBucketPath()));
 EXPECT_EQ(fs2->type_name(), "gcs");
 ASSERT_THAT(fs->PathFromUri("/foo/bar"),
 Raises(StatusCode::Invalid, testing::HasSubstr("Expected a URI")));
 ASSERT_THAT(
 fs->PathFromUri("s3:///foo/bar"),
 Raises(StatusCode::Invalid,
 testing::HasSubstr("expected a URI with one of the schemes (gs, gcs)")));
}

} // namespace
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -473,6 +473,12 @@ bool HadoopFileSystem::Equals(const FileSystem& other) const {
 return options().Equals(hdfs.options());
}

Result<std::string> HadoopFileSystem::PathFromUri(const std::string& uri_string) const {
 return internal::PathFromUriHelper(uri_string, {"hdfs", "viewfs"},
 /*accept_local_paths=*/false,
 internal::AuthorityHandlingBehavior::kIgnore);
}

Result<std::vector<FileInfo>> HadoopFileSystem::GetFileInfo(const FileSelector& select) {
 return impl_->GetFileInfo(select);
}
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/filesystem/hdfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,6 +66,7 @@ class ARROW_EXPORT HadoopFileSystem : public FileSystem {
 std::string type_name() const override { return "hdfs"; }
 HdfsOptions options() const;
 bool Equals(const FileSystem& other) const override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 /// \cond FALSE
 using FileSystem::GetFileInfo;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/filesystem/hdfs_test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,8 @@ class TestHadoopFileSystem : public ::testing::Test, public HadoopFileSystemTest
 ARROW_LOG(INFO) << "!!! uri = " << ss.str();
 ASSERT_OK_AND_ASSIGN(uri_fs, FileSystemFromUri(ss.str(), &path));
 ASSERT_EQ(path, "/");
 ASSERT_OK_AND_ASSIGN(path, uri_fs->PathFromUri(ss.str()));
 ASSERT_EQ(path, "/");

 // Sanity check
 ASSERT_OK(uri_fs->CreateDir("AB"));
Expand Down

57 changes: 21 additions & 36 deletions
cpp/src/arrow/filesystem/localfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -52,37 +52,6 @@ using ::arrow::internal::IOErrorFromWinError;
using ::arrow::internal::NativePathString;
using ::arrow::internal::PlatformFilename;

namespace internal {

#ifdef _WIN32
static bool IsDriveLetter(char c) {
 // Can't use locale-dependent functions from the C/C++ stdlib
 return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}
#endif

bool DetectAbsolutePath(const std::string& s) {
 // Is it a /-prefixed local path?
 if (s.length() >= 1 && s[0] == '/') {
 return true;
 }
#ifdef _WIN32
 // Is it a \-prefixed local path?
 if (s.length() >= 1 && s[0] == '\\') {
 return true;
 }
 // Does it start with a drive letter in addition to being /- or \-prefixed,
 // e.g. "C:\..."?
 if (s.length() >= 3 && s[1] == ':' && (s[2] == '/' || s[2] == '\\') &&
 IsDriveLetter(s[0])) {
 return true;
 }
#endif
 return false;
}

} // namespace internal

namespace {

Status ValidatePath(std::string_view s) {
Expand All@@ -92,6 +61,12 @@ Status ValidatePath(std::string_view s) {
 return Status::OK();
}

Result<std::string> DoNormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
}

#ifdef _WIN32

std::string NativeToString(const NativePathString& ns) {
Expand DownExpand Up@@ -263,13 +238,15 @@ Result<LocalFileSystemOptions> LocalFileSystemOptions::FromUri(
#ifdef _WIN32
 std::stringstream ss;
 ss << "//" << host << "/" << internal::RemoveLeadingSlash(uri.path());
 *out_path = ss.str();
 *out_path =
 std::string(internal::RemoveTrailingSlash(ss.str(), /*preserve_root=*/true));
#else
 return Status::Invalid("Unsupported hostname in non-Windows local URI: '",
 uri.ToString(), "'");
#endif
 } else {
 *out_path = uri.path();
 *out_path =
 std::string(internal::RemoveTrailingSlash(uri.path(), /*preserve_root=*/true));
 }

 // TODO handle use_mmap option
Expand All@@ -286,9 +263,17 @@ LocalFileSystem::LocalFileSystem(const LocalFileSystemOptions& options,
LocalFileSystem::~LocalFileSystem() {}

Result<std::string> LocalFileSystem::NormalizePath(std::string path) {
 RETURN_NOT_OK(ValidatePath(path));
 ARROW_ASSIGN_OR_RAISE(auto fn, PlatformFilename::FromString(path));
 return fn.ToString();
 return DoNormalizePath(std::move(path));
}

Result<std::string> LocalFileSystem::PathFromUri(const std::string& uri_string) const {
#ifdef _WIN32
 auto authority_handling = internal::AuthorityHandlingBehavior::kWindows;
#else
 auto authority_handling = internal::AuthorityHandlingBehavior::kDisallow;
#endif
 return internal::PathFromUriHelper(uri_string, {"file"}, /*accept_local_paths=*/true,
 authority_handling);
}

bool LocalFileSystem::Equals(const FileSystem& other) const {
Expand Down

9 changes: 1 addition & 8 deletions
cpp/src/arrow/filesystem/localfs.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -82,6 +82,7 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 std::string type_name() const override { return "local"; }

 Result<std::string> NormalizePath(std::string path) override;
 Result<std::string> PathFromUri(const std::string& uri_string) const override;

 bool Equals(const FileSystem& other) const override;

Expand DownExpand Up@@ -121,13 +122,5 @@ class ARROW_EXPORT LocalFileSystem : public FileSystem {
 LocalFileSystemOptions options_;
};

namespace internal {

// Return whether the string is detected as a local absolute path.
ARROW_EXPORT
bool DetectAbsolutePath(const std::string& s);

} // namespace internal

} // namespace fs
} // namespace arrow
Loading