GH-38309: [C++] build filesystems as separate modules

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs

cpp/examples/arrow/CMakeLists.txt

@@ -195,3 +195,13 @@ if(ARROW_GANDIVA)
  endif()
  add_arrow_example(gandiva_example EXTRA_LINK_LIBS ${GANDIVA_EXAMPLE_LINK_LIBS})
 endif()
+
+if(ARROW_FILESYSTEM)
+ add_library(filesystem_definition_example MODULE filesystem_definition_example.cc)
+ target_link_libraries(filesystem_definition_example ${ARROW_EXAMPLE_LINK_LIBS})
+
+ add_arrow_example(filesystem_usage_example)
+ target_compile_definitions(filesystem-usage-example
+ PUBLIC FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:filesystem_definition_example>"
+ )
+endif()

cpp/examples/arrow/filesystem_definition_example.cc

@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/filesystem/filesystem_library.h>
+#include <arrow/io/memory.h>
+#include <arrow/result.h>
+#include <arrow/util/uri.h>
+
+// Demonstrate registering a user-defined Arrow FileSystem outside
+// of the Arrow source tree.
+
+using arrow::Result;
+using arrow::Status;
+namespace io = arrow::io;
+namespace fs = arrow::fs;
+
+class ExampleFileSystem : public fs::FileSystem {
+ public:
+ explicit ExampleFileSystem(const io::IOContext& io_context)
+ : fs::FileSystem{io_context} {}
+
+ // This is a mock filesystem whose root directory contains a single file.
+ // All operations which would mutate will simply raise an error.
+ static constexpr std::string_view kPath = "example_file";
+ static constexpr std::string_view kContents = "hello world";
+ static fs::FileInfo info() {
+ fs::FileInfo info;
+ info.set_path(std::string{kPath});
+ info.set_type(fs::FileType::File);
+ info.set_size(kContents.size());
+ return info;
+ }
+
+ static Status NotFound(std::string_view path) {
+ return Status::IOError("Path does not exist '", path, "'");
+ }
+
+ static Status NoMutation() {
+ return Status::IOError("operations which would mutate are not permitted");
+ }
+
+ Result<std::string> PathFromUri(const std::string& uri_string) const override {
+ ARROW_ASSIGN_OR_RAISE(auto uri, arrow::util::Uri::FromString(uri_string));
+ return uri.path();
+ }
+
+ std::string type_name() const override { return "example"; }
+
+ bool Equals(const FileSystem& other) const override {
+ return type_name() == other.type_name();
+ }
+
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
+ Result<fs::FileInfo> GetFileInfo(const std::string& path) override {
+ if (path == kPath) {
+ return info();
+ }
+ return NotFound(path);
+ }
+
+ Result<std::vector<fs::FileInfo>> GetFileInfo(const fs::FileSelector& select) override {
+ if (select.base_dir == "/" || select.base_dir == "") {
+ return std::vector<fs::FileInfo>{info()};
+ }
+ if (select.allow_not_found) {
+ return std::vector<fs::FileInfo>{};
+ }
+ return NotFound(select.base_dir);
+ }
+
+ Status CreateDir(const std::string& path, bool recursive) override {
+ return NoMutation();
+ }
+
+ Status DeleteDir(const std::string& path) override { return NoMutation(); }
+
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override {
+ return NoMutation();
+ }
+
+ Status DeleteRootDirContents() override { return NoMutation(); }
+
+ Status DeleteFile(const std::string& path) override { return NoMutation(); }
+
+ Status Move(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Status CopyFile(const std::string& src, const std::string& dest) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::InputStream>> OpenInputStream(
+ const std::string& path) override {
+ return OpenInputFile(path);
+ }
+
+ Result<std::shared_ptr<io::RandomAccessFile>> OpenInputFile(
+ const std::string& path) override {
+ if (path == kPath) {
+ return io::BufferReader::FromString(std::string{kContents});
+ }
+ return NotFound(path);
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+
+ Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
+ const std::string& path,
+ const std::shared_ptr<const arrow::KeyValueMetadata>& metadata) override {
+ return NoMutation();
+ }
+};
+
+fs::FileSystemRegistrar kExampleFileSystemModule{
+ "example",
+ [](const arrow::util::Uri& uri, const io::IOContext& io_context,
+ std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
+ auto fs = std::make_shared<ExampleFileSystem>(io_context);
+ if (out_path) {
+ ARROW_ASSIGN_OR_RAISE(*out_path, fs->PathFromUri(uri.ToString()));
+ }
+ return fs;
+ },
+};

cpp/examples/arrow/filesystem_usage_example.cc

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <iostream>
+
+#include <arrow/filesystem/filesystem.h>
+#include <arrow/result.h>
+
+namespace fs = arrow::fs;
+
+// Demonstrate dynamically loading a user-defined Arrow FileSystem
+
+arrow::Status Execute() {
+ ARROW_RETURN_NOT_OK(arrow::fs::LoadFileSystemFactories(FILESYSTEM_EXAMPLE_LIBPATH));
+
+ std::string uri = "example:///example_file";
+ std::cout << "Uri: " << uri << std::endl;
+
+ std::string path;
+ ARROW_ASSIGN_OR_RAISE(auto fs, arrow::fs::FileSystemFromUri(uri, &path));
+ std::cout << "Path: " << path << std::endl;
+
+ fs::FileSelector sel;
+ sel.base_dir = "/";
+ ARROW_ASSIGN_OR_RAISE(auto infos, fs->GetFileInfo(sel));
+
+ std::cout << "Root directory contains:" << std::endl;
+ for (const auto& info : infos) {
+ std::cout << "- " << info << std::endl;
+ }
+ return arrow::Status::OK();
+}
+
+int main() {
+ auto status = Execute();
+ if (!status.ok()) {
+ std::cerr << "Error occurred : " << status.message() << std::endl;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}

cpp/src/arrow/dataset/partition.cc

@@ -53,7 +53,7 @@ namespace dataset {
 namespace {
 /// Apply UriUnescape, then ensure the results are valid UTF-8.
 Result<std::string> SafeUriUnescape(std::string_view encoded) {
- auto decoded = ::arrow::internal::UriUnescape(encoded);
+ auto decoded = ::arrow::util::UriUnescape(encoded);
  if (!util::ValidateUTF8(decoded)) {
  return Status::Invalid("Partition segment was not valid UTF-8 after URL decoding: ",
  encoded);
@@ -755,7 +755,7 @@ Result<PartitionPathFormat> HivePartitioning::FormatValues(
  // field_index <-> path nesting relation
  segments[i] = name + "=" + hive_options_.null_fallback;
  } else {
- segments[i] = name + "=" + arrow::internal::UriEscape(values[i]->ToString());
+ segments[i] = name + "=" + arrow::util::UriEscape(values[i]->ToString());
  }
  }
 

cpp/src/arrow/dataset/partition_test.cc

@@ -935,7 +935,7 @@ TEST_F(TestPartitioning, WriteHiveWithSlashesInValues) {
  "experiment/A/f.csv", "experiment/B/f.csv", "experiment/C/k.csv",
  "experiment/M/i.csv"};
  for (auto partition : unique_partitions) {
- encoded_paths.push_back("part=" + arrow::internal::UriEscape(partition));
+ encoded_paths.push_back("part=" + arrow::util::UriEscape(partition));
  }
 
  ASSERT_EQ(all_dirs.size(), encoded_paths.size());

cpp/src/arrow/engine/substrait/relation_internal.cc

@@ -67,7 +67,7 @@ namespace arrow {
 using internal::checked_cast;
 using internal::StartsWith;
 using internal::ToChars;
-using internal::UriFromAbsolutePath;
+using util::UriFromAbsolutePath;
 
 namespace engine {
 
@@ -463,7 +463,7 @@ Result<DeclarationInfo> FromProto(const substrait::Rel& rel, const ExtensionSet&
  }
 
  // Extract and parse the read relation's source URI
- ::arrow::internal::Uri item_uri;
+ ::arrow::util::Uri item_uri;
  switch (item.path_type_case()) {
  case substrait::ReadRel::LocalFiles::FileOrFiles::kUriPath:
  RETURN_NOT_OK(item_uri.Parse(item.uri_path()));

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -26,7 +26,9 @@ add_arrow_test(filesystem-test
  filesystem_test.cc
  localfs_test.cc
  EXTRA_LABELS
- filesystem)
+ filesystem
+ DEFINITIONS
+ ARROW_FILESYSTEM_EXAMPLE_LIBPATH="$<TARGET_FILE:arrow_filesystem_example>")
 
 if(ARROW_BUILD_BENCHMARKS)
  add_arrow_benchmark(localfs_benchmark

cpp/src/arrow/filesystem/azurefs.h

@@ -196,15 +196,23 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  bool Equals(const FileSystem& other) const override;
 
+ /// \cond FALSE
+ using FileSystem::CreateDir;
+ using FileSystem::DeleteDirContents;
+ using FileSystem::GetFileInfo;
+ using FileSystem::OpenAppendStream;
+ using FileSystem::OpenOutputStream;
+ /// \endcond
+
  Result<FileInfo> GetFileInfo(const std::string& path) override;
 
  Result<FileInfoVector> GetFileInfo(const FileSelector& select) override;
 
- Status CreateDir(const std::string& path, bool recursive = true) override;
+ Status CreateDir(const std::string& path, bool recursive) override;
 
  Status DeleteDir(const std::string& path) override;
 
- Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override;
+ Status DeleteDirContents(const std::string& path, bool missing_dir_ok) override;
 
  Status DeleteRootDirContents() override;
 
@@ -246,11 +254,11 @@ class ARROW_EXPORT AzureFileSystem : public FileSystem {
 
  Result<std::shared_ptr<io::OutputStream>> OpenOutputStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 
  Result<std::shared_ptr<io::OutputStream>> OpenAppendStream(
  const std::string& path,
- const std::shared_ptr<const KeyValueMetadata>& metadata = {}) override;
+ const std::shared_ptr<const KeyValueMetadata>& metadata) override;
 };
 
 } // namespace arrow::fs