ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
coryan wants to merge 10 commits into from
Closed

ARROW-14236: [C++] Add GCS testbench for testing #11375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c594032
ARROW-14236: [C++] add GCS testbench for testing
coryan Sep 30, 2021
085832b
Fix builds where git(1) is not installed ci
coryan Oct 9, 2021
fee487b
Address review comments
coryan Oct 11, 2021
df02bfe
Address review comments
coryan Oct 11, 2021
8f18306
Address review comments
coryan Oct 12, 2021
41ab3d7
Address review comments
coryan Oct 12, 2021
184af32
Address review comments
coryan Oct 12, 2021
9712d7a
Avoid using production endpoints in unit tests
coryan Oct 12, 2021
584cb2f
Preserve minio installs
coryan Oct 12, 2021
f3edc48
Fix formatting
coryan Oct 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ci/docker/conda.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
/arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down
6 changes: 4 additions & 2 deletions ci/docker/debian-10-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
ninja-build \
pkg-config \
protobuf-compiler \
python3-pip \
rapidjson-dev \
tzdata \
zlib1g-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
ARROW_DATASET=ON \
Expand Down
6 changes: 4 additions & 2 deletions ci/docker/debian-11-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
ninja-build \
pkg-config \
protobuf-compiler-grpc \
python3-pip \
rapidjson-dev \
tzdata \
zlib1g-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
ARROW_DATASET=ON \
Expand Down
6 changes: 4 additions & 2 deletions ci/docker/fedora-33-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
openssl-devel \
protobuf-devel \
python \
python-pip \
rapidjson-devel \
re2-devel \
snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
which \
zlib-devel

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down
5 changes: 3 additions & 2 deletions ci/docker/linux-apt-r.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down
1 change: 1 addition & 0 deletions ci/docker/linux-r.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down
6 changes: 4 additions & 2 deletions ci/docker/ubuntu-20.04-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
ninja-build \
pkg-config \
protobuf-compiler \
python3-pip \
rapidjson-dev \
tzdata \
wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down
6 changes: 4 additions & 2 deletions ci/docker/ubuntu-20.10-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
pkg-config \
protobuf-compiler \
protobuf-compiler-grpc \
python3-pip \
rapidjson-dev \
tzdata \
wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down
6 changes: 4 additions & 2 deletions ci/docker/ubuntu-21.04-cpp.dockerfile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
pkg-config \
protobuf-compiler \
protobuf-compiler-grpc \
python3-pip \
rapidjson-dev \
tzdata \
wget && \
apt-get clean && \
rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down
32 changes: 32 additions & 0 deletions ci/scripts/install_gcs_testbench.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
echo "Usage: $0 <storage-testbench version>"
exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"
4 changes: 4 additions & 0 deletions ci/scripts/r_docker_configure.sh
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
/arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
fi

if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
/arrow/ci/scripts/install_gcs_testbench.sh default
fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down
4 changes: 2 additions & 2 deletions cpp/src/arrow/CMakeLists.txt
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
filesystem/util_internal.cc)

if(ARROW_GCS)
list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
set_source_files_properties(filesystem/gcsfs.cc
list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
PROPERTIES SKIP_PRECOMPILE_HEADERS ON
SKIP_UNITY_BUILD_INCLUSION ON)
endif()
Expand Down
70 changes: 66 additions & 4 deletions cpp/src/arrow/filesystem/gcsfs.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
std::string full_path;
std::string bucket;
std::string object;

static Result<GcsPath> FromString(const std::string& s) {
const auto src = internal::RemoveTrailingSlash(s);
auto const first_sep = src.find_first_of(kSep);
if (first_sep == 0) {
return Status::Invalid("Path cannot start with a separator ('", s, "')");
}
if (first_sep == std::string::npos) {
return GcsPath{std::string(src), std::string(src), ""};
}
GcsPath path;
path.full_path = std::string(src);
path.bucket = std::string(src.substr(0, first_sep));
path.object = std::string(src.substr(first_sep + 1));
return path;
}

bool empty() const { return bucket.empty() && object.empty(); }

bool operator==(const GcsPath& other) const {
return bucket == other.bucket && object == other.object;
}
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
if (!o.endpoint_override.empty()) {
std::string scheme = o.scheme;
if (scheme.empty()) scheme = "https";
if (scheme == "https") {
options.set<google::cloud::UnifiedCredentialsOption>(
google::cloud::MakeGoogleDefaultCredentials());
} else {
options.set<google::cloud::UnifiedCredentialsOption>(
google::cloud::MakeInsecureCredentials());
}
options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
}
return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
explicit Impl(GcsOptions o)
: options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

GcsOptions const& options() const { return options_; }
const GcsOptions& options() const { return options_; }

Result<FileInfo> GetFileInfo(const GcsPath& path) {
if (!path.object.empty()) {
auto meta = client_.GetObjectMetadata(path.bucket, path.object);
return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
}
auto meta = client_.GetBucketMetadata(path.bucket);
return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
}

private:
static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
const google::cloud::Status& status,
FileType type) {
if (status.ok()) {
return FileInfo(path.full_path, type);
}
using ::google::cloud::StatusCode;
if (status.code() == StatusCode::kNotFound) {
return FileInfo(path.full_path, FileType::NotFound);
}
return internal::ToArrowStatus(status);
}

GcsOptions options_;
gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
return Status::NotImplemented("The GCS FileSystem is not fully implemented");
ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down
69 changes: 69 additions & 0 deletions cpp/src/arrow/filesystem/gcsfs_internal.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
std::ostringstream os;
os << "google::cloud::Status(" << s << ")";
switch (s.code()) {
case google::cloud::StatusCode::kOk:
break;
case google::cloud::StatusCode::kCancelled:
return Status::Cancelled(os.str());
case google::cloud::StatusCode::kUnknown:
return Status::UnknownError(os.str());
case google::cloud::StatusCode::kInvalidArgument:
return Status::Invalid(os.str());
case google::cloud::StatusCode::kDeadlineExceeded:
return Status::IOError(os.str());
case google::cloud::StatusCode::kNotFound:
// TODO: it is unclear if a better mapping would be possible.

@pitroupitrouOct 12, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).

@coryancoryanOct 12, 2021

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

return Status::UnknownError(os.str());
case google::cloud::StatusCode::kAlreadyExists:
return Status::AlreadyExists(os.str());
case google::cloud::StatusCode::kPermissionDenied:
case google::cloud::StatusCode::kUnauthenticated:
return Status::IOError(os.str());
case google::cloud::StatusCode::kResourceExhausted:
return Status::CapacityError(os.str());
case google::cloud::StatusCode::kFailedPrecondition:
case google::cloud::StatusCode::kAborted:
return Status::IOError(os.str());
case google::cloud::StatusCode::kOutOfRange:
return Status::Invalid(os.str());
case google::cloud::StatusCode::kUnimplemented:
return Status::NotImplemented(os.str());
case google::cloud::StatusCode::kInternal:
case google::cloud::StatusCode::kUnavailable:
case google::cloud::StatusCode::kDataLoss:
return Status::IOError(os.str());
}
return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-14236: [C++] Add GCS testbench for testing#11375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

coryan
wants to merge
10
commits into

from

 Closed


ARROW-14236: [C++] Add GCS testbench for testing
#11375
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


10 commits


Select commit
Hold shift + click to select a range
c594032

ARROW-14236: [C++] add GCS testbench for testing


coryan Sep 30, 2021
085832b

Fix builds where git(1) is not installed ci


coryan Oct 9, 2021
fee487b

Address review comments


coryan Oct 11, 2021
df02bfe

Address review comments


coryan Oct 11, 2021
8f18306

Address review comments


coryan Oct 12, 2021
41ab3d7

Address review comments


coryan Oct 12, 2021
184af32

Address review comments


coryan Oct 12, 2021
9712d7a

Avoid using production endpoints in unit tests


coryan Oct 12, 2021
584cb2f

Preserve minio installs


coryan Oct 12, 2021
f3edc48

Fix formatting


coryan Oct 12, 2021

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 2 additions & 0 deletions
ci/docker/conda.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \
 /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix}
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix}
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# create a conda environment
ADD ci/conda_env_unix.txt /arrow/ci/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -65,15 +65,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/debian-11-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -63,15 +63,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 zlib1g-dev && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DATASET=ON \
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/fedora-33-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ RUN dnf update -y && \
 openssl-devel \
 protobuf-devel \
 python \
 python-pip \
 rapidjson-devel \
 re2-devel \
 snappy-devel \
Expand All@@ -62,9 +63,10 @@ RUN dnf update -y && \
 which \
 zlib-devel

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ENV ARROW_BUILD_TESTS=ON \
 ARROW_DEPENDENCY_SOURCE=SYSTEM \
Expand Down

5 changes: 3 additions & 2 deletions
ci/docker/linux-apt-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
COPY r/DESCRIPTION /arrow/r/
RUN /arrow/ci/scripts/r_deps.sh /arrow

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
Expand Down

1 change: 1 addition & 0 deletions
ci/docker/linux-r.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
COPY ci/etc/rprofile /arrow/ci/etc/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh

COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 ninja-build \
 pkg-config \
 protobuf-compiler \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-20.10-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -90,15 +90,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

6 changes: 4 additions & 2 deletions
ci/docker/ubuntu-21.04-cpp.dockerfile

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -88,15 +88,17 @@ RUN apt-get update -y -q && \
 pkg-config \
 protobuf-compiler \
 protobuf-compiler-grpc \
 python3-pip \
 rapidjson-dev \
 tzdata \
 wget && \
 apt-get clean && \
 rm -rf /var/lib/apt/lists*

COPY ci/scripts/install_minio.sh \
 /arrow/ci/scripts/
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/install_gcs_testbench.sh default

# Prioritize system packages and local installation
# The following dependencies will be downloaded due to missing/invalid packages
Expand Down

32 changes: 32 additions & 0 deletions
ci/scripts/install_gcs_testbench.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 1 ]; then
 echo "Usage: $0 <storage-testbench version>"
 exit 1
fi

version=$1
if [[ "${version}" -eq "default" ]]; then
 version="v0.7.0"
fi

pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz"

4 changes: 4 additions & 0 deletions
ci/scripts/r_docker_configure.sh

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
 if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
 /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local
 fi

 if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then
 /arrow/ci/scripts/install_gcs_testbench.sh default
 fi
fi

# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Expand Down

4 changes: 2 additions & 2 deletions
cpp/src/arrow/CMakeLists.txt

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM)
 filesystem/util_internal.cc)

 if(ARROW_GCS)
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc)
 set_source_files_properties(filesystem/gcsfs.cc
 list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
 set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
 PROPERTIES SKIP_PRECOMPILE_HEADERS ON
 SKIP_UNITY_BUILD_INCLUSION ON)
 endif()
Expand Down

70 changes: 66 additions & 4 deletions
cpp/src/arrow/filesystem/gcsfs.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -19,14 +19,46 @@

#include <google/cloud/storage/client.h>

#include <sstream>

#include "arrow/filesystem/gcsfs_internal.h"
#include "arrow/filesystem/path_util.h"
#include "arrow/result.h"
#include "arrow/util/checked_cast.h"

namespace arrow {
namespace fs {
namespace {

auto constexpr kSep = '/';

struct GcsPath {
 std::string full_path;
 std::string bucket;
 std::string object;

 static Result<GcsPath> FromString(const std::string& s) {
 const auto src = internal::RemoveTrailingSlash(s);
 auto const first_sep = src.find_first_of(kSep);
 if (first_sep == 0) {
 return Status::Invalid("Path cannot start with a separator ('", s, "')");
 }
 if (first_sep == std::string::npos) {
 return GcsPath{std::string(src), std::string(src), ""};
 }
 GcsPath path;
 path.full_path = std::string(src);
 path.bucket = std::string(src.substr(0, first_sep));
 path.object = std::string(src.substr(first_sep + 1));
 return path;
 }

 bool empty() const { return bucket.empty() && object.empty(); }

 bool operator==(const GcsPath& other) const {
 return bucket == other.bucket && object == other.object;
 }
};

} // namespace

namespace gcs = google::cloud::storage;

Expand All@@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) {
 if (!o.endpoint_override.empty()) {
 std::string scheme = o.scheme;
 if (scheme.empty()) scheme = "https";
 if (scheme == "https") {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeGoogleDefaultCredentials());
 } else {
 options.set<google::cloud::UnifiedCredentialsOption>(
 google::cloud::MakeInsecureCredentials());
 }
 options.set<gcs::RestEndpointOption>(scheme + "://" + o.endpoint_override);
 }
 return options;
Expand All@@ -45,9 +84,31 @@ class GcsFileSystem::Impl {
 explicit Impl(GcsOptions o)
 : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {}

 GcsOptions const& options() const { return options_; }
 const GcsOptions& options() const { return options_; }

 Result<FileInfo> GetFileInfo(const GcsPath& path) {
 if (!path.object.empty()) {
 auto meta = client_.GetObjectMetadata(path.bucket, path.object);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::File);
 }
 auto meta = client_.GetBucketMetadata(path.bucket);
 return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory);
 }

 private:
 static Result<FileInfo> GetFileInfoImpl(const GcsPath& path,
 const google::cloud::Status& status,
 FileType type) {
 if (status.ok()) {
 return FileInfo(path.full_path, type);
 }
 using ::google::cloud::StatusCode;
 if (status.code() == StatusCode::kNotFound) {
 return FileInfo(path.full_path, FileType::NotFound);
 }
 return internal::ToArrowStatus(status);
 }

 GcsOptions options_;
 gcs::Client client_;
};
Expand All@@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const {
}

Result<FileInfo> GcsFileSystem::GetFileInfo(const std::string& path) {
 return Status::NotImplemented("The GCS FileSystem is not fully implemented");
 ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path));
 return impl_->GetFileInfo(p);
}

Result<FileInfoVector> GcsFileSystem::GetFileInfo(const FileSelector& select) {
Expand Down

69 changes: 69 additions & 0 deletions
cpp/src/arrow/filesystem/gcsfs_internal.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/filesystem/gcsfs_internal.h"

#include <google/cloud/storage/client.h>

#include <sstream>

namespace arrow {
namespace fs {
namespace internal {

Status ToArrowStatus(const google::cloud::Status& s) {
 std::ostringstream os;
 os << "google::cloud::Status(" << s << ")";
 switch (s.code()) {
 case google::cloud::StatusCode::kOk:
 break;
 case google::cloud::StatusCode::kCancelled:
 return Status::Cancelled(os.str());
 case google::cloud::StatusCode::kUnknown:
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kInvalidArgument:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kDeadlineExceeded:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kNotFound:
 // TODO: it is unclear if a better mapping would be possible.
@pitroupitrouOct 12, 2021

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would suggest IOError for all external errors (such as kDeadlineExceeded, kPermissionDenied, kUnauthenticated...).
@coryancoryanOct 12, 2021

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.
 return Status::UnknownError(os.str());
 case google::cloud::StatusCode::kAlreadyExists:
 return Status::AlreadyExists(os.str());
 case google::cloud::StatusCode::kPermissionDenied:
 case google::cloud::StatusCode::kUnauthenticated:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kResourceExhausted:
 return Status::CapacityError(os.str());
 case google::cloud::StatusCode::kFailedPrecondition:
 case google::cloud::StatusCode::kAborted:
 return Status::IOError(os.str());
 case google::cloud::StatusCode::kOutOfRange:
 return Status::Invalid(os.str());
 case google::cloud::StatusCode::kUnimplemented:
 return Status::NotImplemented(os.str());
 case google::cloud::StatusCode::kInternal:
 case google::cloud::StatusCode::kUnavailable:
 case google::cloud::StatusCode::kDataLoss:
 return Status::IOError(os.str());
 }
 return Status::OK();
}

} // namespace internal
} // namespace fs
} // namespace arrow
Loading